From ca537e439d59ea34e640e7664a1efd98c7e1ef17 Mon Sep 17 00:00:00 2001 From: Daniel Huigens Date: Thu, 5 Jul 2018 13:44:33 +0200 Subject: [PATCH] Comments & code style --- src/cleartext.js | 5 +- src/crypto/public_key/elliptic/ecdsa.js | 2 + src/crypto/public_key/elliptic/eddsa.js | 2 + src/crypto/signature.js | 2 + src/encoding/armor.js | 19 +- src/encoding/base64.js | 8 +- src/key.js | 5 +- src/keyring/keyring.js | 2 + src/keyring/localstore.js | 4 + src/message.js | 14 +- src/openpgp.js | 78 ++++--- src/packet/compressed.js | 6 +- src/packet/literal.js | 14 +- src/packet/packet.js | 2 +- src/packet/packetlist.js | 2 +- src/packet/sym_encrypted_aead_protected.js | 10 +- .../sym_encrypted_integrity_protected.js | 2 + src/signature.js | 8 +- src/stream.js | 208 +++++++++++++++--- src/util.js | 82 +++---- test/general/streaming.js | 50 ++--- 21 files changed, 354 insertions(+), 171 deletions(-) diff --git a/src/cleartext.js b/src/cleartext.js index 56b5a272..40a290f4 100644 --- a/src/cleartext.js +++ b/src/cleartext.js @@ -129,7 +129,7 @@ CleartextMessage.prototype.getText = function() { /** * Returns ASCII armored text of cleartext signed message - * @returns {String} ASCII armor + * @returns {String | ReadableStream} ASCII armor */ CleartextMessage.prototype.armor = function() { let hashes = this.signature.packets.map(function(packet) { @@ -147,8 +147,9 @@ CleartextMessage.prototype.armor = function() { /** * reads an OpenPGP cleartext signed message and returns a CleartextMessage object - * @param {String} armoredText text to be parsed + * @param {String | ReadableStream} armoredText text to be parsed * @returns {module:cleartext.CleartextMessage} new cleartext message object + * @async * @static */ export async function readArmored(armoredText) { diff --git a/src/crypto/public_key/elliptic/ecdsa.js b/src/crypto/public_key/elliptic/ecdsa.js index 8330da9f..13c87abe 100644 --- a/src/crypto/public_key/elliptic/ecdsa.js +++ b/src/crypto/public_key/elliptic/ecdsa.js @@ -29,6 +29,7 @@ import Curve from './curves'; * @param {module:enums.hash} hash_algo Hash algorithm used to sign * @param {Uint8Array} m Message to sign * @param {Uint8Array} d Private key used to sign the message + * @param {Uint8Array} hashed The hashed message * @returns {{r: Uint8Array, * s: Uint8Array}} Signature of the message * @async @@ -49,6 +50,7 @@ async function sign(oid, hash_algo, m, d, hashed) { s: Uint8Array}} signature Signature to verify * @param {Uint8Array} m Message to verify * @param {Uint8Array} Q Public key used to verify the message + * @param {Uint8Array} hashed The hashed message * @returns {Boolean} * @async */ diff --git a/src/crypto/public_key/elliptic/eddsa.js b/src/crypto/public_key/elliptic/eddsa.js index 4e99f637..0b09b036 100644 --- a/src/crypto/public_key/elliptic/eddsa.js +++ b/src/crypto/public_key/elliptic/eddsa.js @@ -29,6 +29,7 @@ import Curve from './curves'; * @param {module:enums.hash} hash_algo Hash algorithm used to sign * @param {Uint8Array} m Message to sign * @param {Uint8Array} d Private key used to sign + * @param {Uint8Array} hashed The hashed message * @returns {{R: Uint8Array, * S: Uint8Array}} Signature of the message * @async @@ -50,6 +51,7 @@ async function sign(oid, hash_algo, m, d, hashed) { S: Uint8Array}} signature Signature to verify the message * @param {Uint8Array} m Message to verify * @param {Uint8Array} Q Public key used to verify the message + * @param {Uint8Array} hashed The hashed message * @returns {Boolean} * @async */ diff --git a/src/crypto/signature.js b/src/crypto/signature.js index 5e825e28..9b13d944 100644 --- a/src/crypto/signature.js +++ b/src/crypto/signature.js @@ -25,6 +25,7 @@ export default { * @param {Array} msg_MPIs Algorithm-specific signature parameters * @param {Array} pub_MPIs Algorithm-specific public key parameters * @param {Uint8Array} data Data for which the signature was created + * @param {Uint8Array} hashed The hashed data * @returns {Boolean} True if signature is valid * @async */ @@ -78,6 +79,7 @@ export default { * @param {module:enums.hash} hash_algo Hash algorithm * @param {Array} key_params Algorithm-specific public and private key parameters * @param {Uint8Array} data Data to be signed + * @param {Uint8Array} hashed The hashed data * @returns {Uint8Array} Signature * @async */ diff --git a/src/encoding/armor.js b/src/encoding/armor.js index 48c3caaa..622996de 100644 --- a/src/encoding/armor.js +++ b/src/encoding/armor.js @@ -118,19 +118,14 @@ function addheader(customComment) { /** * Calculates a checksum over the given data and returns it base64 encoded - * @param {String} data Data to create a CRC-24 checksum for - * @returns {String} Base64 encoded checksum + * @param {String | ReadableStream} data Data to create a CRC-24 checksum for + * @returns {String | ReadableStream} Base64 encoded checksum */ function getCheckSum(data) { const crc = createcrc24(data); return base64.encode(crc); } -/** - * Internal function to calculate a CRC-24 checksum over a given string (data) - * @param {String} data Data to create a CRC-24 checksum for - * @returns {Integer} The CRC-24 checksum as number - */ const crc_table = [ 0x00000000, 0x00864cfb, 0x018ad50d, 0x010c99f6, 0x0393e6e1, 0x0315aa1a, 0x021933ec, 0x029f7f17, 0x07a18139, 0x0727cdc2, 0x062b5434, 0x06ad18cf, 0x043267d8, 0x04b42b23, 0x05b8b2d5, 0x053efe2e, 0x0fc54e89, 0x0f430272, @@ -166,6 +161,11 @@ const crc_table = [ 0x57dd8538 ]; +/** + * Internal function to calculate a CRC-24 checksum over a given string (data) + * @param {String | ReadableStream} data Data to create a CRC-24 checksum for + * @returns {Uint8Array | ReadableStream} The CRC-24 checksum + */ function createcrc24(input) { let crc = 0xB704CE; return stream.transform(input, value => { @@ -197,7 +197,8 @@ function verifyHeaders(headers) { * the encoded bytes * @param {String} text OpenPGP armored message * @returns {Object} An object with attribute "text" containing the message text, - * an attribute "data" containing the bytes and "type" for the ASCII armor type + * an attribute "data" containing a stream of bytes and "type" for the ASCII armor type + * @async * @static */ function dearmor(input) { @@ -310,7 +311,7 @@ function dearmor(input) { * @param {Integer} partindex * @param {Integer} parttotal * @param {String} customComment (optional) additional comment to add to the armored string - * @returns {String} Armored text + * @returns {String | ReadableStream} Armored text * @static */ function armor(messagetype, body, partindex, parttotal, customComment) { diff --git a/src/encoding/base64.js b/src/encoding/base64.js index 78a54663..b6071bb6 100644 --- a/src/encoding/base64.js +++ b/src/encoding/base64.js @@ -23,9 +23,9 @@ const b64u = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'; /** * Convert binary array to radix-64 - * @param {Uint8Array} t Uint8Array to convert + * @param {Uint8Array | ReadableStream} t Uint8Array to convert * @param {bool} u if true, output is URL-safe - * @returns {string} radix-64 version of input string + * @returns {String | ReadableStream} radix-64 version of input string * @static */ function s2r(t, u = false) { @@ -92,9 +92,9 @@ function s2r(t, u = false) { /** * Convert radix-64 to binary array - * @param {String} t radix-64 string to convert + * @param {String | ReadableStream} t radix-64 string to convert * @param {bool} u if true, input is interpreted as URL-safe - * @returns {Uint8Array} binary array version of input string + * @returns {Uint8Array | ReadableStream} binary array version of input string * @static */ function r2s(t, u) { diff --git a/src/key.js b/src/key.js index 8465b836..5e8e92de 100644 --- a/src/key.js +++ b/src/key.js @@ -248,7 +248,7 @@ Key.prototype.toPublic = function() { /** * Returns ASCII armored text of key - * @returns {String} ASCII armor + * @returns {ReadableStream} ASCII armor */ Key.prototype.armor = function() { const type = this.isPublic() ? enums.armor.public_key : enums.armor.private_key; @@ -1207,9 +1207,10 @@ export async function read(data) { /** * Reads an OpenPGP armored text and returns one or multiple key objects - * @param {String} armoredText text to be parsed + * @param {String | ReadableStream} armoredText text to be parsed * @returns {{keys: Array, * err: (Array|null)}} result object with key and error arrays + * @async * @static */ export async function readArmored(armoredText) { diff --git a/src/keyring/keyring.js b/src/keyring/keyring.js index adad4d73..9a6dd09d 100644 --- a/src/keyring/keyring.js +++ b/src/keyring/keyring.js @@ -36,6 +36,7 @@ function Keyring(storeHandler) { /** * Calls the storeHandler to load the keys + * @async */ Keyring.prototype.load = async function () { this.publicKeys = new KeyArray(await this.storeHandler.loadPublic()); @@ -44,6 +45,7 @@ Keyring.prototype.load = async function () { /** * Calls the storeHandler to save the keys + * @async */ Keyring.prototype.store = async function () { await Promise.all([ diff --git a/src/keyring/localstore.js b/src/keyring/localstore.js index a984ece2..eeb0ff4c 100644 --- a/src/keyring/localstore.js +++ b/src/keyring/localstore.js @@ -55,6 +55,7 @@ LocalStore.prototype.privateKeysItem = 'private-keys'; /** * Load the public keys from HTML5 local storage. * @returns {Array} array of keys retrieved from localstore + * @async */ LocalStore.prototype.loadPublic = async function () { return loadKeys(this.storage, this.publicKeysItem); @@ -63,6 +64,7 @@ LocalStore.prototype.loadPublic = async function () { /** * Load the private keys from HTML5 local storage. * @returns {Array} array of keys retrieved from localstore + * @async */ LocalStore.prototype.loadPrivate = async function () { return loadKeys(this.storage, this.privateKeysItem); @@ -89,6 +91,7 @@ async function loadKeys(storage, itemname) { * Saves the current state of the public keys to HTML5 local storage. * The key array gets stringified using JSON * @param {Array} keys array of keys to save in localstore + * @async */ LocalStore.prototype.storePublic = async function (keys) { await storeKeys(this.storage, this.publicKeysItem, keys); @@ -98,6 +101,7 @@ LocalStore.prototype.storePublic = async function (keys) { * Saves the current state of the private keys to HTML5 local storage. * The key array gets stringified using JSON * @param {Array} keys array of keys to save in localstore + * @async */ LocalStore.prototype.storePrivate = async function (keys) { await storeKeys(this.storage, this.privateKeysItem, keys); diff --git a/src/message.js b/src/message.js index c85b2658..e4d9f294 100644 --- a/src/message.js +++ b/src/message.js @@ -667,7 +667,7 @@ Message.prototype.appendSignature = async function(detachedSignature) { /** * Returns ASCII armored text of message - * @returns {String} ASCII armor + * @returns {ReadableStream} ASCII armor */ Message.prototype.armor = function() { return armor.encode(enums.armor.message, this.packets.write()); @@ -675,8 +675,9 @@ Message.prototype.armor = function() { /** * reads an OpenPGP armored message and returns a message object - * @param {String} armoredText text to be parsed + * @param {String | ReadableStream} armoredText text to be parsed * @returns {module:message.Message} new message object + * @async * @static */ export async function readArmored(armoredText) { @@ -688,9 +689,10 @@ export async function readArmored(armoredText) { /** * reads an OpenPGP message as byte array and returns a message object - * @param {Uint8Array} input binary message + * @param {Uint8Array | ReadableStream} input binary message * @param {Boolean} fromStream whether the message was created from a Stream - * @returns {Message} new message object + * @returns {module:message.Message} new message object + * @async * @static */ export async function read(input, fromStream) { @@ -703,7 +705,7 @@ export async function read(input, fromStream) { /** * creates new message object from text - * @param {String} text + * @param {String | ReadableStream} text * @param {String} filename (optional) * @param {Date} date (optional) * @param {utf8|binary|text|mime} type (optional) data packet type @@ -726,7 +728,7 @@ export function fromText(text, filename, date=new Date(), type='utf8') { /** * creates new message object from binary data - * @param {Uint8Array} bytes + * @param {Uint8Array | ReadableStream} bytes * @param {String} filename (optional) * @param {Date} date (optional) * @param {utf8|binary|text|mime} type (optional) data packet type diff --git a/src/openpgp.js b/src/openpgp.js index 5f6dc374..7bcbfba5 100644 --- a/src/openpgp.js +++ b/src/openpgp.js @@ -362,27 +362,8 @@ export function decrypt({ message, privateKeys, passwords, sessionKeys, publicKe const result = {}; result.signatures = signature ? await decrypted.verifyDetached(signature, publicKeys, date) : await decrypted.verify(publicKeys, date, asStream); result.data = format === 'binary' ? decrypted.getLiteralData() : decrypted.getText(); - result.data = await convertStream(result.data, asStream); - if (asStream) { - result.data = stream.transformPair(message.packets.stream, async (readable, writable) => { - await stream.pipe(result.data, writable, { - preventClose: true - }); - const writer = stream.getWriter(writable); - try { - await stream.readToEnd(decrypted.packets.stream, arr => arr); - await writer.close(); - } catch(e) { - await writer.abort(e); - } - }); - } else { - await Promise.all(result.signatures.map(async signature => { - signature.signature = await signature.signature; - signature.valid = await signature.verified; - })); - } result.filename = decrypted.getFilename(); + await postProcess(result, asStream, message, decrypted.packets.stream); return result; }).catch(onError.bind(null, 'Error decrypting message')); } @@ -461,26 +442,7 @@ export function verify({ message, publicKeys, asStream=message&&message.fromStre const result = {}; result.signatures = signature ? await message.verifyDetached(signature, publicKeys, date) : await message.verify(publicKeys, date, asStream); result.data = message instanceof CleartextMessage ? message.getText() : message.getLiteralData(); - result.data = await convertStream(result.data, asStream); - if (asStream) { - result.data = stream.transformPair(message.packets.stream, async (readable, writable) => { - await stream.pipe(result.data, writable, { - preventClose: true - }); - const writer = stream.getWriter(writable); - try { - await stream.readToEnd(readable, arr => arr); - await writer.close(); - } catch(e) { - await writer.abort(e); - } - }); - } else { - await Promise.all(result.signatures.map(async signature => { - signature.signature = await signature.signature; - signature.valid = await signature.verified; - })); - } + await postProcess(result, asStream, message); return result; }).catch(onError.bind(null, 'Error verifying cleartext signed message')); } @@ -633,6 +595,42 @@ async function convertStreams(obj, asStream, keys=[]) { return obj; } +/** + * Post process the result of decrypt() and verify() before returning. + * See comments in the function body for more details. + * @param {Object} result the data to convert + * @param {Boolean} asStream whether to return ReadableStreams + * @param {Message} message message object + * @param {ReadableStream} errorStream (optional) stream which either errors or gets closed without data + * @returns {Object} the data in the respective format + */ +async function postProcess(result, asStream, message, errorStream) { + // Convert result.data to a stream or Uint8Array depending on asStream + result.data = await convertStream(result.data, asStream); + if (asStream) { + // Link result.data to the message stream for cancellation + result.data = stream.transformPair(message.packets.stream, async (readable, writable) => { + await stream.pipe(result.data, writable, { + preventClose: true + }); + const writer = stream.getWriter(writable); + try { + // Forward errors in errorStream (defaults to the message stream) to result.data + await stream.readToEnd(errorStream || readable, arr => arr); + await writer.close(); + } catch(e) { + await writer.abort(e); + } + }); + } else { + // Convert signature promises to values + await Promise.all(result.signatures.map(async signature => { + signature.signature = await signature.signature; + signature.valid = await signature.verified; + })); + } +} + /** * Global error handler that logs the stack trace and rethrows a high lvl error message. diff --git a/src/packet/compressed.js b/src/packet/compressed.js index 84dcacaa..f4c8948a 100644 --- a/src/packet/compressed.js +++ b/src/packet/compressed.js @@ -60,14 +60,14 @@ function Compressed() { /** * Compressed packet data - * @type {String} + * @type {Uint8Array | ReadableStream} */ this.compressed = null; } /** * Parsing function for the packet. - * @param {String} bytes Payload of a tag 8 packet + * @param {Uint8Array | ReadableStream} bytes Payload of a tag 8 packet */ Compressed.prototype.read = async function (bytes) { await stream.parse(bytes, async reader => { @@ -85,7 +85,7 @@ Compressed.prototype.read = async function (bytes) { /** * Return the compressed packet. - * @returns {String} binary compressed packet + * @returns {Uint8Array | ReadableStream} binary compressed packet */ Compressed.prototype.write = function () { if (this.compressed === null) { diff --git a/src/packet/literal.js b/src/packet/literal.js index a9e7c343..5ba46026 100644 --- a/src/packet/literal.js +++ b/src/packet/literal.js @@ -47,7 +47,7 @@ function Literal(date=new Date()) { /** * Set the packet data to a javascript native string, end of line * will be normalized to \r\n and by default text is converted to UTF8 - * @param {String} text Any native javascript string + * @param {String | ReadableStream} text Any native javascript string * @param {utf8|binary|text|mime} format (optional) The format of the string of bytes */ Literal.prototype.setText = function(text, format='utf8') { @@ -59,7 +59,8 @@ Literal.prototype.setText = function(text, format='utf8') { /** * Returns literal data packets as native JavaScript string * with normalized end of line to \n - * @returns {String} literal data as text + * @param {Boolean} clone (optional) Whether to return a clone so that getBytes/getText can be called again + * @returns {String | ReadableStream} literal data as text */ Literal.prototype.getText = function(clone=false) { if (this.text === null || util.isStream(this.text)) { // Assume that this.text has been read @@ -86,7 +87,7 @@ Literal.prototype.getText = function(clone=false) { /** * Set the packet data to value represented by the provided string of bytes. - * @param {Uint8Array} bytes The string of bytes + * @param {Uint8Array | ReadableStream} bytes The string of bytes * @param {utf8|binary|text|mime} format The format of the string of bytes */ Literal.prototype.setBytes = function(bytes, format) { @@ -98,7 +99,8 @@ Literal.prototype.setBytes = function(bytes, format) { /** * Get the byte sequence representing the literal packet data - * @returns {Uint8Array} A sequence of bytes + * @param {Boolean} clone (optional) Whether to return a clone so that getBytes/getText can be called again + * @returns {Uint8Array | ReadableStream} A sequence of bytes */ Literal.prototype.getBytes = function(clone=false) { if (this.data === null) { @@ -135,7 +137,7 @@ Literal.prototype.getFilename = function() { /** * Parsing function for a literal data packet (tag 11). * - * @param {Uint8Array} input Payload of a tag 11 packet + * @param {Uint8Array | ReadableStream} input Payload of a tag 11 packet * @returns {module:packet.Literal} object representation */ Literal.prototype.read = async function(bytes) { @@ -157,7 +159,7 @@ Literal.prototype.read = async function(bytes) { /** * Creates a string representation of the packet * - * @returns {Uint8Array} Uint8Array representation of the packet + * @returns {Uint8Array | ReadableStream} Uint8Array representation of the packet */ Literal.prototype.write = function() { const filename = util.str_to_Uint8Array(util.encode_utf8(this.filename)); diff --git a/src/packet/packet.js b/src/packet/packet.js index 5eeb11e4..181941f2 100644 --- a/src/packet/packet.js +++ b/src/packet/packet.js @@ -133,7 +133,7 @@ export default { /** * Generic static Packet Parser function * - * @param {Uint8Array} input Input stream as string + * @param {Uint8Array | ReadableStream} input Input stream as string * @param {Function} callback Function to call with the parsed packet * @returns {Boolean} Returns false if the stream was empty and parsing is done, and true otherwise. */ diff --git a/src/packet/packetlist.js b/src/packet/packetlist.js index 40eeeb5e..bb311b61 100644 --- a/src/packet/packetlist.js +++ b/src/packet/packetlist.js @@ -33,7 +33,7 @@ function List() { /** * Reads a stream of binary data and interprents it as a list of packets. - * @param {Uint8Array} A Uint8Array of bytes. + * @param {Uint8Array | ReadableStream} A Uint8Array of bytes. */ List.prototype.read = async function (bytes) { this.stream = stream.transformPair(bytes, async (readable, writable) => { diff --git a/src/packet/sym_encrypted_aead_protected.js b/src/packet/sym_encrypted_aead_protected.js index 785d8cd7..8ea728a9 100644 --- a/src/packet/sym_encrypted_aead_protected.js +++ b/src/packet/sym_encrypted_aead_protected.js @@ -56,6 +56,7 @@ export default SymEncryptedAEADProtected; /** * Parse an encrypted payload of bytes in the order: version, IV, ciphertext (see specification) + * @param {Uint8Array | ReadableStream} bytes */ SymEncryptedAEADProtected.prototype.read = async function (bytes) { await stream.parse(bytes, async reader => { @@ -77,7 +78,7 @@ SymEncryptedAEADProtected.prototype.read = async function (bytes) { /** * Write the encrypted payload of bytes in the order: version, IV, ciphertext (see specification) - * @returns {Uint8Array} The encrypted payload + * @returns {Uint8Array | ReadableStream} The encrypted payload */ SymEncryptedAEADProtected.prototype.write = function () { if (config.aead_protect_version === 4) { @@ -90,7 +91,7 @@ SymEncryptedAEADProtected.prototype.write = function () { * Decrypt the encrypted payload. * @param {String} sessionKeyAlgorithm The session key's cipher algorithm e.g. 'aes128' * @param {Uint8Array} key The session key used to encrypt the payload - * @returns {Promise} + * @returns {Boolean} * @async */ SymEncryptedAEADProtected.prototype.decrypt = async function (sessionKeyAlgorithm, key) { @@ -105,7 +106,6 @@ SymEncryptedAEADProtected.prototype.decrypt = async function (sessionKeyAlgorith * Encrypt the packet list payload. * @param {String} sessionKeyAlgorithm The session key's cipher algorithm e.g. 'aes128' * @param {Uint8Array} key The session key used to encrypt the payload - * @returns {Promise} * @async */ SymEncryptedAEADProtected.prototype.encrypt = async function (sessionKeyAlgorithm, key) { @@ -122,8 +122,8 @@ SymEncryptedAEADProtected.prototype.encrypt = async function (sessionKeyAlgorith * En/decrypt the payload. * @param {encrypt|decrypt} fn Whether to encrypt or decrypt * @param {Uint8Array} key The session key used to en/decrypt the payload - * @param {Uint8Array} data The data to en/decrypt - * @returns {Promise} + * @param {Uint8Array | ReadableStream} data The data to en/decrypt + * @returns {Uint8Array | ReadableStream} * @async */ SymEncryptedAEADProtected.prototype.crypt = async function (fn, key, data) { diff --git a/src/packet/sym_encrypted_integrity_protected.js b/src/packet/sym_encrypted_integrity_protected.js index d5176dda..c305a2e8 100644 --- a/src/packet/sym_encrypted_integrity_protected.js +++ b/src/packet/sym_encrypted_integrity_protected.js @@ -87,6 +87,7 @@ SymEncryptedIntegrityProtected.prototype.write = function () { * Encrypt the payload in the packet. * @param {String} sessionKeyAlgorithm The selected symmetric encryption algorithm to be used e.g. 'aes128' * @param {Uint8Array} key The key of cipher blocksize length to be used + * @param {Boolean} asStream Whether to set this.encrypted to a stream * @returns {Promise} * @async */ @@ -116,6 +117,7 @@ SymEncryptedIntegrityProtected.prototype.encrypt = async function (sessionKeyAlg * Decrypts the encrypted data contained in the packet. * @param {String} sessionKeyAlgorithm The selected symmetric encryption algorithm to be used e.g. 'aes128' * @param {Uint8Array} key The key of cipher blocksize length to be used + * @param {Boolean} asStream Whether to read this.encrypted as a stream * @returns {Promise} * @async */ diff --git a/src/signature.js b/src/signature.js index 473b3490..a533eee5 100644 --- a/src/signature.js +++ b/src/signature.js @@ -41,7 +41,7 @@ export function Signature(packetlist) { /** * Returns ASCII armored text of signature - * @returns {String} ASCII armor + * @returns {ReadableStream} ASCII armor */ Signature.prototype.armor = function() { return armor.encode(enums.armor.signature, this.packets.write()); @@ -49,8 +49,9 @@ Signature.prototype.armor = function() { /** * reads an OpenPGP armored signature and returns a signature object - * @param {String} armoredText text to be parsed + * @param {String | ReadableStream} armoredText text to be parsed * @returns {Signature} new signature object + * @async * @static */ export async function readArmored(armoredText) { @@ -60,8 +61,9 @@ export async function readArmored(armoredText) { /** * reads an OpenPGP signature as byte array and returns a signature object - * @param {Uint8Array} input binary signature + * @param {Uint8Array | ReadableStream} input binary signature * @returns {Signature} new signature object + * @async * @static */ export async function read(input) { diff --git a/src/stream.js b/src/stream.js index 719cae01..a7ab0152 100644 --- a/src/stream.js +++ b/src/stream.js @@ -1,7 +1,12 @@ import util from './util'; -const nodeStream = util.getNodeStream(); +const NodeReadableStream = util.getNodeStream(); +/** + * Convert data to Stream + * @param {ReadableStream|Uint8array|String} input data to convert + * @returns {ReadableStream} Converted data + */ function toStream(input) { if (util.isStream(input)) { return input; @@ -14,39 +19,61 @@ function toStream(input) { }); } -function concat(arrays) { - arrays = arrays.map(toStream); +/** + * Concat a list of Streams + * @param {Array} list Array of Uint8Arrays/Strings/Streams to concatenate + * @returns {ReadableStream} Concatenated array + */ +function concat(list) { + list = list.map(toStream); const transform = transformWithCancel(async function(reason) { await Promise.all(transforms.map(array => cancel(array, reason))); }); let prev = Promise.resolve(); - const transforms = arrays.map((array, i) => transformPair(array, (readable, writable) => { + const transforms = list.map((array, i) => transformPair(array, (readable, writable) => { prev = prev.then(() => pipe(readable, transform.writable, { - preventClose: i !== arrays.length - 1 + preventClose: i !== list.length - 1 })); return prev; })); return transform.readable; } +/** + * Get a Reader + * @param {ReadableStream|Uint8array|String} input + * @returns {Reader} + */ function getReader(input) { return new Reader(input); } +/** + * Get a Writer + * @param {WritableStream} input + * @returns {WritableStreamDefaultWriter} + */ function getWriter(input) { return input.getWriter(); } +/** + * Pipe a readable stream to a writable stream. Don't throw on input stream errors, but forward them to the output stream. + * @param {ReadableStream|Uint8array|String} input + * @param {WritableStream} target + * @param {Object} (optional) options + * @returns {Promise} Promise indicating when piping has finished (input stream closed or errored) + */ async function pipe(input, target, options) { if (!util.isStream(input)) { input = toStream(input); } try { - if (input.externalBuffer) { + if (input[externalBuffer]) { const writer = target.getWriter(); - for (let i = 0; i < input.externalBuffer.length; i++) { + for (let i = 0; i < input[externalBuffer].length; i++) { await writer.ready; - await writer.write(input.externalBuffer[i]); + await writer.write(input[externalBuffer][i]); } writer.releaseLock(); } @@ -56,12 +83,23 @@ async function pipe(input, target, options) { } } +/** + * Pipe a readable stream through a transform stream. + * @param {ReadableStream|Uint8array|String} input + * @param {Object} (optional) options + * @returns {ReadableStream} transformed stream + */ function transformRaw(input, options) { const transformStream = new TransformStream(options); pipe(input, transformStream.writable); return transformStream.readable; } +/** + * Create a cancelable TransformStream. + * @param {Function} cancel + * @returns {TransformStream} + */ function transformWithCancel(cancel) { let backpressureChangePromiseResolve = function() {}; let outputController; @@ -90,6 +128,13 @@ function transformWithCancel(cancel) { }; } +/** + * Transform a stream using helper functions which are called on each chunk, and on stream close, respectively. + * @param {ReadableStream|Uint8array|String} input + * @param {Function} process + * @param {Function} finish + * @returns {ReadableStream|Uint8array|String} + */ function transform(input, process = () => undefined, finish = () => undefined) { if (util.isStream(input)) { return transformRaw(input, { @@ -117,6 +162,15 @@ function transform(input, process = () => undefined, finish = () => undefined) { return result1 !== undefined ? result1 : result2; } +/** + * Transform a stream using a helper function which is passed a readable and a writable stream. + * This function also maintains the possibility to cancel the input stream, + * and does so on cancelation of the output stream, despite cancelation + * normally being impossible when the input stream is being read from. + * @param {ReadableStream|Uint8array|String} input + * @param {Function} fn + * @returns {ReadableStream} + */ function transformPair(input, fn) { let incomingTransformController; const incoming = new TransformStream({ @@ -136,6 +190,15 @@ function transformPair(input, fn) { return outgoing.readable; } +/** + * Parse a stream using a helper function which is passed a Reader. + * The reader additionally has a remainder() method which returns a + * stream pointing to the remainder of input, and is linked to input + * for cancelation. + * @param {ReadableStream|Uint8array|String} input + * @param {Function} fn + * @returns {Any} the return value of fn() + */ function parse(input, fn) { let returnValue; const transformed = transformPair(input, (readable, writable) => { @@ -150,15 +213,29 @@ function parse(input, fn) { return returnValue; } +/** + * Tee a Stream for reading it twice. The input stream can no longer be read after tee()ing. + * Reading either of the two returned streams will pull from the input stream. + * The input stream will only be canceled if both of the returned streams are canceled. + * @param {ReadableStream|Uint8array|String} input + * @returns {Array} array containing two copies of input + */ function tee(input) { if (util.isStream(input)) { const teed = input.tee(); - teed[0].externalBuffer = teed[1].externalBuffer = input.externalBuffer; + teed[0][externalBuffer] = teed[1][externalBuffer] = input[externalBuffer]; return teed; } return [slice(input), slice(input)]; } +/** + * Clone a Stream for reading it twice. The input stream can still be read after clone()ing. + * Reading from the clone will pull from the input stream. + * The input stream will only be canceled if both the clone and the input stream are canceled. + * @param {ReadableStream|Uint8array|String} input + * @returns {ReadableStream|Uint8array|String} cloned input + */ function clone(input) { if (util.isStream(input)) { const teed = tee(input); @@ -168,6 +245,14 @@ function clone(input) { return slice(input); } +/** + * Clone a Stream for reading it twice. Data will arrive at the same rate as the input stream is being read. + * Reading from the clone will NOT pull from the input stream. Data only arrives when reading the input stream. + * The input stream will NOT be canceled if the clone is canceled, only if the input stream are canceled. + * If the input stream is canceled, the clone will be errored. + * @param {ReadableStream|Uint8array|String} input + * @returns {ReadableStream|Uint8array|String} cloned input + */ function passiveClone(input) { if (util.isStream(input)) { return new ReadableStream({ @@ -199,6 +284,12 @@ function passiveClone(input) { return slice(input); } +/** + * Modify a stream object to point to a different stream object. + * This is used internally by clone() and passiveClone() to provide an abstraction over tee(). + * @param {ReadableStream} input + * @param {ReadableStream} clone + */ function overwrite(input, clone) { // Overwrite input.getReader, input.locked, etc to point to clone Object.entries(Object.getOwnPropertyDescriptors(ReadableStream.prototype)).forEach(([name, descriptor]) => { @@ -214,6 +305,11 @@ function overwrite(input, clone) { }); } +/** + * Return a stream pointing to a part of the input stream. + * @param {ReadableStream|Uint8array|String} input + * @returns {ReadableStream|Uint8array|String} clone + */ function slice(input, begin=0, end=Infinity) { if (util.isStream(input)) { if (begin >= 0 && end >= 0) { @@ -254,8 +350,8 @@ function slice(input, begin=0, end=Infinity) { util.print_debug_error(`stream.slice(input, ${begin}, ${end}) not implemented efficiently.`); return fromAsync(async () => slice(await readToEnd(input), begin, end)); } - if (input.externalBuffer) { - input = util.concat(input.externalBuffer.concat([input])); + if (input[externalBuffer]) { + input = util.concat(input[externalBuffer].concat([input])); } if (util.isUint8Array(input)) { return input.subarray(begin, end); @@ -263,19 +359,36 @@ function slice(input, begin=0, end=Infinity) { return input.slice(begin, end); } -async function readToEnd(input, join) { +/** + * Read a stream to the end and return its contents, concatenated by the concat function (defaults to util.concat). + * @param {ReadableStream|Uint8array|String} input + * @param {Function} concat + * @returns {Uint8array|String|Any} the return value of concat() + */ +async function readToEnd(input, concat) { if (util.isStream(input)) { - return getReader(input).readToEnd(join); + return getReader(input).readToEnd(concat); } return input; } +/** + * Cancel a stream. + * @param {ReadableStream|Uint8array|String} input + * @param {Any} reason + * @returns {Promise} indicates when the stream has been canceled + */ async function cancel(input, reason) { if (util.isStream(input)) { return input.cancel(reason); } } +/** + * Convert an async function to a Stream. When the function returns, its return value is enqueued to the stream. + * @param {Function} fn + * @returns {ReadableStream} + */ function fromAsync(fn) { return new ReadableStream({ pull: async controller => { @@ -298,8 +411,13 @@ function fromAsync(fn) { let nodeToWeb; let webToNode; -if (nodeStream) { +if (NodeReadableStream) { + /** + * Convert a Node Readable Stream to a Web ReadableStream + * @param {Readable} nodeStream + * @returns {ReadableStream} + */ nodeToWeb = function(nodeStream) { return new ReadableStream({ start(controller) { @@ -321,7 +439,7 @@ if (nodeStream) { }; - class NodeReadable extends nodeStream.Readable { + class NodeReadable extends NodeReadableStream { constructor(webStream, options) { super(options); this._webStream = webStream; @@ -352,6 +470,11 @@ if (nodeStream) { } } + /** + * Convert a Web ReadableStream to a Node Readable Stream + * @param {ReadableStream} webStream + * @returns {Readable} + */ webToNode = function(webStream) { return new NodeReadable(webStream); }; @@ -363,10 +486,11 @@ export default { toStream, concat, getReader, getWriter, pipe, transformRaw, tra const doneReadingSet = new WeakSet(); +const externalBuffer = Symbol('externalBuffer'); function Reader(input) { this.stream = input; - if (input.externalBuffer) { - this.externalBuffer = input.externalBuffer.slice(); + if (input[externalBuffer]) { + this[externalBuffer] = input[externalBuffer].slice(); } if (util.isStream(input)) { const reader = input.getReader(); @@ -391,21 +515,32 @@ function Reader(input) { }; } +/** + * Read a chunk of data. + * @returns {Object} Either { done: false, value: Uint8Array | String } or { done: true, value: undefined } + */ Reader.prototype.read = async function() { - if (this.externalBuffer && this.externalBuffer.length) { - const value = this.externalBuffer.shift(); + if (this[externalBuffer] && this[externalBuffer].length) { + const value = this[externalBuffer].shift(); return { done: false, value }; } return this._read(); }; +/** + * Allow others to read the stream. + */ Reader.prototype.releaseLock = function() { - if (this.externalBuffer) { - this.stream.externalBuffer = this.externalBuffer; + if (this[externalBuffer]) { + this.stream[externalBuffer] = this[externalBuffer]; } this._releaseLock(); }; +/** + * Read up to and including the first \n character. + * @returns {String|Undefined} + */ Reader.prototype.readLine = async function() { let buffer = []; let returnVal; @@ -428,6 +563,10 @@ Reader.prototype.readLine = async function() { return returnVal; }; +/** + * Read a single byte/character. + * @returns {Number|String|Undefined} + */ Reader.prototype.readByte = async function() { const { done, value } = await this.read(); if (done) return; @@ -436,6 +575,10 @@ Reader.prototype.readByte = async function() { return byte; }; +/** + * Read a specific amount of bytes/characters, unless the stream ends before that amount. + * @returns {Uint8Array|String|Undefined} + */ Reader.prototype.readBytes = async function(length) { const buffer = []; let bufferLength = 0; @@ -455,25 +598,38 @@ Reader.prototype.readBytes = async function(length) { } }; +/** + * Peek (look ahead) a specific amount of bytes/characters, unless the stream ends before that amount. + * @returns {Uint8Array|String|Undefined} + */ Reader.prototype.peekBytes = async function(length) { const bytes = await this.readBytes(length); this.unshift(bytes); return bytes; }; +/** + * Push data to the front of the stream. + * @param {Uint8Array|String|Undefined} ...values + */ Reader.prototype.unshift = function(...values) { - if (!this.externalBuffer) { - this.externalBuffer = []; + if (!this[externalBuffer]) { + this[externalBuffer] = []; } - this.externalBuffer.unshift(...values.filter(value => value && value.length)); + this[externalBuffer].unshift(...values.filter(value => value && value.length)); }; -Reader.prototype.readToEnd = async function(join=util.concat) { +/** + * Read the stream to the end and return its contents, concatenated by the concat function (defaults to util.concat). + * @param {Function} concat + * @returns {Uint8array|String|Any} the return value of concat() + */ +Reader.prototype.readToEnd = async function(concat=util.concat) { const result = []; while (true) { const { done, value } = await this.read(); if (done) break; result.push(value); } - return join(result); + return concat(result); }; diff --git a/src/util.js b/src/util.js index f4355e7c..64df40a2 100644 --- a/src/util.js +++ b/src/util.js @@ -53,7 +53,7 @@ export default { /** * Get transferable objects to pass buffers with zero copy (similar to "pass by reference" in C++) * See: https://developer.mozilla.org/en-US/docs/Web/API/Worker/postMessage - * Also, convert ReadableStreams to Uint8Arrays + * Also, convert ReadableStreams to MessagePorts * @param {Object} obj the options object to be passed to the web worker * @returns {Array} an array of binary data to be passed */ @@ -105,6 +105,11 @@ export default { } }, + /** + * Convert MessagePorts back to ReadableStreams + * @param {Object} obj + * @returns {Object} + */ restoreStreams: function(obj) { if (Object.prototype.isPrototypeOf(obj)) { Object.entries(obj).forEach(([key, value]) => { // recursively search all children @@ -490,16 +495,18 @@ export default { } }, - print_entire_stream: function (str, input, fn = result => result) { - stream.readToEnd(stream.clone(input)).then(result => { - console.log(str + ': ', fn(result)); + /** + * Read a stream to the end and print it to the console when it's closed. + * @param {String} str String of the debug message + * @param {ReadableStream|Uint8array|String} input Stream to print + * @param {Function} concat Function to concatenate chunks of the stream (defaults to util.concat). + */ + print_entire_stream: function (str, input, concat) { + stream.readToEnd(stream.clone(input), concat).then(result => { + console.log(str + ': ', result); }); }, - print_entire_stream_str: function (str, stream, fn = result => result) { - util.print_entire_stream(str, stream, result => fn(util.Uint8Array_to_str(result))); - }, - getLeftNBits: function (array, bitcount) { const rest = bitcount % 8; if (rest === 0) { @@ -622,17 +629,41 @@ export default { return typeof window === 'undefined'; }, + /** + * Get native Node.js module + * @param {String} The module to require + * @returns {Object} The required module or 'undefined' + */ + nodeRequire: function(module) { + if (!util.detectNode()) { + return; + } + + // Requiring the module dynamically allows us to access the native node module. + // otherwise, it gets replaced with the browserified version + // eslint-disable-next-line import/no-dynamic-require + return require(module); + }, + /** * Get native Node.js crypto api. The default configuration is to use * the api when available. But it can also be deactivated with config.use_native * @returns {Object} The crypto module or 'undefined' */ getNodeCrypto: function() { - if (!util.detectNode() || !config.use_native) { + if (!config.use_native) { return; } - return require('crypto'); + return util.nodeRequire('crypto'); + }, + + getNodeZlib: function() { + if (!config.use_native) { + return; + } + + return util.nodeRequire('zlib'); }, /** @@ -641,40 +672,15 @@ export default { * @returns {Function} The Buffer constructor or 'undefined' */ getNodeBuffer: function() { - if (!util.detectNode()) { - return; - } - - // This "hack" allows us to access the native node buffer module. - // otherwise, it gets replaced with the browserified version - // eslint-disable-next-line no-useless-concat, import/no-dynamic-require - return require('buffer'+'').Buffer; - }, - - getNodeZlib: function() { - if (!util.detectNode() || !config.use_native) { - return; - } - - return require('zlib'); + return (util.nodeRequire('buffer') || {}).Buffer; }, getNodeStream: function() { - if (!util.detectNode()) { - return; - } - - // eslint-disable-next-line no-useless-concat, import/no-dynamic-require - return require('stream'+''); + return (util.nodeRequire('stream') || {}).Readable; }, getNodeTextDecoder: function() { - if (!util.detectNode()) { - return; - } - - // eslint-disable-next-line no-useless-concat, import/no-dynamic-require - return require('util'+'').TextDecoder; + return (util.nodeRequire('util') || {}).TextDecoder; }, isEmailAddress: function(data) { diff --git a/test/general/streaming.js b/test/general/streaming.js index 0945d82a..6437132a 100644 --- a/test/general/streaming.js +++ b/test/general/streaming.js @@ -182,15 +182,15 @@ describe('Streaming', function() { canceled = true; } }); - const encrypted = await openpgp.sign({ + const signed = await openpgp.sign({ message: openpgp.message.fromBinary(data), privateKeys: privKey }); - const reader = openpgp.stream.getReader(encrypted.data); + const reader = openpgp.stream.getReader(signed.data); expect(await reader.readBytes(1024)).to.match(/^-----BEGIN PGP MESSAGE-----\r\n/); if (i > 10) throw new Error('Data did not arrive early.'); reader.releaseLock(); - await openpgp.stream.cancel(encrypted.data); + await openpgp.stream.cancel(signed.data); expect(canceled).to.be.true; }); @@ -476,25 +476,25 @@ describe('Streaming', function() { } } }); - const encrypted = await openpgp.sign({ + const signed = await openpgp.sign({ message: openpgp.message.fromBinary(data), privateKeys: privKey }); - const msgAsciiArmored = encrypted.data; + const msgAsciiArmored = signed.data; const message = await openpgp.message.readArmored(openpgp.stream.transform(msgAsciiArmored, value => { if (value.length > 1000) return value.slice(0, 499) + 'a' + value.slice(500); return value; })); - const decrypted = await openpgp.verify({ + const verified = await openpgp.verify({ publicKeys: pubKey, message }); - expect(util.isStream(decrypted.data)).to.be.true; - expect(await openpgp.stream.getReader(openpgp.stream.clone(decrypted.data)).readBytes(10)).not.to.deep.equal(plaintext[0]); + expect(util.isStream(verified.data)).to.be.true; + expect(await openpgp.stream.getReader(openpgp.stream.clone(verified.data)).readBytes(10)).not.to.deep.equal(plaintext[0]); if (i > 10) throw new Error('Data did not arrive early.'); - await expect(openpgp.stream.readToEnd(decrypted.data)).to.be.rejectedWith('Ascii armor integrity check on message failed'); - expect(decrypted.signatures).to.exist.and.have.length(1); + await expect(openpgp.stream.readToEnd(verified.data)).to.be.rejectedWith('Ascii armor integrity check on message failed'); + expect(verified.signatures).to.exist.and.have.length(1); } finally { openpgp.config.allow_unauthenticated_stream = allow_unauthenticated_streamValue; } @@ -702,26 +702,26 @@ describe('Streaming', function() { canceled = true; } }); - const encrypted = await openpgp.sign({ + const signed = await openpgp.sign({ message: openpgp.message.fromBinary(data), privateKeys: privKey }); - const msgAsciiArmored = encrypted.data; + const msgAsciiArmored = signed.data; const message = await openpgp.message.readArmored(msgAsciiArmored); - const decrypted = await openpgp.verify({ + const verified = await openpgp.verify({ publicKeys: pubKey, message }); - expect(util.isStream(decrypted.data)).to.be.true; - const reader = openpgp.stream.getReader(decrypted.data); + expect(util.isStream(verified.data)).to.be.true; + const reader = openpgp.stream.getReader(verified.data); expect(await reader.readBytes(1024)).to.deep.equal(plaintext[0]); if (i > 10) throw new Error('Data did not arrive early.'); reader.releaseLock(); - await openpgp.stream.cancel(decrypted.data, new Error('canceled by test')); + await openpgp.stream.cancel(verified.data, new Error('canceled by test')); expect(canceled).to.be.true; - expect(decrypted.signatures).to.exist.and.have.length(1); - expect(await decrypted.signatures[0].verified).to.be.undefined; + expect(verified.signatures).to.exist.and.have.length(1); + expect(await verified.signatures[0].verified).to.be.undefined; } finally { openpgp.config.aead_protect = aead_protectValue; openpgp.config.aead_chunk_size_byte = aead_chunk_size_byteValue; @@ -773,11 +773,11 @@ describe('Streaming', function() { await new Promise(setTimeout); } }); - const encrypted = await openpgp.sign({ + const signed = await openpgp.sign({ message: openpgp.message.fromBinary(data), privateKeys: privKey }); - const reader = openpgp.stream.getReader(encrypted.data); + const reader = openpgp.stream.getReader(signed.data); expect(await reader.readBytes(1024)).to.match(/^-----BEGIN PGP MESSAGE-----\r\n/); if (i > 10) throw new Error('Data did not arrive early.'); await new Promise(resolve => setTimeout(resolve, 3000)); @@ -851,18 +851,18 @@ describe('Streaming', function() { await new Promise(setTimeout); } }); - const encrypted = await openpgp.sign({ + const signed = await openpgp.sign({ message: openpgp.message.fromBinary(data), privateKeys: privKey }); - const msgAsciiArmored = encrypted.data; + const msgAsciiArmored = signed.data; const message = await openpgp.message.readArmored(msgAsciiArmored); - const decrypted = await openpgp.verify({ + const verified = await openpgp.verify({ publicKeys: pubKey, message }); - expect(util.isStream(decrypted.data)).to.be.true; - const reader = openpgp.stream.getReader(decrypted.data); + expect(util.isStream(verified.data)).to.be.true; + const reader = openpgp.stream.getReader(verified.data); expect(await reader.readBytes(1024)).to.deep.equal(plaintext[0]); if (i > 10) throw new Error('Data did not arrive early.'); await new Promise(resolve => setTimeout(resolve, 3000));