Support unicode surrogate code points

src/key.js

@@ -195,7 +195,7 @@ Key.prototype.getKeyIds = function() {
  */
 Key.prototype.getUserIds = function() {
   return this.users.map(user => {
-    return user.userId ? util.encode_utf8(user.userId.userid) : null;
+    return user.userId ? user.userId.userid : null;
   }).filter(userid => userid !== null);
 };
 

src/packet/literal.js

@@ -64,23 +64,7 @@ Literal.prototype.setText = function(text, format='utf8') {
  */
 Literal.prototype.getText = function(clone=false) {
   if (this.text === null || util.isStream(this.text)) { // Assume that this.text has been read
-    let lastChar = '';
-    const decoder = new TextDecoder('utf-8');
-    // eslint-disable-next-line no-inner-declarations
-    function process(value, lastChunk=false) {
-      // decode UTF8
-      const text = lastChar + decoder.decode(value, { stream: !lastChunk });
-      // normalize EOL to \n
-      const normalized = util.nativeEOL(text);
-      // if last char is \r, store it for the next chunk so we can normalize \r\n
-      if (normalized[normalized.length - 1] === '\r') {
-        lastChar = '\r';
-        return normalized.slice(0, -1);
-      }
-      lastChar = '';
-      return normalized;
-    }
-    this.text = stream.transform(this.getBytes(clone), process, () => process(new Uint8Array(), true));
+    this.text = util.nativeEOL(util.decode_utf8(this.getBytes(clone)));
   }
   return this.text;
 };
@@ -104,10 +88,8 @@ Literal.prototype.setBytes = function(bytes, format) {
  */
 Literal.prototype.getBytes = function(clone=false) {
   if (this.data === null) {
-    // normalize EOL to \r\n
-    const text = util.canonicalizeEOL(this.text);
-    // encode UTF8
-    this.data = util.str_to_Uint8Array(util.encode_utf8(text));
+    // normalize EOL to \r\n and encode UTF8
+    this.data = util.encode_utf8(util.canonicalizeEOL(this.text));
   }
   if (clone) {
     return stream.passiveClone(this.data);
@@ -146,7 +128,7 @@ Literal.prototype.read = async function(bytes) {
     const format = enums.read(enums.literal, await reader.readByte());
 
     const filename_len = await reader.readByte();
-    this.filename = util.decode_utf8(util.Uint8Array_to_str(await reader.readBytes(filename_len)));
+    this.filename = util.decode_utf8(await reader.readBytes(filename_len));
 
     this.date = util.readDate(await reader.readBytes(4));
 
@@ -162,7 +144,7 @@ Literal.prototype.read = async function(bytes) {
  * @returns {Uint8Array | ReadableStream<Uint8Array>} Uint8Array representation of the packet
  */
 Literal.prototype.write = function() {
-  const filename = util.str_to_Uint8Array(util.encode_utf8(this.filename));
+  const filename = util.encode_utf8(this.filename);
   const filename_length = new Uint8Array([filename.length]);
 
   const format = new Uint8Array([enums.write(enums.literal, this.format)]);

src/packet/signature.js

@@ -525,7 +525,7 @@ Signature.prototype.toSign = function (type, data) {
       // normalize EOL to \r\n
       text = util.canonicalizeEOL(text);
       // encode UTF8
-      return util.str_to_Uint8Array(util.encode_utf8(text));
+      return util.encode_utf8(text);
     }
     case t.standalone:
       return new Uint8Array(0);

src/packet/userid.js

@@ -52,7 +52,7 @@ function Userid() {
  * @param {Uint8Array} input payload of a tag 13 packet
  */
 Userid.prototype.read = function (bytes) {
-  this.parse(util.decode_utf8(util.Uint8Array_to_str(bytes)));
+  this.parse(util.decode_utf8(bytes));
 };
 
 /**
@@ -70,7 +70,7 @@ Userid.prototype.parse = function (userid) {
  * @returns {Uint8Array} binary representation
  */
 Userid.prototype.write = function () {
-  return util.str_to_Uint8Array(util.encode_utf8(this.userid));
+  return util.encode_utf8(this.userid);
 };
 
 /**

src/polyfills.js

@@ -47,9 +47,13 @@ if (typeof window !== 'undefined') {
 if (typeof TransformStream === 'undefined') {
   require('@mattiasbuelens/web-streams-polyfill');
 }
-if (typeof TextDecoder === 'undefined') {
-  global.TextDecoder = util.getNodeTextDecoder();
+if (typeof TextEncoder === 'undefined') {
+  const nodeUtil = util.nodeRequire('util') || {};
+  global.TextEncoder = nodeUtil.TextEncoder;
+  global.TextDecoder = nodeUtil.TextDecoder;
 }
-if (typeof TextDecoder === 'undefined') {
-  global.TextDecoder = require('text-encoding-utf-8').TextDecoder;
+if (typeof TextEncoder === 'undefined') {
+  const textEncoding = require('text-encoding-utf-8');
+  global.TextEncoder = textEncoding.TextEncoder;
+  global.TextDecoder = textEncoding.TextDecoder;
 }

src/type/s2k.js

@@ -143,7 +143,7 @@ S2K.prototype.write = function () {
  * hashAlgorithm hash length
  */
 S2K.prototype.produce_key = function (passphrase, numBytes) {
-  passphrase = util.str_to_Uint8Array(util.encode_utf8(passphrase));
+  passphrase = util.encode_utf8(passphrase);
 
   function round(prefix, s2k) {
     const algorithm = enums.write(enums.hash, s2k.algorithm);

src/util.js

@@ -309,28 +309,31 @@ export default {
   },
 
   /**
-   * Convert a native javascript string to a string of utf8 bytes
-   * @param {String} str The string to convert
-   * @returns {String} A valid squence of utf8 bytes
+   * Convert a native javascript string to a Uint8Array of utf8 bytes
+   * @param {String|ReadableStream} str The string to convert
+   * @returns {Uint8Array|ReadableStream} A valid squence of utf8 bytes
    */
   encode_utf8: function (str) {
-    return stream.transform(str, value => unescape(encodeURIComponent(value)));
+    const encoder = new TextEncoder('utf-8');
+    // eslint-disable-next-line no-inner-declarations
+    function process(value, lastChunk=false) {
+      return encoder.encode(value, { stream: !lastChunk });
+    }
+    return stream.transform(str, process, () => process('', true));
   },
 
   /**
-   * Convert a string of utf8 bytes to a native javascript string
-   * @param {String} utf8 A valid squence of utf8 bytes
-   * @returns {String} A native javascript string
+   * Convert a Uint8Array of utf8 bytes to a native javascript string
+   * @param {Uint8Array|ReadableStream} utf8 A valid squence of utf8 bytes
+   * @returns {String|ReadableStream} A native javascript string
    */
   decode_utf8: function (utf8) {
-    if (typeof utf8 !== 'string') {
-      throw new Error('Parameter "utf8" is not of type string');
-    }
-    try {
-      return decodeURIComponent(escape(utf8));
-    } catch (e) {
-      return utf8;
+    const decoder = new TextDecoder('utf-8');
+    // eslint-disable-next-line no-inner-declarations
+    function process(value, lastChunk=false) {
+      return decoder.decode(value, { stream: !lastChunk });
     }
+    return stream.transform(utf8, process, () => process(new Uint8Array(), true));
   },
 
   /**
@@ -627,10 +630,6 @@ export default {
     return (util.nodeRequire('stream') || {}).Readable;
   },
 
-  getNodeTextDecoder: function() {
-    return (util.nodeRequire('util') || {}).TextDecoder;
-  },
-
   getHardwareConcurrency: function() {
     if (util.detectNode()) {
       const os = util.nodeRequire('os');
@@ -678,14 +677,24 @@ export default {
    * Normalize line endings to \r\n
    */
   canonicalizeEOL: function(text) {
-    return stream.transform(text, value => value.replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(/\n/g, "\r\n"));
+    return stream.transform(util.nativeEOL(text), value => value.replace(/\r/g, "\n").replace(/\n/g, "\r\n"));
   },
 
   /**
    * Convert line endings from canonicalized \r\n to native \n
    */
   nativeEOL: function(text) {
-    return text.replace(/\r\n/g, "\n");
+    let lastChar = '';
+    return stream.transform(text, value => {
+      value = lastChar + value;
+      if (value[value.length - 1] === '\r') {
+        lastChar = '\r';
+        value = value.slice(0, -1);
+      } else {
+        lastChar = '';
+      }
+      return value.replace(/\r\n/g, '\n');
+    }, () => lastChar);
   },
 
   /**

test/general/signature.js

@@ -707,7 +707,7 @@ yYDnCgA=
     await privKey.decrypt('hello world');
     return openpgp.sign({ privateKeys:[privKey], message: openpgp.message.fromText(plaintext), detached: true}).then(async function(signed) {
       const signature = await openpgp.signature.readArmored(signed.signature);
-      return openpgp.verify({ publicKeys:[pubKey], message: openpgp.message.fromBinary(openpgp.util.str_to_Uint8Array(openpgp.util.encode_utf8(plaintext))), signature: signature });
+      return openpgp.verify({ publicKeys:[pubKey], message: openpgp.message.fromBinary(openpgp.util.encode_utf8(plaintext)), signature: signature });
     }).then(function(cleartextSig) {
       expect(cleartextSig).to.exist;
       expect(cleartextSig.signatures).to.have.length(1);
@@ -740,7 +740,7 @@ yYDnCgA=
     await Promise.all([privKey.primaryKey.decrypt('hello world'), privKey.subKeys[0].keyPacket.decrypt('hello world')]);
     return openpgp.sign({ privateKeys:[privKey], message: openpgp.message.fromText(plaintext), detached: true}).then(async function(signed) {
       const signature = await openpgp.signature.readArmored(signed.signature);
-      return openpgp.encrypt({ message: openpgp.message.fromBinary(openpgp.util.str_to_Uint8Array(openpgp.util.encode_utf8(plaintext))), publicKeys: [pubKey], signature })
+      return openpgp.encrypt({ message: openpgp.message.fromBinary(openpgp.util.encode_utf8(plaintext)), publicKeys: [pubKey], signature })
     }).then(async ({ data }) => {
       const csMsg = await openpgp.message.readArmored(data);
       return openpgp.decrypt({ message: csMsg, privateKeys: [ privKey ], publicKeys: [ pubKey ] });

test/general/streaming.js

@@ -431,7 +431,7 @@ function tests() {
       });
       expect(util.isStream(decrypted.data)).to.equal(expectedType);
       const reader = openpgp.stream.getReader(decrypted.data);
-      expect((await reader.peekBytes(256)).toString('utf8').substr(0, 64)).to.equal(plaintext[0]);
+      expect((await reader.peekBytes(plaintext[0].length * 4)).toString('utf8').substr(0, plaintext[0].length)).to.equal(plaintext[0]);
       if (i > 10) throw new Error('Data did not arrive early.');
       expect((await reader.readToEnd()).toString('utf8')).to.equal(util.concat(plaintext));
     } finally {

test/general/testInputs.js

@@ -3,15 +3,16 @@
  * Generates a 64 character long javascript string out of the whole utf-8 range.
  */
 function createSomeMessage(){
-    const length = 50;
     let arr = [];
-    for (let i= 0; i < length; i++){
-        arr.push(String.fromCharCode(
-            Math.floor(Math.random() * 10174) + 1));
+    for (let i = 0; i < 30; i++) {
+        arr.push(Math.floor(Math.random() * 10174) + 1);
     }
-    return '  \t' + arr.join('').replace(/\r/g, '\n') + '  \t\n한국어/조선말';
+    for (let i = 0; i < 10; i++) {
+        arr.push(0x1F600 + Math.floor(Math.random() * (0x1F64F - 0x1F600)) + 1);
+    }
+    return '  \t' + String.fromCodePoint(...arr).replace(/\r/g, '\n') + '  \t\n한국어/조선말';
 }
 
- module.exports = {
-     createSomeMessage: createSomeMessage
- };
+module.exports = {
+    createSomeMessage: createSomeMessage
+};

test/general/util.js

@@ -153,10 +153,6 @@ describe('Util unit tests', function() {
   });
 
   describe("Misc.", function() {
-    it('util.decode_utf8 throws error if invalid parameter type', function () {
-      const test = openpgp.util.decode_utf8.bind(null, {chameleon: true});
-      expect(test).to.throw(Error, /Parameter "utf8" is not of type string/);
-    });
     it('util.readNumber should not overflow until full range of uint32', function () {
       const ints = [Math.pow(2, 20), Math.pow(2, 25), Math.pow(2, 30), Math.pow(2, 32) - 1];
       for(let i = 0; i < ints.length; i++) {