Gzip-compress API uploads larger than 1000 characters

chrome/content/zotero/xpcom/http.js

@@ -181,10 +181,26 @@ Zotero.HTTP = new function() {
 		}
 		
 		// Send headers
-		var headers = (options && options.headers) || {};
+		var headers = {};
-		if (options.body && !headers["Content-Type"]) {
+		if (options && options.headers) {
+			Object.assign(headers, options.headers);
+		}
+		var compressedBody = false;
+		if (options.body) {
+			if (!headers["Content-Type"]) {
 				headers["Content-Type"] = "application/x-www-form-urlencoded";
 			}
+			
+			if (options.compressBody && this.isWriteMethod(method)) {
+				headers['Content-Encoding'] = 'gzip';
+				compressedBody = yield Zotero.Utilities.Internal.gzip(options.body);
+				
+				let oldLen = options.body.length;
+				let newLen = compressedBody.length;
+				Zotero.debug(`${method} body gzipped from ${oldLen} to ${newLen}; `
+					+ Math.round(((oldLen - newLen) / oldLen) * 100) + "% savings");
+			}
+		}
 		if (options.debug) {
 			if (headers["Zotero-API-Key"]) {
 				let dispHeaders = {};
@@ -248,7 +264,19 @@ Zotero.HTTP = new function() {
 			options.cookieSandbox.attachToInterfaceRequestor(xmlhttp);
 		}
 		
+		// Send binary data
+		if (compressedBody) {
+			let numBytes = compressedBody.length;
+			let ui8Data = new Uint8Array(numBytes);
+			for (let i = 0; i < numBytes; i++) {
+				ui8Data[i] = compressedBody.charCodeAt(i) & 0xff;
+			}
+			xmlhttp.send(ui8Data);
+		}
+		// Send regular request
+		else {
 			xmlhttp.send(options.body || null);
+		}
 		
 		return deferred.promise;
 	});
@@ -705,6 +733,11 @@ Zotero.HTTP = new function() {
 	}
 	
 	
+	this.isWriteMethod = function (method) {
+		return method == 'POST' || method == 'PUT' || method == 'PATCH';
+	};
+	
+	
 	this.getDisplayURI = function (uri) {
 		var disp = uri.clone();
 		if (disp.password) {

chrome/content/zotero/xpcom/sync/syncAPIClient.js

@@ -43,6 +43,7 @@ Zotero.Sync.APIClient = function (options) {
 
 Zotero.Sync.APIClient.prototype = {
 	MAX_OBJECTS_PER_REQUEST: 100,
+	MIN_GZIP_SIZE: 1000,
 	
 	
 	getKeyInfo: Zotero.Promise.coroutine(function* (options={}) {
@@ -571,6 +572,10 @@ Zotero.Sync.APIClient.prototype = {
 		opts.dontCache = true;
 		opts.foreground = !options.background;
 		opts.responseType = options.responseType || 'text';
+		if (options.body && options.body.length >= this.MIN_GZIP_SIZE) {
+			opts.compressBody = true;
+		}
+		
 		var tries = 0;
 		var failureDelayGenerator = null;
 		while (true) {

chrome/content/zotero/xpcom/utilities_internal.js

@@ -225,6 +225,125 @@ Zotero.Utilities.Internal = {
 	},
 	
 	
+	gzip: Zotero.Promise.coroutine(function* (data) {
+		var deferred = Zotero.Promise.defer();
+		
+		// Get input stream from POST data
+		var unicodeConverter = Components.classes["@mozilla.org/intl/scriptableunicodeconverter"]
+			.createInstance(Components.interfaces.nsIScriptableUnicodeConverter);
+		unicodeConverter.charset = "UTF-8";
+		var is = unicodeConverter.convertToInputStream(data);
+		
+		// Initialize stream converter
+		var converter = Components.classes["@mozilla.org/streamconv;1?from=uncompressed&to=gzip"]
+			.createInstance(Components.interfaces.nsIStreamConverter);
+		converter.asyncConvertData(
+			"uncompressed",
+			"gzip",
+			{
+				binaryInputStream: null,
+				size: 0,
+				data: '',
+				
+				onStartRequest: function (request, context) {},
+				
+				onStopRequest: function (request, context, status) {
+					this.binaryInputStream.close();
+					delete this.binaryInputStream;
+					
+					deferred.resolve(this.data);
+				},
+				
+				onDataAvailable: function (request, context, inputStream, offset, count) {
+					this.size += count;
+					
+					this.binaryInputStream = Components.classes["@mozilla.org/binaryinputstream;1"]
+						.createInstance(Components.interfaces.nsIBinaryInputStream)
+					this.binaryInputStream.setInputStream(inputStream);
+					this.data += this.binaryInputStream.readBytes(this.binaryInputStream.available());
+				},
+				
+				QueryInterface: function (iid) {
+					if (iid.equals(Components.interfaces.nsISupports)
+						   || iid.equals(Components.interfaces.nsIStreamListener)) {
+						return this;
+					}
+					throw Components.results.NS_ERROR_NO_INTERFACE;
+				}
+			},
+			null
+		);
+		
+		// Send input stream to stream converter
+		var pump = Components.classes["@mozilla.org/network/input-stream-pump;1"]
+			.createInstance(Components.interfaces.nsIInputStreamPump);
+		pump.init(is, -1, -1, 0, 0, true);
+		pump.asyncRead(converter, null);
+		
+		return deferred.promise;
+	}),
+	
+	
+	gunzip: Zotero.Promise.coroutine(function* (data) {
+		var deferred = Zotero.Promise.defer();
+		
+		Components.utils.import("resource://gre/modules/NetUtil.jsm");
+		
+		var is = Components.classes["@mozilla.org/io/string-input-stream;1"]
+			.createInstance(Ci.nsIStringInputStream);
+		is.setData(data, data.length);
+		
+		var bis = Components.classes["@mozilla.org/binaryinputstream;1"]
+			.createInstance(Components.interfaces.nsIBinaryInputStream);
+		bis.setInputStream(is);
+		
+		// Initialize stream converter
+		var converter = Components.classes["@mozilla.org/streamconv;1?from=gzip&to=uncompressed"]
+			.createInstance(Components.interfaces.nsIStreamConverter);
+		converter.asyncConvertData(
+			"gzip",
+			"uncompressed",
+			{
+				data: '',
+				
+				onStartRequest: function (request, context) {},
+				
+				onStopRequest: function (request, context, status) {
+					deferred.resolve(this.data);
+				},
+				
+				onDataAvailable: function (request, context, inputStream, offset, count) {
+					this.data += NetUtil.readInputStreamToString(
+						inputStream,
+						inputStream.available(),
+						{
+							charset: 'UTF-8',
+							replacement: 65533
+						}
+					)
+				},
+				
+				QueryInterface: function (iid) {
+					if (iid.equals(Components.interfaces.nsISupports)
+						   || iid.equals(Components.interfaces.nsIStreamListener)) {
+						return this;
+					}
+					throw Components.results.NS_ERROR_NO_INTERFACE;
+				}
+			},
+			null
+		);
+		
+		// Send input stream to stream converter
+		var pump = Components.classes["@mozilla.org/network/input-stream-pump;1"]
+			.createInstance(Components.interfaces.nsIInputStreamPump);
+		pump.init(bis, -1, -1, 0, 0, true);
+		pump.asyncRead(converter, null);
+		
+		return deferred.promise;
+	}),
+	
+	
 	/**
 	 * Unicode normalization
 	 */

test/content/runtests.html

@@ -11,6 +11,7 @@
 	<script src="resource://zotero-unit/mocha/mocha.js"></script>
 	<script src="resource://zotero-unit/sinon.js"></script>
 	<script src="resource://zotero-unit/sinon-as-promised.js"></script>
+	<script src="resource://zotero-unit/pako_inflate.js"></script>
 	<script src="support.js" type="application/javascript;version=1.8"></script>
 	<script src="runtests.js" type="application/javascript;version=1.8"></script>
 </body>

test/content/support.js

@@ -281,6 +281,13 @@ function clickOnItemsRow(itemsView, row, button = 0) {
 }
 
 
+/**
+ * Synchronous inflate
+ */
+function gunzip(gzdata) {
+	return pako.inflate(gzdata, { to: 'string' });
+}
+
 
 /**
  * Get a default group used by all tests that want one, creating one if necessary

test/resource/sinon.js

@@ -5278,6 +5278,11 @@ if (typeof sinon === "undefined") {
                         this.requestHeaders["Content-Type"] = "text/plain;charset=utf-8";
                     }
 
+                    // Added by Zotero
+                    if (this.requestHeaders['Content-Encoding'] == 'gzip') {
+                        data = gunzip(data, true);
+                    }
+
                     this.requestBody = data;
                 }
 

test/tests/utilities_internalTest.js

@@ -12,6 +12,18 @@ describe("Zotero.Utilities.Internal", function () {
 	})
 	
 	
+	describe("#gzip()/gunzip()", function () {
+		it("should compress and decompress a Unicode text string", function* () {
+			var text = "Voilà! \u1F429";
+			var compstr = yield Zotero.Utilities.Internal.gzip(text);
+			assert.isAbove(compstr.length, 0);
+			assert.notEqual(compstr.length, text.length);
+			var str = yield Zotero.Utilities.Internal.gunzip(compstr);
+			assert.equal(str, text);
+		});
+	});
+	
+	
 	describe("#delayGenerator", function () {
 		var spy;