From 6d289797bf1c941c57b72adeca01f0a4fd364bc9 Mon Sep 17 00:00:00 2001
From: Dan Stillman <dstillman@zotero.org>
Date: Thu, 12 Nov 2015 02:40:01 -0500
Subject: [PATCH] Retry API requests automatically after 5xx errors

---
 chrome/content/zotero/xpcom/http.js           |  3 +
 .../zotero/xpcom/sync/syncAPIClient.js        | 55 ++++++++++++++-----
 2 files changed, 44 insertions(+), 14 deletions(-)

diff --git a/chrome/content/zotero/xpcom/http.js b/chrome/content/zotero/xpcom/http.js
index dbb8e2782..825450554 100644
--- a/chrome/content/zotero/xpcom/http.js
+++ b/chrome/content/zotero/xpcom/http.js
@@ -36,6 +36,9 @@ Zotero.HTTP = new function() {
 	this.UnexpectedStatusException.prototype.is4xx = function () {
 		return this.status >= 400 && this.status < 500;
 	}
+	this.UnexpectedStatusException.prototype.is5xx = function () {
+		return this.status >= 500 && this.status < 600;
+	}
 	this.UnexpectedStatusException.prototype.toString = function() {
 		return this.message;
 	};
diff --git a/chrome/content/zotero/xpcom/sync/syncAPIClient.js b/chrome/content/zotero/xpcom/sync/syncAPIClient.js
index 3f8a718a0..a38228e5c 100644
--- a/chrome/content/zotero/xpcom/sync/syncAPIClient.js
+++ b/chrome/content/zotero/xpcom/sync/syncAPIClient.js
@@ -37,6 +37,8 @@ Zotero.Sync.APIClient = function (options) {
 	this.apiVersion = options.apiVersion;
 	this.apiKey = options.apiKey;
 	this.caller = options.caller;
+	
+	this.failureDelayIntervals = [2500, 5000, 10000, 20000, 40000, 60000, 120000, 240000, 300000];
 }
 
 Zotero.Sync.APIClient.prototype = {
@@ -437,25 +439,50 @@ Zotero.Sync.APIClient.prototype = {
 	},
 	
 	
-	makeRequest: function (method, uri, options = {}) {
+	makeRequest: Zotero.Promise.coroutine(function* (method, uri, options = {}) {
 		options.headers = this.getHeaders(options.headers);
 		options.dontCache = true;
 		options.foreground = !options.background;
 		options.responseType = options.responseType || 'text';
-		return this.caller.start(Zotero.Promise.coroutine(function* () {
-			try {
-				var xmlhttp = yield Zotero.HTTP.request(method, uri, options);
-				this._checkBackoff(xmlhttp);
-				return xmlhttp;
+		var tries = 0;
+		var failureDelayGenerator = null;
+		while (true) {
+			var result = yield this.caller.start(Zotero.Promise.coroutine(function* () {
+				try {
+					var xmlhttp = yield Zotero.HTTP.request(method, uri, options);
+					this._checkBackoff(xmlhttp);
+					return xmlhttp;
+				}
+				catch (e) {
+					tries++;
+					if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
+						//this._checkRetry(e.xmlhttp);
+						
+						if (e.is5xx()) {
+							Zotero.logError(e);
+							if (!failureDelayGenerator) {
+								// Keep trying for up to an hour
+								failureDelayGenerator = Zotero.Utilities.Internal.delayGenerator(
+									this.failureDelayIntervals, 60 * 60 * 1000
+								);
+							}
+							let keepGoing = yield failureDelayGenerator.next();
+							if (!keepGoing) {
+								Zotero.logError("Failed too many times");
+								throw lastError;
+							}
+							return false;
+						}
+					}
+					throw e;
+				}
+			}.bind(this)));
+			
+			if (result) {
+				return result;
 			}
-			catch (e) {
-				/*if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
-					this._checkRetry(e.xmlhttp);
-				}*/
-				throw e;
-			}
-		}.bind(this)));
-	},
+		}
+	}),
 	
 	
 	_parseJSON: function (json) {