From 984789d304179a75edbe5d0e185d0fd88da0cbe7 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Mon, 20 Jul 2015 17:27:55 -0400 Subject: [PATCH] API syncing megacommit There's a lot more to do, and this isn't ready for actual usage, but the basic functionality is mostly in place and has decent test coverage. It can successfully upgrade a library last used with classic syncing and pull down changes via the API. Uploading mostly works but is currently disabled for safety until it has better test coverage. Downloaded JSON is first saved to a cache table, which is then used to populate other tables and later for generating PATCH requests and automatically resolving conflicts (since it shows what was changed locally and what was changed remotely). Objects with unmet dependencies or unknown fields are skipped for now but don't block the rest of the sync. Some of the bigger remaining to-dos: - Tests for uploading - Re-do the preferences to get an API key - File sync integration - Full-text syncing integration - Manual conflict resolution (though this already includes much smarter conflict handling that automatically resolves many conflicts) --- .../zotero/preferences/preferences_sync.js | 2 +- .../zotero/xpcom/data/dataObjectUtilities.js | 20 +- .../content/zotero/xpcom/data/dataObjects.js | 30 - chrome/content/zotero/xpcom/data/group.js | 10 +- chrome/content/zotero/xpcom/data/libraries.js | 12 +- chrome/content/zotero/xpcom/error.js | 9 +- chrome/content/zotero/xpcom/fulltext.js | 2 + chrome/content/zotero/xpcom/http.js | 3 + chrome/content/zotero/xpcom/schema.js | 3 + chrome/content/zotero/xpcom/storage/zfs.js | 2 +- chrome/content/zotero/xpcom/sync.js | 1082 +---------------- .../zotero/xpcom/sync/syncAPIClient.js | 445 +++++++ .../content/zotero/xpcom/sync/syncEngine.js | 1075 ++++++++++++++++ .../zotero/xpcom/sync/syncEventListeners.js | 216 ++++ chrome/content/zotero/xpcom/sync/syncLocal.js | 706 +++++++++++ .../content/zotero/xpcom/sync/syncRunner.js | 1056 ++++++++++++++++ .../zotero/xpcom/sync/syncUtilities.js | 49 + chrome/content/zotero/xpcom/syncedSettings.js | 32 +- chrome/content/zotero/xpcom/zotero.js | 5 +- chrome/content/zotero/zoteroPane.js | 10 +- chrome/locale/en-US/zotero/zotero.properties | 2 +- components/zotero-service.js | 6 + resource/config.js | 2 +- test/content/support.js | 37 + test/tests/syncEngineTest.js | 754 ++++++++++++ test/tests/syncLocalTest.js | 984 +++++++++++++++ test/tests/syncRunnerTest.js | 653 ++++++++++ 27 files changed, 6058 insertions(+), 1149 deletions(-) create mode 100644 chrome/content/zotero/xpcom/sync/syncAPIClient.js create mode 100644 chrome/content/zotero/xpcom/sync/syncEngine.js create mode 100644 chrome/content/zotero/xpcom/sync/syncEventListeners.js create mode 100644 chrome/content/zotero/xpcom/sync/syncLocal.js create mode 100644 chrome/content/zotero/xpcom/sync/syncRunner.js create mode 100644 chrome/content/zotero/xpcom/sync/syncUtilities.js create mode 100644 test/tests/syncEngineTest.js create mode 100644 test/tests/syncLocalTest.js create mode 100644 test/tests/syncRunnerTest.js diff --git a/chrome/content/zotero/preferences/preferences_sync.js b/chrome/content/zotero/preferences/preferences_sync.js index 4b454697e..1b5f5fe04 100644 --- a/chrome/content/zotero/preferences/preferences_sync.js +++ b/chrome/content/zotero/preferences/preferences_sync.js @@ -373,7 +373,7 @@ Zotero_Preferences.Sync = { available to the custom callbacks onSuccess: function () { - Zotero.Sync.Runner.setSyncIcon(); + Zotero.Sync.Runner.updateIcons(); ps.alert( null, "Restore Completed", diff --git a/chrome/content/zotero/xpcom/data/dataObjectUtilities.js b/chrome/content/zotero/xpcom/data/dataObjectUtilities.js index 43598b857..15af32b1d 100644 --- a/chrome/content/zotero/xpcom/data/dataObjectUtilities.js +++ b/chrome/content/zotero/xpcom/data/dataObjectUtilities.js @@ -26,14 +26,30 @@ Zotero.DataObjectUtilities = { /** - * Get an array of all DataObject types + * Get all DataObject types * - * @return {String[]} + * @return {String[]} - An array of DataObject types */ getTypes: function () { return ['collection', 'search', 'item']; }, + /** + * Get DataObject types that are valid for a given library + * + * @param {Integer} libraryID + * @return {String[]} - An array of DataObject types + */ + getTypesForLibrary: function (libraryID) { + switch (Zotero.Libraries.getType(libraryID)) { + case 'publications': + return ['item']; + + default: + return this.getTypes(); + } + }, + "checkLibraryID": function (libraryID) { if (!libraryID) { throw new Error("libraryID not provided"); diff --git a/chrome/content/zotero/xpcom/data/dataObjects.js b/chrome/content/zotero/xpcom/data/dataObjects.js index 38f3df535..952f7c456 100644 --- a/chrome/content/zotero/xpcom/data/dataObjects.js +++ b/chrome/content/zotero/xpcom/data/dataObjects.js @@ -327,36 +327,6 @@ Zotero.DataObjects.prototype.getNewer = Zotero.Promise.method(function (libraryI }); -/** - * @param {Integer} libraryID - * @return {Promise} A promise for an array of object ids - */ -Zotero.DataObjects.prototype.getUnsynced = function (libraryID) { - var sql = "SELECT " + this._ZDO_id + " FROM " + this._ZDO_table - + " WHERE libraryID=? AND synced=0"; - return Zotero.DB.columnQueryAsync(sql, [libraryID]); -} - - -/** - * Get JSON from the sync cache that hasn't yet been written to the - * main object tables - * - * @param {Integer} libraryID - * @return {Promise} A promise for an array of JSON objects - */ -Zotero.DataObjects.prototype.getUnwrittenData = function (libraryID) { - var sql = "SELECT data FROM syncCache SC " - + "LEFT JOIN " + this._ZDO_table + " " - + "USING (libraryID) " - + "WHERE SC.libraryID=? AND " - + "syncObjectTypeID IN (SELECT syncObjectTypeID FROM " - + "syncObjectTypes WHERE name='" + this._ZDO_object + "') " - + "AND IFNULL(O.version, 0) < SC.version"; - return Zotero.DB.columnQueryAsync(sql, [libraryID]); -} - - /** * Reload loaded data of loaded objects * diff --git a/chrome/content/zotero/xpcom/data/group.js b/chrome/content/zotero/xpcom/data/group.js index ae24b056b..22615d1ee 100644 --- a/chrome/content/zotero/xpcom/data/group.js +++ b/chrome/content/zotero/xpcom/data/group.js @@ -188,13 +188,9 @@ Zotero.Group.prototype.hasItem = function (item) { Zotero.Group.prototype.save = Zotero.Promise.coroutine(function* () { - if (!this.id) { - throw new Error("Group id not set"); - } - - if (!this.name) { - throw new Error("Group name not set"); - } + if (!this.id) throw new Error("Group id not set"); + if (!this.name) throw new Error("Group name not set"); + if (!this.version) throw new Error("Group version not set"); if (!this._changed) { Zotero.debug("Group " + this.id + " has not changed"); diff --git a/chrome/content/zotero/xpcom/data/libraries.js b/chrome/content/zotero/xpcom/data/libraries.js index 159d363e8..933291440 100644 --- a/chrome/content/zotero/xpcom/data/libraries.js +++ b/chrome/content/zotero/xpcom/data/libraries.js @@ -155,7 +155,7 @@ Zotero.Libraries = new function () { /** * @param {Integer} libraryID - * @param {Integer} version + * @param {Integer} version - Library version, or -1 to indicate that a full sync is required * @return {Promise} */ this.setVersion = Zotero.Promise.coroutine(function* (libraryID, version) { @@ -173,14 +173,14 @@ Zotero.Libraries = new function () { /** * @param {Integer} libraryID - * @param {Date} lastSyncTime * @return {Promise} */ - this.setLastSyncTime = function (libraryID, lastSyncTime) { - var lastSyncTime = Math.round(lastSyncTime.getTime() / 1000); - _libraryData[libraryID].lastSyncTime = lastSyncTime; + this.updateLastSyncTime = function (libraryID) { + var d = new Date(); + _libraryData[libraryID].lastSyncTime = d; return Zotero.DB.queryAsync( - "UPDATE libraries SET lastsync=? WHERE libraryID=?", [lastSyncTime, libraryID] + "UPDATE libraries SET lastsync=? WHERE libraryID=?", + [Math.round(d.getTime() / 1000), libraryID] ); }; diff --git a/chrome/content/zotero/xpcom/error.js b/chrome/content/zotero/xpcom/error.js index 69090c732..7b8bdd056 100644 --- a/chrome/content/zotero/xpcom/error.js +++ b/chrome/content/zotero/xpcom/error.js @@ -25,7 +25,6 @@ Zotero.Error = function (message, error, data) { - this.name = "Zotero Error"; this.message = message; this.data = data; if (parseInt(error) == error) { @@ -35,8 +34,8 @@ Zotero.Error = function (message, error, data) { this.error = Zotero.Error["ERROR_" + error] ? Zotero.Error["ERROR_" + error] : 0; } } -Zotero.Error.prototype = new Error; - +Zotero.Error.prototype = Object.create(Error.prototype); +Zotero.Error.prototype.name = "Zotero Error"; Zotero.Error.ERROR_UNKNOWN = 0; Zotero.Error.ERROR_MISSING_OBJECT = 1; @@ -51,10 +50,6 @@ Zotero.Error.ERROR_USER_NOT_AVAILABLE = 9; //Zotero.Error.ERROR_SYNC_EMPTY_RESPONSE_FROM_SERVER = 6; //Zotero.Error.ERROR_SYNC_INVALID_RESPONSE_FROM_SERVER = 7; -Zotero.Error.prototype.toString = function () { - return this.message; -} - /** * Namespace for runtime exceptions * @namespace diff --git a/chrome/content/zotero/xpcom/fulltext.js b/chrome/content/zotero/xpcom/fulltext.js index 31441be1f..d35b24d94 100644 --- a/chrome/content/zotero/xpcom/fulltext.js +++ b/chrome/content/zotero/xpcom/fulltext.js @@ -863,6 +863,8 @@ Zotero.Fulltext = new function(){ * @return {String} PHP-formatted POST data for items not yet downloaded */ this.getUndownloadedPostData = Zotero.Promise.coroutine(function* () { + // TODO: Redo for API syncing + // On upgrade, get all content var sql = "SELECT value FROM settings WHERE setting='fulltext' AND key='downloadAll'"; if (yield Zotero.DB.valueQueryAsync(sql)) { diff --git a/chrome/content/zotero/xpcom/http.js b/chrome/content/zotero/xpcom/http.js index 76babfefe..2c0749082 100644 --- a/chrome/content/zotero/xpcom/http.js +++ b/chrome/content/zotero/xpcom/http.js @@ -33,6 +33,9 @@ Zotero.HTTP = new function() { } }; this.UnexpectedStatusException.prototype = Object.create(Error.prototype); + this.UnexpectedStatusException.prototype.is4xx = function () { + return this.status >= 400 && this.status < 500; + } this.UnexpectedStatusException.prototype.toString = function() { return this.message; }; diff --git a/chrome/content/zotero/xpcom/schema.js b/chrome/content/zotero/xpcom/schema.js index 561d402a4..d11446dbe 100644 --- a/chrome/content/zotero/xpcom/schema.js +++ b/chrome/content/zotero/xpcom/schema.js @@ -2213,6 +2213,9 @@ Zotero.Schema = new function(){ yield Zotero.DB.queryAsync("INSERT OR IGNORE INTO syncDeleteLog SELECT * FROM syncDeleteLogOld"); yield Zotero.DB.queryAsync("DROP INDEX IF EXISTS syncDeleteLog_timestamp"); yield Zotero.DB.queryAsync("CREATE INDEX syncDeleteLog_synced ON syncDeleteLog(synced)"); + // TODO: Something special for tag deletions? + //yield Zotero.DB.queryAsync("DELETE FROM syncDeleteLog WHERE syncObjectTypeID IN (2, 5, 6)"); + //yield Zotero.DB.queryAsync("DELETE FROM syncObjectTypes WHERE syncObjectTypeID IN (2, 5, 6)"); yield Zotero.DB.queryAsync("ALTER TABLE storageDeleteLog RENAME TO storageDeleteLogOld"); yield Zotero.DB.queryAsync("CREATE TABLE storageDeleteLog (\n libraryID INT NOT NULL,\n key TEXT NOT NULL,\n synced INT NOT NULL DEFAULT 0,\n PRIMARY KEY (libraryID, key),\n FOREIGN KEY (libraryID) REFERENCES libraries(libraryID) ON DELETE CASCADE\n)"); diff --git a/chrome/content/zotero/xpcom/storage/zfs.js b/chrome/content/zotero/xpcom/storage/zfs.js index 6cb705366..df52f547a 100644 --- a/chrome/content/zotero/xpcom/storage/zfs.js +++ b/chrome/content/zotero/xpcom/storage/zfs.js @@ -325,7 +325,7 @@ Zotero.Sync.Storage.ZFS = (function () { dialogButtonCallback: buttonCallback } ); - e.errorMode = 'warning'; + e.errorType = 'warning'; Zotero.debug(e, 2); Components.utils.reportError(e); throw e; diff --git a/chrome/content/zotero/xpcom/sync.js b/chrome/content/zotero/xpcom/sync.js index dce2879f1..8301743e8 100644 --- a/chrome/content/zotero/xpcom/sync.js +++ b/chrome/content/zotero/xpcom/sync.js @@ -91,28 +91,6 @@ Zotero.Sync = new function() { } Zotero.DB.commitTransaction(); - - this.EventListener.init(); - } - - - this.getObjectTypeID = function (type) { - if (!_typesLoaded) { - _loadObjectTypes(); - } - - var id = _objectTypeIDs[type]; - return id ? id : false; - } - - - this.getObjectTypeName = function (typeID, plural) { - if (!_typesLoaded) { - _loadObjectTypes(); - } - - var name = _objectTypeNames[typeID]; - return name ? name : false; } @@ -379,828 +357,6 @@ Zotero.Sync.ObjectKeySet.prototype.removeLibraryKeyPairs = function (type, keyPa } - -/** - * Notifier observer to add deleted objects to syncDeleteLog/storageDeleteLog - * plus related methods - */ -Zotero.Sync.EventListener = new function () { - this.init = init; - this.ignoreDeletions = ignoreDeletions; - this.notify = notify; - - var _deleteBlacklist = {}; - - - function init() { - // Initialize delete log listener - Zotero.Notifier.registerObserver(this); - } - - - /** - * Blacklist objects from going into the sync delete log - */ - function ignoreDeletions(type, ids) { - if (!Zotero.Sync.syncObjects[type]) { - throw ("Invalid type '" + type + - "' in Zotero.Sync.EventListener.ignoreDeletions()"); - } - - if (!_deleteBlacklist[type]) { - _deleteBlacklist[type] = {}; - } - - ids = Zotero.flattenArguments(ids); - for each(var id in ids) { - _deleteBlacklist[type][id] = true; - } - } - - this.resetIgnored = function () { - _deleteBlacklist = {}; - } - - - function notify(event, type, ids, extraData) { - var objectTypeID = Zotero.Sync.getObjectTypeID(type); - if (!objectTypeID) { - return; - } - - var isItem = Zotero.Sync.getObjectTypeName(objectTypeID) == 'item'; - - Zotero.DB.beginTransaction(); - - if (event == 'delete') { - var sql = "REPLACE INTO syncDeleteLog VALUES (?, ?, ?, ?)"; - var syncStatement = Zotero.DB.getStatement(sql); - - if (isItem && Zotero.Sync.Storage.WebDAV.includeUserFiles) { - var storageEnabled = true; - var sql = "INSERT INTO storageDeleteLog VALUES (?, ?, ?)"; - var storageStatement = Zotero.DB.getStatement(sql); - } - var storageBound = false; - - var ts = Zotero.Date.getUnixTimestamp(); - - for (var i=0, len=ids.length; i errorModes[primaryError.errorMode]) { - primaryError = error; - } - } - return primaryError; - } - - - /** - * Set the main sync error icon across all windows - */ - this.setSyncIcon = function (e) { - e = this.parseSyncError(e); - - if (Zotero.Sync.Server.upgradeRequired) { - e.errorMode = 'upgrade'; - Zotero.Sync.Server.upgradeRequired = false; - } - - if (e.frontWindowOnly) { - // Fake an nsISimpleEnumerator with just the topmost window - var enumerator = { - _returned: false, - hasMoreElements: function () { - return !this._returned; - }, - getNext: function () { - if (this._returned) { - throw ("No more windows to return in Zotero.Sync.Runner.setSyncIcon()"); - } - this._returned = true; - var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] - .getService(Components.interfaces.nsIWindowMediator); - return wm.getMostRecentWindow("navigator:browser"); - } - }; - } - // Update all windows - else { - var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] - .getService(Components.interfaces.nsIWindowMediator); - var enumerator = wm.getEnumerator('navigator:browser'); - } - - while (enumerator.hasMoreElements()) { - var win = enumerator.getNext(); - if (!win.ZoteroPane) continue; - var doc = win.ZoteroPane.document; - - var button = doc.getElementById('zotero-tb-sync-error'); - this.setErrorIcon(button, [e]); - - var syncIcon = doc.getElementById('zotero-tb-sync'); - // Update sync icon state - syncIcon.setAttribute('status', e.errorMode ? e.errorMode : ""); - // Disable button while spinning - syncIcon.disabled = e.errorMode == 'animate'; - } - - // Clear status - Zotero.Sync.Runner.setSyncStatus(); - } - - - /** - * Set the sync icon tooltip message - */ - this.setSyncStatus = function (msg) { - _lastSyncStatus = msg; - - // If a label is registered, update it - if (_currentSyncStatusLabel) { - _updateSyncStatusLabel(); - } - } - - - this.parseSyncError = function (e) { - if (!e) { - return { parsed: true }; - } - - var parsed = { - parsed: true - }; - - // In addition to actual errors, string states (e.g., 'animate') - // can be passed - if (typeof e == 'string') { - parsed.errorMode = e; - return parsed; - } - - // Already parsed - if (e.parsed) { - return e; - } - - if (typeof e.libraryID != 'undefined') { - parsed.libraryID = e.libraryID; - } - parsed.errorMode = e.errorMode ? e.errorMode : 'error'; - - if (e.data) { - if (e.data.dialogText) { - parsed.message = e.data.dialogText; - } - if (typeof e.data.dialogButtonText != 'undefined') { - parsed.buttonText = e.data.dialogButtonText; - parsed.buttonCallback = e.data.dialogButtonCallback; - } - } - if (!parsed.message) { - parsed.message = e.message ? e.message : e; - parsed.fileName = e.fileName; - parsed.lineNumber = e.lineNumber; - } - - parsed.frontWindowOnly = !!(e && e.data && e.data.frontWindowOnly); - - return parsed; - } - - - /** - * Set the state of the sync error icon and add an onclick to populate - * the error panel - */ - this.setErrorIcon = function (icon, errors) { - if (!errors || !errors.length) { - icon.hidden = true; - icon.onclick = null; - return; - } - - // TEMP: for now, use the first error - var e = this.getPrimaryError(errors); - - if (!e.errorMode) { - icon.hidden = true; - icon.onclick = null; - return; - } - - icon.hidden = false; - icon.setAttribute('mode', e.errorMode); - icon.onclick = function () { - var doc = this.ownerDocument; - - var panel = Zotero.Sync.Runner.updateErrorPanel(doc, errors); - - panel.openPopup(this, "after_end", 15, 0, false, false); - } - } - - - this.updateErrorPanel = function (doc, errors) { - var panel = doc.getElementById('zotero-sync-error-panel'); - - // Clear existing panel content - while (panel.hasChildNodes()) { - panel.removeChild(panel.firstChild); - } - - var e = this - - for each(var e in errors.concat()) { - e = this.parseSyncError(e); - - var box = doc.createElement('vbox'); - var label = doc.createElement('label'); - if (typeof e.libraryID != 'undefined') { - label.className = "zotero-sync-error-panel-library-name"; - if (e.libraryID == 0) { - var libraryName = Zotero.getString('pane.collections.library'); - } - else { - let group = Zotero.Groups.getByLibraryID(e.libraryID); - var libraryName = group.name; - } - label.setAttribute('value', libraryName); - } - var content = doc.createElement('hbox'); - var buttons = doc.createElement('hbox'); - buttons.pack = 'end'; - box.appendChild(label); - box.appendChild(content); - box.appendChild(buttons); - - var msg = e.message; - /*if (e.fileName) { - msg += '\n\nFile: ' + e.fileName + '\nLine: ' + e.lineNumber; - }*/ - - var desc = doc.createElement('description'); - desc.textContent = msg; - // Make the text selectable - desc.setAttribute('style', '-moz-user-select: text; cursor: text'); - content.appendChild(desc); - - // If not an error and there's no explicit button text, don't show - // button to report errors - if (e.errorMode != 'error' && typeof e.buttonText == 'undefined') { - e.buttonText = null; - } - - if (e.buttonText !== null) { - if (typeof e.buttonText == 'undefined') { - var buttonText = Zotero.getString('errorReport.reportError'); - var buttonCallback = function () { - doc.defaultView.ZoteroPane.reportErrors(); - }; - } - else { - var buttonText = e.buttonText; - var buttonCallback = e.buttonCallback; - } - - var button = doc.createElement('button'); - button.setAttribute('label', buttonText); - button.onclick = function () { - buttonCallback.call(this); - panel.hidePopup(); - } - buttons.appendChild(button); - } - - panel.appendChild(box) - - // TEMP: Only show one error for now - break; - } - - return panel; - } - - - /** - * Register label in sync icon tooltip to receive updates - * - * If no label passed, unregister current label - * - * @param {Tooltip} [label] - */ - this.registerSyncStatusLabel = function (statusLabel, lastSyncLabel) { - _currentSyncStatusLabel = statusLabel; - _currentLastSyncLabel = lastSyncLabel; - if (_currentSyncStatusLabel) { - _updateSyncStatusLabel(); - } - } - - - function _updateSyncStatusLabel() { - if (_lastSyncStatus) { - _currentSyncStatusLabel.value = _lastSyncStatus; - _currentSyncStatusLabel.hidden = false; - } - else { - _currentSyncStatusLabel.hidden = true; - } - - // Always update last sync time - var lastSyncTime = Zotero.Sync.Server.lastLocalSyncTime; - if (lastSyncTime) { - var time = new Date(lastSyncTime * 1000); - var msg = Zotero.Date.toRelativeDate(time); - } - // Don't show "Not yet synced" if a sync is in progress - else if (_lastSyncStatus) { - _currentLastSyncLabel.hidden = true; - return; - } - else { - var msg = Zotero.getString('sync.status.notYetSynced'); - } - - _currentLastSyncLabel.value = Zotero.localeJoin([ - Zotero.getString('sync.status.lastSync'), - msg - ]); - _currentLastSyncLabel.hidden = false; - } -} - - -Zotero.Sync.Runner.EventListener = { - init: function () { - // Initialize save observer - Zotero.Notifier.registerObserver(this); - }, - - notify: function (event, type, ids, extraData) { - // TODO: skip others - if (event == 'refresh' || event == 'redraw') { - return; - } - - if (Zotero.Prefs.get('sync.autoSync') && Zotero.Sync.Server.enabled) { - Zotero.Sync.Runner.setSyncTimeout(false, false, true); - } - } -} - - -Zotero.Sync.Runner.IdleListener = { - _idleTimeout: 3600, - _backTimeout: 900, - - init: function () { - // DEBUG: Allow override for testing - var idleTimeout = Zotero.Prefs.get("sync.autoSync.idleTimeout"); - if (idleTimeout) { - this._idleTimeout = idleTimeout; - } - var backTimeout = Zotero.Prefs.get("sync.autoSync.backTimeout"); - if (backTimeout) { - this._backTimeout = backTimeout; - } - - if (Zotero.Prefs.get("sync.autoSync")) { - this.register(); - } - }, - - register: function () { - Zotero.debug("Initializing sync idle observer"); - var idleService = Components.classes["@mozilla.org/widget/idleservice;1"] - .getService(Components.interfaces.nsIIdleService); - idleService.addIdleObserver(this, this._idleTimeout); - idleService.addIdleObserver(this._backObserver, this._backTimeout); - }, - - observe: function (subject, topic, data) { - if (topic != 'idle') { - return; - } - - if (!Zotero.Sync.Server.enabled - || Zotero.Sync.Server.syncInProgress - || Zotero.Sync.Storage.syncInProgress) { - return; - } - - // TODO: move to Runner.sync()? - if (Zotero.locked) { - Zotero.debug('Zotero is locked -- skipping idle sync', 4); - return; - } - - if (Zotero.Sync.Server.manualSyncRequired) { - Zotero.debug('Manual sync required -- skipping idle sync', 4); - return; - } - - Zotero.debug("Beginning idle sync"); - - Zotero.Sync.Runner.sync({ - background: true - }); - Zotero.Sync.Runner.setSyncTimeout(this._idleTimeout, true, true); - }, - - _backObserver: { - observe: function (subject, topic, data) { - if (topic != 'back') { - return; - } - - Zotero.Sync.Runner.clearSyncTimeout(); - if (!Zotero.Sync.Server.enabled - || Zotero.Sync.Server.syncInProgress - || Zotero.Sync.Storage.syncInProgress) { - return; - } - Zotero.debug("Beginning return-from-idle sync"); - Zotero.Sync.Runner.sync({ - background: true - }); - } - }, - - unregister: function () { - Zotero.debug("Stopping sync idle observer"); - var idleService = Components.classes["@mozilla.org/widget/idleservice;1"] - .getService(Components.interfaces.nsIIdleService); - idleService.removeIdleObserver(this, this._idleTimeout); - idleService.removeIdleObserver(this._backObserver, this._backTimeout); - } -} - - - /** * Methods for syncing with the Zotero Server */ @@ -1314,15 +470,6 @@ Zotero.Sync.Server = new function () { this.__defineGetter__("sessionIDComponent", function () { return 'sessionid=' + _sessionID; }); - this.__defineGetter__("lastRemoteSyncTime", function () { - return Zotero.DB.valueQuery("SELECT version FROM version WHERE schema='lastremotesync'"); - }); - this.__defineSetter__("lastRemoteSyncTime", function (val) { - Zotero.DB.query("REPLACE INTO version VALUES ('lastremotesync', ?)", val); - }); - this.__defineGetter__("lastLocalSyncTime", function () { - return Zotero.DB.valueQuery("SELECT version FROM version WHERE schema='lastlocalsync'"); - }); this.__defineSetter__("lastLocalSyncTime", function (val) { Zotero.DB.query("REPLACE INTO version VALUES ('lastlocalsync', ?)", { int: val }); }); @@ -2338,130 +1485,7 @@ Zotero.Sync.Server = new function () { } - /** - * Make sure we're syncing with the same account we used last time - * - * @param {Integer} userID New userID - * @param {Integer} libraryID New libraryID - * @param {Integer} noServerData The server account is empty — this is - * the account after a server clear - * @return 1 if sync should continue, 0 if sync should restart, -1 if sync should cancel - */ - function _checkSyncUser(userID, libraryID, noServerData) { - if (Zotero.DB.transactionInProgress()) { - throw ("Transaction in progress in Zotero.Sync.Server._checkSyncUser"); - } - - Zotero.DB.beginTransaction(); - - var sql = "SELECT value FROM settings WHERE setting='account' AND key='username'"; - var lastUsername = Zotero.DB.valueQuery(sql); - var username = Zotero.Sync.Server.username; - var lastUserID = Zotero.userID; - var lastLibraryID = Zotero.libraryID; - - var restartSync = false; - - if (lastUserID && lastUserID != userID) { - var groups = Zotero.Groups.getAll(); - - var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"] - .getService(Components.interfaces.nsIPromptService); - var buttonFlags = (ps.BUTTON_POS_0) * (ps.BUTTON_TITLE_IS_STRING) - + (ps.BUTTON_POS_1) * (ps.BUTTON_TITLE_CANCEL) - + (ps.BUTTON_POS_2) * (ps.BUTTON_TITLE_IS_STRING) - + ps.BUTTON_POS_1_DEFAULT - + ps.BUTTON_DELAY_ENABLE; - - var msg = Zotero.getString('sync.lastSyncWithDifferentAccount', [lastUsername, username]); - - if (!noServerData) { - msg += " " + Zotero.getString('sync.localDataWillBeCombined', username); - // If there are local groups belonging to the previous user, - // we need to remove them - if (groups.length) { - msg += " " + Zotero.getString('sync.localGroupsWillBeRemoved1'); - } - msg += "\n\n" + Zotero.getString('sync.avoidCombiningData', lastUsername); - var syncButtonText = Zotero.getString('sync.sync'); - } - else if (groups.length) { - msg += " " + Zotero.getString('sync.localGroupsWillBeRemoved2', [username, lastUsername]); - var syncButtonText = Zotero.getString('sync.removeGroupsAndSync'); - } - // If there are no local groups and the server is empty, - // don't bother prompting - else { - var noPrompt = true; - } - - if (!noPrompt) { - var index = ps.confirmEx( - null, - Zotero.getString('general.warning'), - msg, - buttonFlags, - syncButtonText, - null, - Zotero.getString('sync.openSyncPreferences'), - null, {} - ); - - if (index > 0) { - if (index == 1) { - // Cancel - } - else if (index == 2) { - var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] - .getService(Components.interfaces.nsIWindowMediator); - var lastWin = wm.getMostRecentWindow("navigator:browser"); - lastWin.ZoteroPane.openPreferences('zotero-prefpane-sync'); - } - - Zotero.DB.commitTransaction(); - return -1; - } - - // Delete all local groups - for each(var group in groups) { - group.erase(); - } - - restartSync = true; - } - } - - if (lastUserID != userID || lastLibraryID != libraryID) { - if (!lastLibraryID) { - var repl = "local/" + Zotero.Users.getLocalUserKey(); - } - - Zotero.userID = userID; - Zotero.libraryID = libraryID; - - // Update libraryID in relations, which we store for the local - // for some reason. All other objects use null for the local library. - if (lastUserID && lastLibraryID) { - Zotero.Relations.updateUser(lastUserID, lastLibraryID, userID, libraryID); - - Zotero.Sync.Server.resetClient(); - Zotero.Sync.Storage.resetAllSyncStates(); - } - // Replace local user key with libraryID, in case duplicates were - // merged before the first sync - else if (!lastLibraryID) { - Zotero.Relations.updateUser(repl, repl, userID, libraryID); - } - } - - if (lastUsername != username) { - Zotero.username = username; - } - - Zotero.DB.commitTransaction(); - - return restartSync ? 0 : 1; - } + @@ -2475,110 +1499,6 @@ Zotero.Sync.Server = new function () { var code = xmlhttp.responseXML.childNodes[0].firstChild.getAttribute('code'); return (code == 'INVALID_SESSION_ID') || (code == 'SESSION_TIMED_OUT'); } - - - function _error(e, extraInfo, skipReload) { - if (e instanceof Zotero.Error) { - switch (e.error) { - case Zotero.Error.ERROR_MISSING_OBJECT: - case Zotero.Error.ERROR_FULL_SYNC_REQUIRED: - // Let current sync fail, and then do a full sync - var background = Zotero.Sync.Runner.background; - setTimeout(function () { - if (Zotero.Prefs.get('sync.debugNoAutoResetClient')) { - Components.utils.reportError("Skipping automatic client reset due to debug pref"); - return; - } - if (!Zotero.Sync.Server.canAutoResetClient) { - Components.utils.reportError("Client has already been auto-reset in Zotero.Sync.Server._error() -- manual sync required"); - return; - } - Zotero.Sync.Server.resetClient(); - Zotero.Sync.Server.canAutoResetClient = false; - Zotero.Sync.Runner.sync({ - background: background - }); - }, 1); - break; - - case Zotero.Error.ERROR_SYNC_USERNAME_NOT_SET: - case Zotero.Error.ERROR_INVALID_SYNC_LOGIN: - // TODO: the setTimeout() call below should just simulate a click on the sync error icon - // instead of creating its own dialog, but setSyncIcon() doesn't yet provide full control - // over dialog title and primary button text/action, which is why this version of the - // dialog is a bit uglier than the manual click version - // TODO: localize (=>done) and combine with below (=>?) - var msg = Zotero.getString('sync.error.invalidLogin.text'); - e.data = {}; - e.data.dialogText = msg; - e.data.dialogButtonText = Zotero.getString('sync.openSyncPreferences'); - e.data.dialogButtonCallback = function () { - var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] - .getService(Components.interfaces.nsIWindowMediator); - var win = wm.getMostRecentWindow("navigator:browser"); - win.ZoteroPane.openPreferences("zotero-prefpane-sync"); - }; - - // Manual click - setTimeout(function () { - var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] - .getService(Components.interfaces.nsIWindowMediator); - var win = wm.getMostRecentWindow("navigator:browser"); - - var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"] - .getService(Components.interfaces.nsIPromptService); - var buttonFlags = (ps.BUTTON_POS_0) * (ps.BUTTON_TITLE_IS_STRING) - + (ps.BUTTON_POS_1) * (ps.BUTTON_TITLE_CANCEL); - if (e.error == Zotero.Error.ERROR_SYNC_USERNAME_NOT_SET) { - var title = Zotero.getString('sync.error.usernameNotSet'); - var msg = Zotero.getString('sync.error.usernameNotSet.text'); - } - else { - var title = Zotero.getString('sync.error.invalidLogin'); - var msg = Zotero.getString('sync.error.invalidLogin.text'); - } - var index = ps.confirmEx( - win, - title, - msg, - buttonFlags, - Zotero.getString('sync.openSyncPreferences'), - null, null, null, {} - ); - - if (index == 0) { - win.ZoteroPane.openPreferences("zotero-prefpane-sync"); - return; - } - }, 1); - break; - } - } - - if (e.message == 'script stack space quota is exhausted') { - var e = "Firefox 4.0 or higher is required to process sync operations of this size."; - } - - if (extraInfo) { - // Server errors will generally be HTML - extraInfo = Zotero.Utilities.unescapeHTML(extraInfo); - Components.utils.reportError(extraInfo); - } - - Zotero.debug(e, 1); - Components.utils.reportError(e); - - _syncInProgress = false; - Zotero.DB.rollbackAllTransactions(); - if (!skipReload) { - Zotero.reloadDataObjects(); - } - Zotero.Sync.EventListener.resetIgnored(); - - _callbacks.onError(e); - - throw (e); - } } diff --git a/chrome/content/zotero/xpcom/sync/syncAPIClient.js b/chrome/content/zotero/xpcom/sync/syncAPIClient.js new file mode 100644 index 000000000..ba3f34ba9 --- /dev/null +++ b/chrome/content/zotero/xpcom/sync/syncAPIClient.js @@ -0,0 +1,445 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2014 Center for History and New Media + George Mason University, Fairfax, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +if (!Zotero.Sync) { + Zotero.Sync = {}; +} + +Zotero.Sync.APIClient = function (options) { + this.baseURL = options.baseURL; + this.apiKey = options.apiKey; + this.concurrentCaller = options.concurrentCaller; + + if (options.apiVersion == undefined) { + throw new Error("options.apiVersion not set"); + } + this.apiVersion = options.apiVersion; +} + +Zotero.Sync.APIClient.prototype = { + MAX_OBJECTS_PER_REQUEST: 100, + + + getKeyInfo: Zotero.Promise.coroutine(function* () { + var uri = this.baseURL + "keys/" + this.apiKey; + var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 404] }); + if (xmlhttp.status == 404) { + return false; + } + var json = this._parseJSON(xmlhttp.responseText); + delete json.key; + return json; + }), + + + /** + * Get group metadata versions + * + * Note: This is the version for group metadata, not library data. + */ + getGroupVersions: Zotero.Promise.coroutine(function* (userID) { + if (!userID) throw new Error("User ID not provided"); + + var uri = this.baseURL + "users/" + userID + "/groups?format=versions"; + var xmlhttp = yield this._makeRequest("GET", uri); + return this._parseJSON(xmlhttp.responseText); + }), + + + /** + * @param {Integer} groupID + * @return {Object|false} - Group metadata response, or false if group not found + */ + getGroupInfo: Zotero.Promise.coroutine(function* (groupID) { + if (!groupID) throw new Error("Group ID not provided"); + + var uri = this.baseURL + "groups/" + groupID; + var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 404] }); + if (xmlhttp.status == 404) { + return false; + } + return this._parseJSON(xmlhttp.responseText); + }), + + + getSettings: Zotero.Promise.coroutine(function* (libraryType, libraryTypeID, since) { + var params = { + target: "settings", + libraryType: libraryType, + libraryTypeID: libraryTypeID + }; + if (since) { + params.since = since; + } + var uri = this._buildRequestURI(params); + var options = { + successCodes: [200, 304] + }; + if (since) { + options.headers = { + "If-Modified-Since-Version": since + }; + } + var xmlhttp = yield this._makeRequest("GET", uri, options); + if (xmlhttp.status == 304) { + return false; + } + var libraryVersion = xmlhttp.getResponseHeader('Last-Modified-Version'); + if (!libraryVersion) { + throw new Error("Last-Modified-Version not provided"); + } + return { + libraryVersion: libraryVersion, + settings: this._parseJSON(xmlhttp.responseText) + }; + }), + + + getDeleted: Zotero.Promise.coroutine(function* (libraryType, libraryTypeID, since) { + var params = { + target: "deleted", + libraryType: libraryType, + libraryTypeID: libraryTypeID, + since: since || 0 + }; + var uri = this._buildRequestURI(params); + var xmlhttp = yield this._makeRequest("GET", uri); + var libraryVersion = xmlhttp.getResponseHeader('Last-Modified-Version'); + if (!libraryVersion) { + throw new Error("Last-Modified-Version not provided"); + } + return { + libraryVersion: libraryVersion, + deleted: this._parseJSON(xmlhttp.responseText) + }; + }), + + + /** + * Return a promise for a JS object with object keys as keys and version + * numbers as values. By default, returns all objects in the library. + * Additional parameters (such as 'since', 'sincetime', 'libraryVersion') + * can be passed in 'params'. + * + * @param {String} libraryType 'user' or 'group' + * @param {Integer} libraryTypeID userID or groupID + * @param {String} objectType 'item', 'collection', 'search' + * @param {Object} queryParams Query parameters (see _buildRequestURI()) + * @return {Promise|FALSE} Object with 'libraryVersion' and 'results' + */ + getVersions: Zotero.Promise.coroutine(function* (libraryType, libraryTypeID, objectType, queryParams, libraryVersion) { + var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); + + var params = { + target: objectTypePlural, + libraryType: libraryType, + libraryTypeID: libraryTypeID, + format: 'versions' + }; + if (queryParams) { + for (let i in queryParams) { + params[i] = queryParams[i]; + } + } + if (objectType == 'item') { + params.includeTrashed = 1; + } + + // TODO: Use pagination + var uri = this._buildRequestURI(params); + + var options = { + successCodes: [200, 304] + }; + if (libraryVersion) { + options.headers = { + "If-Modified-Since-Version": libraryVersion + }; + } + var xmlhttp = yield this._makeRequest("GET", uri, options); + if (xmlhttp.status == 304) { + return false; + } + var libraryVersion = xmlhttp.getResponseHeader('Last-Modified-Version'); + if (!libraryVersion) { + throw new Error("Last-Modified-Version not provided"); + } + return { + libraryVersion: libraryVersion, + versions: this._parseJSON(xmlhttp.responseText) + }; + }), + + + /** + * Retrieve JSON from API for requested objects + * + * If necessary, multiple API requests will be made. + * + * @param {String} libraryType - 'user', 'group' + * @param {Integer} libraryTypeID - userID or groupID + * @param {String} objectType - 'collection', 'item', 'search' + * @param {String[]} objectKeys - Keys of objects to request + * @return {Array>} - An array of promises for batches of JSON objects + * or Errors for failures + */ + downloadObjects: function (libraryType, libraryTypeID, objectType, objectKeys) { + if (!objectKeys.length) { + return []; + } + + // If more than max per request, call in batches + if (objectKeys.length > this.MAX_OBJECTS_PER_REQUEST) { + let allKeys = objectKeys.concat(); + let promises = []; + while (true) { + let requestKeys = allKeys.splice(0, this.MAX_OBJECTS_PER_REQUEST) + if (!requestKeys.length) { + break; + } + let promise = this.downloadObjects( + libraryType, + libraryTypeID, + objectType, + requestKeys + )[0]; + if (promise) { + promises.push(promise); + } + } + return promises; + } + + // Otherwise make request + var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); + + Zotero.debug("Retrieving " + objectKeys.length + " " + + (objectKeys.length == 1 ? objectType : objectTypePlural)); + + var params = { + target: objectTypePlural, + libraryType: libraryType, + libraryTypeID: libraryTypeID, + format: 'json' + }; + params[objectType + "Key"] = objectKeys.join(","); + if (objectType == 'item') { + params.includeTrashed = 1; + } + var uri = this._buildRequestURI(params); + + return [ + this._makeRequest("GET", uri) + .then(function (xmlhttp) { + return this._parseJSON(xmlhttp.responseText) + }.bind(this)) + // Return the error without failing the whole chain + .catch(function (e) { + if (e instanceof Zotero.HTTP.UnexpectedStatusException && e.is4xx()) { + Zotero.logError(e); + throw e; + } + Zotero.logError(e); + return e; + }) + ]; + }, + + + uploadObjects: Zotero.Promise.coroutine(function* (libraryType, libraryTypeID, objectType, method, version, objects) { + throw new Error("Uploading disabled"); + + if (method != 'POST' && method != 'PATCH') { + throw new Error("Invalid method '" + method + "'"); + } + + var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); + + Zotero.debug("Uploading " + objects.length + " " + + (objects.length == 1 ? objectType : objectTypePlural)); + + Zotero.debug("Sending If-Unmodified-Since-Version: " + version); + + var json = JSON.stringify(objects); + var params = { + target: objectTypePlural, + libraryType: libraryType, + libraryTypeID: libraryTypeID + }; + var uri = this._buildRequestURI(params); + + var xmlhttp = yield this._makeRequest(method, uri, { + headers: { + "If-Unmodified-Since-Version": version + }, + body: json, + successCodes: [200, 412] + }); + // Avoid logging error from Zotero.HTTP.request() in ConcurrentCaller + if (xmlhttp.status == 412) { + Zotero.debug("Server returned 412: " + xmlhttp.responseText, 2); + throw new Zotero.HTTP.UnexpectedStatusException(xmlhttp); + } + var libraryVersion = xmlhttp.getResponseHeader('Last-Modified-Version'); + if (!libraryVersion) { + throw new Error("Last-Modified-Version not provided"); + } + return { + libraryVersion: libraryVersion, + results: this._parseJSON(xmlhttp.responseText) + }; + }), + + + _buildRequestURI: function (params) { + var uri = this.baseURL; + + switch (params.libraryType) { + case 'publications': + uri += 'users/' + params.libraryTypeID + '/' + params.libraryType; + break; + + default: + uri += params.libraryType + 's/' + params.libraryTypeID; + break; + } + + uri += "/" + params.target; + + if (params.objectKey) { + uri += "/" + params.objectKey; + } + + var queryString = '?'; + var queryParamsArray = []; + var queryParamOptions = [ + 'session', + 'format', + 'include', + 'includeTrashed', + 'itemType', + 'itemKey', + 'collectionKey', + 'searchKey', + 'linkMode', + 'start', + 'limit', + 'sort', + 'direction', + 'since', + 'sincetime' + ]; + queryParams = {}; + + for (let option in params) { + let value = params[option]; + if (value !== undefined && value !== '' && queryParamOptions.indexOf(option) != -1) { + queryParams[option] = value; + } + } + + for (let index in queryParams) { + let value = queryParams[index]; + if (Array.isArray(value)) { + value.forEach(function(v, i) { + queryParamsArray.push(encodeURIComponent(index) + '=' + encodeURIComponent(v)); + }); + } + else { + queryParamsArray.push(encodeURIComponent(index) + '=' + encodeURIComponent(value)); + } + } + + return uri + (queryParamsArray.length ? "?" + queryParamsArray.join('&') : ""); + }, + + + _makeRequest: function (method, uri, options) { + if (!options) { + options = {}; + } + if (!options.headers) { + options.headers = {}; + } + options.headers["Zotero-API-Version"] = this.apiVersion; + options.dontCache = true; + options.foreground = !options.background; + options.responseType = options.responseType || 'text'; + if (this.apiKey) { + options.headers.Authorization = "Bearer " + this.apiKey; + } + var self = this; + return this.concurrentCaller.fcall(function () { + return Zotero.HTTP.request(method, uri, options) + .catch(function (e) { + if (e instanceof Zotero.HTTP.UnexpectedStatusException) { + self._checkResponse(e.xmlhttp); + } + throw e; + }); + }); + }, + + + _parseJSON: function (json) { + try { + json = JSON.parse(json); + } + catch (e) { + Zotero.debug(e, 1); + Zotero.debug(json, 1); + throw e; + } + return json; + }, + + + _checkResponse: function (xmlhttp) { + this._checkBackoff(xmlhttp); + this._checkAuth(xmlhttp); + }, + + + _checkAuth: function (xmlhttp) { + if (xmlhttp.status == 403) { + var e = new Zotero.Error(Zotero.getString('sync.error.invalidLogin'), "INVALID_SYNC_LOGIN"); + e.fatal = true; + throw e; + } + }, + + + _checkBackoff: function (xmlhttp) { + var backoff = xmlhttp.getResponseHeader("Backoff"); + if (backoff) { + // Sanity check -- don't wait longer than an hour + if (backoff > 3600) { + // TODO: Update status? + + this.concurrentCaller.pause(backoff * 1000); + } + } + } +} diff --git a/chrome/content/zotero/xpcom/sync/syncEngine.js b/chrome/content/zotero/xpcom/sync/syncEngine.js new file mode 100644 index 000000000..f3566511e --- /dev/null +++ b/chrome/content/zotero/xpcom/sync/syncEngine.js @@ -0,0 +1,1075 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2014 Center for History and New Media + George Mason University, Fairfax, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +if (!Zotero.Sync.Data) { + Zotero.Sync.Data = {}; +} + +// TODO: move? +Zotero.Sync.Data.conflictDelayIntervals = [10000, 20000, 40000, 60000, 120000, 240000, 300000]; +Zotero.Sync.Data.failureDelayIntervals = [2500, 5000, 10000, 20000, 40000, 60000, 120000, 240000, 300000]; + +/** + * An Engine manages sync processes for a given library + * + * @param {Object} options + * @param {Zotero.Sync.APIClient} options.apiClient + * @param {Integer} options.libraryID + */ +Zotero.Sync.Data.Engine = function (options) { + if (options.apiClient == undefined) { + throw new Error("options.apiClient not set"); + } + if (options.libraryID == undefined) { + throw new Error("options.libraryID not set"); + } + + this.apiClient = options.apiClient; + this.libraryID = options.libraryID; + this.libraryName = Zotero.Libraries.getName(options.libraryID); + this.libraryType = Zotero.Libraries.getType(options.libraryID); + switch (this.libraryType) { + case 'user': + case 'publications': + this.libraryTypeID = Zotero.Users.getCurrentUserID(); + break; + + case 'group': + this.libraryTypeID = Zotero.Groups.getGroupIDFromLibraryID(options.libraryID); + break; + } + this.setStatus = options.setStatus || function () {}; + this.onError = options.onError || function (e) {}; + this.stopOnError = options.stopOnError; + this.requests = []; + this.uploadBatchSize = 25; + + this.failed = false; + + this.options = { + setStatus: this.setStatus, + stopOnError: this.stopOnError, + onError: this.onError + } + + this.syncCachePromise = Zotero.Promise.resolve().bind(this); +}; + +Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_SUCCESS = 1; +Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_NOTHING_TO_UPLOAD = 2; +Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_LIBRARY_CONFLICT = 3; +Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_OBJECT_CONFLICT = 4; + +Zotero.Sync.Data.Engine.prototype.start = Zotero.Promise.coroutine(function* () { + Zotero.debug("Starting data sync for " + this.libraryName); + + // TODO: Handle new/changed user when setting key + if (this.libraryType == 'user' && !this.libraryTypeID) { + let info = yield this.apiClient.getKeyInfo(); + Zotero.debug("Got userID " + info.userID + " for API key"); + this.libraryTypeID = info.userID; + } + + // Check if we've synced this library with the current architecture yet + var libraryVersion = Zotero.Libraries.getVersion(this.libraryID); + if (!libraryVersion || libraryVersion == -1) { + let versionResults = yield this._upgradeCheck(); + if (versionResults) { + libraryVersion = Zotero.Libraries.getVersion(this.libraryID) + } + + // Perform a full sync if necessary, passing the getVersions() results if available. + // + // The full-sync flag (libraryID == -1) is set at the end of a successful upgrade, so this + // won't run for installations that have just never synced before (which also lack library + // versions). We can't rely on last classic sync time because it's cleared after the last + // library is upgraded. + // + // Version results won't be available if an upgrade happened on a previous run but the + // full sync failed. + if (libraryVersion == -1) { + yield this._fullSync(versionResults); + } + } + + var autoReset = false; + + sync: + while (true) { + let uploadResult = yield this._startUpload(); + Zotero.debug("UPLOAD RESULT WITH " + uploadResult); + + switch (uploadResult) { + // If upload succeeded, we're done + case this.UPLOAD_RESULT_SUCCESS: + break sync; + + case this.UPLOAD_RESULT_OBJECT_CONFLICT: + if (Zotero.Prefs.get('sync.debugNoAutoResetClient')) { + throw new Error("Skipping automatic client reset due to debug pref"); + } + if (autoReset) { + throw new Error(this.libraryName + " has already been auto-reset"); + } + Zotero.logError("Object in " + this.libraryName + " is out of date -- resetting library"); + autoReset = true; + yield this._fullSync(); + break; + + // If conflict, start at beginning with downloads + case this.UPLOAD_RESULT_NOTHING_TO_UPLOAD: + let localChanges = yield this._startDownload(); + if (!localChanges) { + break sync; + } + break; + + case this.UPLOAD_RESULT_LIBRARY_CONFLICT: + yield this._startDownload(); + + if (!gen) { + var gen = Zotero.Utilities.Internal.delayGenerator( + Zotero.Sync.Data.delayIntervals, 60 * 1000 + ); + } + // After the first upload version conflict (which is expected after remote changes), + // start delaying to give other sync sessions time to complete + else { + let keepGoing = yield gen.next(); + if (!keepGoing) { + throw new Error("Could not sync " + this.libraryName + " -- too many retries"); + } + } + } + } + + // TEMP: make more reliable + while (this.syncCachePromise.isPending()) { + Zotero.debug("Waiting for sync cache to be processed"); + yield this.syncCachePromise; + yield Zotero.Promise.delay(50); + } + + yield Zotero.Libraries.updateLastSyncTime(this.libraryID); + + Zotero.debug("Done syncing " + this.libraryName); +}); + + +/** + * Stop all active requests + * + * @return {Promise} Promise from Zotero.Promise.settle() + */ +Zotero.Sync.Data.Engine.prototype.stop = function () { + var funcs; + var request; + while (request = this.requests.shift()) { + funcs.push(() => request.stop()); + } + return Zotero.Promise.settle(funcs); +} + + +/** + * Download updated objects from API and save to local cache + * + * @return {Boolean} True if an upload is needed, false otherwise + */ +Zotero.Sync.Data.Engine.prototype._startDownload = Zotero.Promise.coroutine(function* () { + var localChanges = false; + var libraryVersion = Zotero.Libraries.getVersion(this.libraryID); + var lastLibraryVersion; + + var gen = Zotero.Utilities.Internal.delayGenerator( + Zotero.Sync.Data.delayIntervals, 60 * 60 * 1000 + ); + + loop: + while (true) { + // Get synced settings first, since they affect how other data is displayed + lastLibraryVersion = yield this._downloadSettings(libraryVersion); + if (lastLibraryVersion === false) { + break; + } + + // + // Get other object types + // + for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(this.libraryID)) { + this._failedCheck(); + this._processCache(objectType); + + let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); + let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); + + // Get versions of all objects updated remotely since the current local library version + Zotero.debug("Checking for updated " + objectTypePlural + " in " + this.libraryName); + let results = yield this.apiClient.getVersions( + this.libraryType, + this.libraryTypeID, + objectType, + libraryVersion ? { since: libraryVersion } : undefined + ); + + Zotero.debug("VERSIONS:"); + Zotero.debug(JSON.stringify(results)); + + if (lastLibraryVersion) { + // If something else modified the remote library while we were getting updates, + // wait for increasing amounts of time before trying again, and then start from + // the beginning + if (lastLibraryVersion != results.libraryVersion) { + Zotero.logError("Library version changed since last download -- restarting sync"); + let keepGoing = yield gen.next(); + if (!keepGoing) { + throw new Error("Could not update " + this.libraryName + " -- library in use"); + } + continue loop; + } + } + else { + lastLibraryVersion = results.libraryVersion; + } + + var numObjects = Object.keys(results.versions).length; + if (!numObjects) { + Zotero.debug("No " + objectTypePlural + " modified remotely since last check"); + continue; + } + Zotero.debug(numObjects + " " + (numObjects == 1 ? objectType : objectTypePlural) + + " modified since last check"); + + let keys = []; + for (let key in results.versions) { + // Skip objects that are already up-to-date in the sync cache. Generally all returned + // objects should have newer version numbers, but there are some situations, such as + // full syncs or interrupted syncs, where we may get versions for objects that are + // already up-to-date locally. + let version = yield Zotero.Sync.Data.Local.getLatestCacheObjectVersion( + objectType, this.libraryID, key + ); + if (version == results.versions[key]) { + Zotero.debug("Skipping up-to-date " + objectType + " " + this.libraryID + "/" + key); + continue; + } + keys.push(key); + } + + if (keys.length) { + yield this._downloadObjects(objectType, keys); + } + } + + // Wait for sync process to clear + // TEMP: make more reliable + while (this.syncCachePromise.isPending()) { + Zotero.debug("Waiting for sync cache to be processed"); + yield this.syncCachePromise; + yield Zotero.Promise.delay(50); + } + + // + // Get deleted objects + // + results = yield this.apiClient.getDeleted( + this.libraryType, + this.libraryTypeID, + libraryVersion + ); + if (lastLibraryVersion) { + // If something else modified the remote library while we were getting updates, + // wait for increasing amounts of time before trying again, and then start from + // the beginning + if (lastLibraryVersion != results.libraryVersion) { + Zotero.logError("Library version changed since last download -- restarting sync"); + let keepGoing = yield gen.next(); + if (!keepGoing) { + throw new Error("Could not update " + this.libraryName + " -- library in use"); + } + continue loop; + } + } + else { + lastLibraryVersion = results.libraryVersion; + } + + var numObjects = Object.keys(results.deleted).reduce((n, k) => n + results.deleted[k].length, 0); + if (numObjects) { + Zotero.debug(numObjects + " objects deleted remotely since last check"); + + // Process deletions + for (let objectTypePlural in results.deleted) { + let objectType = Zotero.DataObjectUtilities.getObjectTypeSingular(objectTypePlural); + let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); + let toDelete = []; + for (let key of results.deleted[objectTypePlural]) { + // TODO: Remove from request? + if (objectType == 'tag') { + continue; + } + + if (objectType == 'setting') { + let meta = yield Zotero.SyncedSettings.getMetadata(this.libraryID, key); + if (!meta) { + continue; + } + if (meta.synced) { + yield Zotero.SyncedSettings.clear(this.libraryID, key, { + skipDeleteLog: true + }); + } + + // Ignore setting if changed locally + continue; + } + + let obj = yield objectsClass.getByLibraryAndKeyAsync( + this.libraryID, key, { noCache: true } + ); + if (!obj) { + continue; + } + if (obj.synced) { + toDelete.push(obj); + } + // Conflict resolution + else if (objectType == 'item') { + throw new Error("Unimplemented: delete conflict"); + } + + // Ignore deletion if collection/search changed locally + } + if (toDelete.length) { + yield Zotero.DB.executeTransaction(function* () { + for (let obj of toDelete) { + yield obj.erase({ + skipDeleteLog: true + }); + } + }); + } + } + } + else { + Zotero.debug("No objects deleted remotely since last check"); + } + + break; + } + + if (lastLibraryVersion) { + yield Zotero.Libraries.setVersion(this.libraryID, lastLibraryVersion); + } + + return localChanges; +}); + + +/** + * @param {Integer} libraryVersion - Last library version + * @return {Integer|Boolean} - Library version returned from server, or false if no changes since + * specified version + */ +Zotero.Sync.Data.Engine.prototype._downloadSettings = Zotero.Promise.coroutine(function* (libraryVersion) { + let results = yield this.apiClient.getSettings( + this.libraryType, + this.libraryTypeID, + libraryVersion + ); + // If library version hasn't changed remotely, the local library is up-to-date and we + // can skip all remaining downloads + if (results === false) { + Zotero.debug("Library " + this.libraryID + " hasn't been modified " + + "-- skipping further object downloads"); + return false; + } + var numObjects = Object.keys(results.settings).length; + if (numObjects) { + Zotero.debug(numObjects + " settings modified since last check"); + // Settings we process immediately rather than caching + for (let setting in results.settings) { + yield Zotero.SyncedSettings.set( + this.libraryID, + setting, + results.settings[setting].value, + results.settings[setting].version, + true + ); + } + } + else { + Zotero.debug("No settings modified remotely since last check"); + } + return results.libraryVersion; +}) + + +Zotero.Sync.Data.Engine.prototype._downloadObjects = Zotero.Promise.coroutine(function* (objectType, keys) { + var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); + var failureDelayGenerator = null; + + var lastLength = keys.length; + + while (true) { + this._failedCheck(); + + let lastError = false; + + // TODO: localize + this.setStatus( + "Downloading " + + (keys.length == 1 + ? "1 " + objectType + : Zotero.Utilities.numberFormat(keys.length, 0) + " " + objectTypePlural) + + " in " + this.libraryName + ); + + // Process batches as soon as they're available + yield Zotero.Promise.map( + this.apiClient.downloadObjects( + this.libraryType, + this.libraryTypeID, + objectType, + keys + ), + function (batch) { + this._failedCheck(); + + Zotero.debug("MAPPING"); + if (!Array.isArray(batch)) { + Zotero.debug("WE GOT AN ERROR"); + Components.utils.reportError(batch); + Zotero.debug(batch, 1); + this.failed = batch; + lastError = batch; + return; + } + + // Save objects to sync cache + return Zotero.Sync.Data.Local.saveCacheObjects( + objectType, this.libraryID, batch + ) + .then(function () { + let processedKeys = batch.map(item => item.key); + keys = Zotero.Utilities.arrayDiff(keys, processedKeys); + + // Create/update objects as they come in + this._processCache(objectType); + }.bind(this)); + }.bind(this) + ); + + if (!keys.length) { + Zotero.debug("All " + objectTypePlural + " for library " + + this.libraryID + " saved to sync cache"); + break; + } + + // If we're not making process, delay for increasing amounts of time + // and then keep going + if (keys.length == lastLength) { + if (!failureDelayGenerator) { + // Keep trying for up to an hour + failureDelayGenerator = Zotero.Utilities.Internal.delayGenerator( + Zotero.Sync.Data.failureDelayIntervals, 60 * 60 * 1000 + ); + } + let keepGoing = yield failureDelayGenerator.next(); + if (!keepGoing) { + Zotero.logError("Failed too many times"); + throw lastError; + } + } + else { + failureDelayGenerator = null; + } + + lastLength = keys.length; + } +}); + + +/** + * Get unsynced objects, build upload JSON, and start API requests + * + * @throws {Zotero.HTTP.UnexpectedStatusException} + * @return {Promise} - An upload result code (this.UPLOAD_RESULT_*) + */ +Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(function* () { + var libraryVersion = Zotero.Libraries.getVersion(this.libraryID); + + var uploadNeeded = false; + var objectIDs = {}; + + // Get unsynced local objects for each object type + for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(this.libraryID)) { + let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); + + let ids = yield Zotero.Sync.Data.Local.getUnsynced(this.libraryID, objectType); + if (!ids.length) { + Zotero.debug("No " + objectTypePlural + " to upload in " + this.libraryName); + continue; + } + Zotero.debug(ids.length + " " + + (ids.length == 1 ? objectType : objectTypePlural) + + " to upload in library " + this.libraryID); + objectIDs[objectType] = ids; + uploadNeeded = true; + } + + if (!uploadNeeded) { + return this.UPLOAD_RESULT_NOTHING_TO_UPLOAD; + } + + Zotero.debug(JSON.stringify(objectIDs)); + + for (let objectType in objectIDs) { + let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); + let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); + + let ids = objectIDs[objectType]; + let queue = []; + for (let id of ids) { + queue.push({ + id: id, + json: null, + tries: 0, + failed: false + }); + } + + let failureDelayGenerator = null; + + while (queue.length) { + // Get a slice of the queue and generate JSON for objects if necessary + let batch = []; + for (let i = 0; i < queue.length && queue.length < this.uploadBatchSize; i++) { + let o = queue[i]; + // Skip requests that failed with 4xx + if (o.failed) { + continue; + } + if (!o.json) { + o.json = yield this._getJSONForObject(objectType, o.id); + } + batch.push(o.json); + } + + // No more non-failed requests + if (!batch.length) { + break; + } + + // Remove selected and skipped objects from queue + queue.splice(0, batch.length); + + Zotero.debug("UPLOAD BATCH:"); + Zotero.debug(batch); + + let numSuccessful = 0; + try { + let json = yield this.apiClient.uploadObjects( + this.libraryType, + this.libraryTypeID, + objectType, + "POST", + libraryVersion, + batch + ); + + Zotero.debug('======'); + Zotero.debug(json); + + libraryVersion = json.libraryVersion; + yield Zotero.Libraries.setVersion(this.libraryID, json.libraryVersion); + + // Mark successful and unchanged objects as synced with new version + var toRemove = []; + for (let state of ['success', 'unchanged']) { + for (let index in json.results[state]) { + let key = json.results[state][index]; + if (key != batch[index].key) { + throw new Error("Key mismatch (" + key + " != " + batch[index].key + ")"); + } + let obj = yield objectsClass.getByLibraryAndKeyAsync( + this.libraryID, key, { noCache: true } + ); + obj.version = json.libraryVersion; + yield Zotero.Sync.Data.Local.markObjectAsSynced(obj) + numSuccessful++; + // Remove from batch to mark as successful + delete batch[index]; + } + } + + // Handle failed objects + for (let index in json.results.failed) { + let e = json.results.failed[index]; + Zotero.logError(e.message); + + // This shouldn't happen, because the upload request includes a library + // version and should prevent an outdated upload before the object version is + // checked. If it does, we need to do a full sync. + if (e.code == 412) { + return this.UPLOAD_RESULT_OBJECT_CONFLICT; + } + + if (this.stopOnError) { + Zotero.debug("WE FAILED!!!"); + throw new Error(e.message); + } + if (this.onError) { + this.onError(e.message); + } + batch[index].tries++; + // Mark 400 errors as permanently failed + if (e.code >= 400 && e.code < 500) { + batch[index].failed = true; + } + // 500 errors should stay in queue and be retried + } + + // Add failed objects back to end of queue + Zotero.debug("ADDING BACK FAILED"); + Zotero.debug(batch); + var numFailed = 0; + for (let o of batch) { + if (o !== undefined) { + queue.push(o); + // TODO: Clear JSON? + numFailed++; + } + } + Zotero.debug(queue); + Zotero.debug("Failed: " + numFailed, 2); + } + catch (e) { + if (e instanceof Zotero.HTTP.UnexpectedStatusException) { + if (e.status == 412) { + return this.UPLOAD_RESULT_LIBRARY_CONFLICT; + } + + // On 5xx, delay and retry + if (e.status >= 500 && e.status <= 600) { + if (!failureDelayGenerator) { + // Keep trying for up to an hour + failureDelayGenerator = Zotero.Utilities.Internal.delayGenerator( + Zotero.Sync.Data.failureDelayIntervals, 60 * 60 * 1000 + ); + } + let keepGoing = yield failureDelayGenerator.next(); + if (!keepGoing) { + Zotero.logError("Failed too many times"); + throw e; + } + continue; + } + } + throw e; + } + // If we didn't make any progress, bail + if (!numSuccessful) { + throw new Error("Made no progress during upload -- stopping"); + } + } + Zotero.debug("Done uploading " + objectTypePlural + " in library " + this.libraryID); + } + + return this.UPLOAD_RESULT_SUCCESS; +}); + + +Zotero.Sync.Data.Engine.prototype._getJSONForObject = function (objectType, id) { + return Zotero.DB.executeTransaction(function* () { + var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); + var obj = yield objectsClass.getAsync(id, { noCache: true }); + var cacheObj = false; + if (obj.version) { + cacheObj = yield Zotero.Sync.Data.Local.getCacheObject( + objectType, obj.libraryID, obj.key, obj.version + ); + } + return obj.toJSON({ + // JSON generation mode depends on whether a copy is in the cache + // and, failing that, whether the object is new + mode: cacheObj + ? "patch" + : (obj.version ? "full" : "new"), + includeKey: true, + includeVersion: true, // DEBUG: remove? + includeDate: true, + patchBase: cacheObj ? cacheObj.data : false + }); + }); +} + + +/** + * Upgrade library to current sync architecture + * + * This sets the 'synced' and 'version' properties based on classic last-sync times and object + * modification times. Objects are marked as: + * + * - synced=1 if modified locally before the last classic sync time + * - synced=0 (unchanged) if modified locally since the last classic sync time + * - version= if modified remotely before the last classic sync time + * - version=0 if modified remotely since the last classic sync time + * + * If both are 0, that's a conflict. + * + * @return {Object[]} - Objects returned from getVersions(), keyed by objectType, for use + * by _fullSync() + */ +Zotero.Sync.Data.Engine.prototype._upgradeCheck = Zotero.Promise.coroutine(function* () { + var libraryVersion = Zotero.Libraries.getVersion(this.libraryID); + if (libraryVersion) return; + + var lastLocalSyncTime = yield Zotero.DB.valueQueryAsync( + "SELECT version FROM version WHERE schema='lastlocalsync'" + ); + // Never synced with classic architecture, or already upgraded and full sync (which updates + // library version) didn't finish + if (!lastLocalSyncTime) return; + + Zotero.debug("Upgrading library to current sync architecture"); + + var lastRemoteSyncTime = yield Zotero.DB.valueQueryAsync( + "SELECT version FROM version WHERE schema='lastremotesync'" + ); + // Shouldn't happen + if (!lastRemoteSyncTime) lastRemoteSyncTime = lastLocalSyncTime; + + var objectTypes = Zotero.DataObjectUtilities.getTypesForLibrary(this.libraryID); + + // Mark all items modified locally before the last classic sync time as synced + if (lastLocalSyncTime) { + lastLocalSyncTime = new Date(lastLocalSyncTime * 1000); + for (let objectType of objectTypes) { + let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); + let ids = yield objectsClass.getOlder(this.libraryID, lastLocalSyncTime); + yield objectsClass.updateSynced(ids, true); + } + } + + var versionResults = {}; + var currentVersions = {}; + var gen; + loop: + while (true) { + let lastLibraryVersion = 0; + for (let objectType of objectTypes) { + currentVersions[objectType] = {}; + + let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); + // TODO: localize + this.setStatus("Updating " + objectTypePlural + " in " + this.libraryName); + + // Get versions from API for all objects + let allResults = yield this.apiClient.getVersions( + this.libraryType, + this.libraryTypeID, + objectType + ); + + // Get versions from API for objects modified remotely since the last classic sync time + let sinceResults = yield this.apiClient.getVersions( + this.libraryType, + this.libraryTypeID, + objectType, + { + sincetime: lastRemoteSyncTime + } + ); + + // If something else modified the remote library while we were getting updates, + // wait for increasing amounts of time before trying again, and then start from + // the first object type + if (allResults.libraryVersion != sinceResults.libraryVersion + || (lastLibraryVersion && allResults.libraryVersion != lastLibraryVersion)) { + if (!gen) { + gen = Zotero.Utilities.Internal.delayGenerator( + Zotero.Sync.Data.conflictDelayIntervals, 60 * 60 * 1000 + ); + } + Zotero.debug("Library version changed since last check (" + + allResults.libraryVersion + " != " + + sinceResults.libraryVersion + " != " + + lastLibraryVersion + ") -- waiting"); + let keepGoing = yield gen.next(); + if (!keepGoing) { + throw new Error("Could not update " + this.libraryName + " -- library in use"); + } + continue loop; + } + else { + lastLibraryVersion = allResults.libraryVersion; + } + + versionResults[objectType] = allResults; + + // Get versions for remote objects modified remotely before the last classic sync time, + // which is all the objects not modified since that time + for (let key in allResults.versions) { + if (!sinceResults.versions[key]) { + currentVersions[objectType][key] = allResults.versions[key]; + } + } + } + break; + } + + // Update versions on local objects modified remotely before last classic sync time, + // to indicate that they don't need to receive remote updates + yield Zotero.DB.executeTransaction(function* () { + for (let objectType in currentVersions) { + let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); + let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); + + // TODO: localize + this.setStatus("Updating " + objectTypePlural + " in " + this.libraryName); + + // Group objects with the same version together and update in batches + let versionObjects = {}; + for (let key in currentVersions[objectType]) { + let id = objectsClass.getIDFromLibraryAndKey(this.libraryID, key); + // If local object doesn't exist, skip + if (!id) continue; + let version = currentVersions[objectType][key]; + if (!versionObjects[version]) { + versionObjects[version] = []; + } + versionObjects[version].push(id); + } + for (let version in versionObjects) { + yield objectsClass.updateVersion(versionObjects[version], version); + } + } + + // Mark library as requiring full sync + yield Zotero.Libraries.setVersion(this.libraryID, -1); + + // If this is the last classic sync library, delete old timestamps + if (!(yield Zotero.DB.valueQueryAsync("SELECT COUNT(*) FROM libraries WHERE version=0"))) { + yield Zotero.DB.queryAsync( + "DELETE FROM version WHERE schema IN ('lastlocalsync', 'lastremotesync')" + ); + } + }.bind(this)); + + Zotero.debug("Done upgrading " + this.libraryName); + + return versionResults; +}); + + +/** + * Perform a full sync + * + * Get all object versions from the API and compare to the local database. If any objects are + * missing or outdated and not up-to-date in the sync cache, download them. If any local objects + * are marked as synced but aren't available remotely, mark them as unsynced for later uploading. + * + * (Technically this isn't a _full_ sync on its own, because settings aren't downloaded here and + * objects are only flagged for later upload.) + * + * @param {Object[]} [versionResults] - Objects returned from getVersions(), keyed by objectType + * @return {Promise} - Promise for the library version after syncing + */ +Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function* (versionResults) { + Zotero.debug("Performing a full sync of " + this.libraryName); + + var gen; + var lastLibraryVersion; + + loop: + while (true) { + // Get synced settings + lastLibraryVersion = yield this._downloadSettings(); + + // Get other object types + for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(this.libraryID)) { + this._failedCheck(); + + let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); + let ObjectType = objectType[0].toUpperCase() + objectType.substr(1); + + // TODO: localize + this.setStatus("Updating " + objectTypePlural + " in " + this.libraryName); + + // Start processing cached objects while waiting for API + this._processCache(objectType); + + let results = {}; + // Use provided versions + if (versionResults) { + results = versionResults[objectType]; + } + // If not available, get from API + else { + results = yield this.apiClient.getVersions( + this.libraryType, + this.libraryTypeID, + objectType + ); + } + if (lastLibraryVersion) { + // If something else modified the remote library while we were getting updates, + // wait for increasing amounts of time before trying again, and then start from + // the first object type + if (lastLibraryVersion != results.libraryVersion) { + if (!gen) { + gen = Zotero.Utilities.Internal.delayGenerator( + Zotero.Sync.Data.conflictDelayIntervals, 60 * 60 * 1000 + ); + } + let keepGoing = yield gen.next(); + if (!keepGoing) { + throw new Error("Could not update " + this.libraryName + " -- library in use"); + } + continue loop; + } + } + else { + lastLibraryVersion = results.libraryVersion; + } + + let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); + let toDownload = []; + let cacheVersions = yield Zotero.Sync.Data.Local.getLatestCacheObjectVersions( + this.libraryID, objectType + ); + // Queue objects that are out of date or don't exist locally and aren't up-to-date + // in the cache + for (let key in results.versions) { + let version = results.versions[key]; + let obj = yield objectsClass.getByLibraryAndKeyAsync(this.libraryID, key, { + noCache: true + }); + // If object already at latest version, skip + if (obj && obj.version === version) { + continue; + } + let cacheVersion = cacheVersions[key]; + // If cache already has latest version, skip + if (cacheVersion == version) { + continue; + } + if (cacheVersion > version) { + throw new Error("Sync cache had later version than remote for " + + objectType + + this.libraryID + "/" + key + + "(" + cacheVersion + " > " + version + ")"); + } + + if (obj) { + Zotero.debug(Zotero.Utilities.capitalize(objectType) + " " + obj.libraryKey + + " is older than version in sync cache"); + } + else { + Zotero.debug(Zotero.Utilities.capitalize(objectType) + " " + + this.libraryID + "/" + key + " in sync cache not found locally"); + } + + toDownload.push(key); + } + + if (toDownload.length) { + Zotero.debug("Downloading missing/outdated " + objectTypePlural + " in " + this.libraryName); + yield this._downloadObjects(objectType, toDownload); + } + + // Mark synced objects that don't exist remotely as unsynced + let syncedKeys = yield Zotero.Sync.Data.Local.getSynced(this.libraryID, objectType); + let remoteMissing = Zotero.Utilities.arrayDiff(syncedKeys, Object.keys(results.versions)); + if (remoteMissing.length) { + Zotero.debug("Marking remotely missing synced " + objectTypePlural + " as unsynced"); + Zotero.debug(remoteMissing); + + // TODO: Check remote deleted + // If remote delete log doesn't go back far enough, add to recovered items? + // Do we care about local timestamp of last edit? + + let ids = []; + for (let key of remoteMissing) { + let id = objectsClass.getIDFromLibraryAndKey(this.libraryID, key); + if (!id) { + Zotero.logError(ObjectType + " " + this.libraryID + "/" + key + + " not found to mark as unsynced"); + continue; + } + ids.push(id); + } + // Reset version, since old version will no longer match remote + yield objectsClass.updateVersion(ids, 0); + yield objectsClass.updateSynced(ids, false); + } + + // Process newly cached objects + this._processCache(objectType); + } + break; + } + + yield this.syncCachePromise; + + yield Zotero.Libraries.setVersion(this.libraryID, lastLibraryVersion); + + Zotero.debug("Done with full sync for " + this.libraryName); + + return lastLibraryVersion; +}); + + +/** + * Chain sync cache processing for a given object type + * + * On error, check if errors should be fatal and set the .failed flag + * + * @param {String} objectType + */ +Zotero.Sync.Data.Engine.prototype._processCache = function (objectType) { + var self = this; + this.syncCachePromise = this.syncCachePromise.then(function () { + self._failedCheck(); + return Zotero.Sync.Data.Local.processSyncCacheForObjectType( + self.libraryID, objectType, self.options + ) + .catch(function (e) { + Zotero.logError(e); + if (self.stopOnError) { + Zotero.debug("WE FAILED!!!"); + self.failed = e; + } + }); + }) +} + + +Zotero.Sync.Data.Engine.prototype._failedCheck = function () { + if (this.stopOnError && this.failed) { + Zotero.debug("STOPPING ON ERROR 1"); + throw this.failed; + } +}; diff --git a/chrome/content/zotero/xpcom/sync/syncEventListeners.js b/chrome/content/zotero/xpcom/sync/syncEventListeners.js new file mode 100644 index 000000000..1f8a37f3a --- /dev/null +++ b/chrome/content/zotero/xpcom/sync/syncEventListeners.js @@ -0,0 +1,216 @@ +Zotero.Sync.EventListeners = { + /** + * Start all listeners + */ + init: function () { + for (let i in this) { + if (i.indexOf('Listener') != -1) { + if (this[i].init) { + this[i].init(); + } + } + } + } +}; + + +/** + * Notifier observer to add deleted objects to syncDeleteLog/storageDeleteLog + * plus related methods + */ +Zotero.Sync.EventListeners.ChangeListener = new function () { + this.init = function () { + // Initialize delete log listener + // TODO: Support clearing of full-text for an item? + Zotero.Notifier.registerObserver( + this, ['collection', 'item', 'search', 'setting'], 'deleteLog' + ); + } + + this.notify = Zotero.Promise.method(function (event, type, ids, extraData) { + var syncObjectTypeID = Zotero.Sync.Data.Utilities.getSyncObjectTypeID(type); + if (!syncObjectTypeID) { + return; + } + + if (event != 'delete') { + return; + } + + var syncSQL = "REPLACE INTO syncDeleteLog VALUES (?, ?, ?, 0)"; + + if (type == 'item' && Zotero.Sync.Storage.WebDAV.includeUserFiles) { + var storageSQL = "REPLACE INTO storageDeleteLog VALUES (?, ?, 0)"; + } + + return Zotero.DB.executeTransaction(function* () { + for (let i = 0; i < ids.length; i++) { + let id = ids[i]; + + if (extraData[id] && extraData[id].skipDeleteLog) { + continue; + } + + var libraryID, key; + if (type == 'setting') { + [libraryID, key] = ids[i].split("/"); + } + else { + let d = extraData[ids[i]]; + libraryID = d.libraryID; + key = d.key; + } + + if (!key) { + throw new Error("Key not provided in notifier object"); + } + + yield Zotero.DB.queryAsync( + syncSQL, + [ + syncObjectTypeID, + libraryID, + key + ] + ); + if (storageSQL && oldItem.itemType == 'attachment' && + [ + Zotero.Attachments.LINK_MODE_IMPORTED_FILE, + Zotero.Attachments.LINK_MODE_IMPORTED_URL + ].indexOf(oldItem.linkMode) != -1) { + yield Zotero.DB.queryAsync( + storageSQL, + [ + libraryID, + key + ] + ); + } + } + }); + }); +} + + +Zotero.Sync.EventListeners.AutoSyncListener = { + init: function () { + // Initialize save observer + Zotero.Notifier.registerObserver(this); + }, + + notify: function (event, type, ids, extraData) { + // TODO: skip others + if (event == 'refresh' || event == 'redraw') { + return; + } + + if (Zotero.Prefs.get('sync.autoSync') && Zotero.Sync.Server.enabled) { + Zotero.Sync.Runner.setSyncTimeout(false, false, true); + } + } +} + + +Zotero.Sync.EventListeners.IdleListener = { + _idleTimeout: 3600, + _backTimeout: 900, + + init: function () { + // DEBUG: Allow override for testing + var idleTimeout = Zotero.Prefs.get("sync.autoSync.idleTimeout"); + if (idleTimeout) { + this._idleTimeout = idleTimeout; + } + var backTimeout = Zotero.Prefs.get("sync.autoSync.backTimeout"); + if (backTimeout) { + this._backTimeout = backTimeout; + } + + if (Zotero.Prefs.get("sync.autoSync")) { + this.register(); + } + }, + + register: function () { + Zotero.debug("Initializing sync idle observer"); + var idleService = Components.classes["@mozilla.org/widget/idleservice;1"] + .getService(Components.interfaces.nsIIdleService); + idleService.addIdleObserver(this, this._idleTimeout); + idleService.addIdleObserver(this._backObserver, this._backTimeout); + }, + + observe: function (subject, topic, data) { + if (topic != 'idle') { + return; + } + + if (!Zotero.Sync.Server.enabled + || Zotero.Sync.Server.syncInProgress + || Zotero.Sync.Storage.syncInProgress) { + return; + } + + // TODO: move to Runner.sync()? + if (Zotero.locked) { + Zotero.debug('Zotero is locked -- skipping idle sync', 4); + return; + } + + if (Zotero.Sync.Server.manualSyncRequired) { + Zotero.debug('Manual sync required -- skipping idle sync', 4); + return; + } + + Zotero.debug("Beginning idle sync"); + + Zotero.Sync.Runner.sync({ + background: true + }); + Zotero.Sync.Runner.setSyncTimeout(this._idleTimeout, true, true); + }, + + _backObserver: { + observe: function (subject, topic, data) { + if (topic != 'back') { + return; + } + + Zotero.Sync.Runner.clearSyncTimeout(); + if (!Zotero.Sync.Server.enabled + || Zotero.Sync.Server.syncInProgress + || Zotero.Sync.Storage.syncInProgress) { + return; + } + Zotero.debug("Beginning return-from-idle sync"); + Zotero.Sync.Runner.sync({ + background: true + }); + } + }, + + unregister: function () { + Zotero.debug("Stopping sync idle observer"); + var idleService = Components.classes["@mozilla.org/widget/idleservice;1"] + .getService(Components.interfaces.nsIIdleService); + idleService.removeIdleObserver(this, this._idleTimeout); + idleService.removeIdleObserver(this._backObserver, this._backTimeout); + } +} + + + +Zotero.Sync.EventListeners.progressListener = { + onStart: function () { + + }, + + + onProgress: function (current, max) { + + }, + + + onStop: function () { + + } +}; diff --git a/chrome/content/zotero/xpcom/sync/syncLocal.js b/chrome/content/zotero/xpcom/sync/syncLocal.js new file mode 100644 index 000000000..9bdf43320 --- /dev/null +++ b/chrome/content/zotero/xpcom/sync/syncLocal.js @@ -0,0 +1,706 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2014 Center for History and New Media + George Mason University, Fairfax, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +if (!Zotero.Sync.Data) { + Zotero.Sync.Data = {}; +} + +Zotero.Sync.Data.Local = { + _lastSyncTime: null, + _lastClassicSyncTime: null, + + init: Zotero.Promise.coroutine(function* () { + yield this._loadLastSyncTime(); + if (!_lastSyncTime) { + yield this._loadLastClassicSyncTime(); + } + }), + + + getLastSyncTime: function () { + if (_lastSyncTime === null) { + throw new Error("Last sync time not yet loaded"); + } + return _lastSyncTime; + }, + + + /** + * @return {Promise} + */ + updateLastSyncTime: function () { + _lastSyncTime = new Date(); + return Zotero.DB.queryAsync( + "REPLACE INTO version (schema, version) VALUES ('lastsync', ?)", + Math.round(_lastSyncTime.getTime() / 1000) + ); + }, + + + _loadLastSyncTime: Zotero.Promise.coroutine(function* () { + var sql = "SELECT version FROM version WHERE schema='lastsync'"; + var lastsync = yield Zotero.DB.valueQueryAsync(sql); + _lastSyncTime = (lastsync ? new Date(lastsync * 1000) : false); + }), + + + /** + * @param {Integer} libraryID + * @return {Promise} - A promise for an array of object keys + */ + getSynced: function (libraryID, objectType) { + var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); + var sql = "SELECT key FROM " + objectsClass.table + " WHERE libraryID=? AND synced=1"; + return Zotero.DB.columnQueryAsync(sql, [libraryID]); + }, + + + /** + * @param {Integer} libraryID + * @return {Promise} - A promise for an array of object ids + */ + getUnsynced: Zotero.Promise.coroutine(function* (libraryID, objectType) { + var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); + var sql = "SELECT " + objectsClass.idColumn + " FROM " + objectsClass.table + + " WHERE libraryID=? AND synced=0"; + + // RETRIEVE PARENT DOWN? EVEN POSSIBLE? + // items via parent + // collections via getDescendents? + + return Zotero.DB.columnQueryAsync(sql, [libraryID]); + }), + + + // + // Cache management + // + /** + * Gets the latest version for each object of a given type in the given library + * + * @return {Promise} - A promise for an object with object keys as keys and versions + * as properties + */ + getLatestCacheObjectVersions: Zotero.Promise.coroutine(function* (libraryID, objectType) { + var sql = "SELECT key, version FROM syncCache WHERE libraryID=? AND " + + "syncObjectTypeID IN (SELECT syncObjectTypeID FROM " + + "syncObjectTypes WHERE name=?) ORDER BY version"; + var rows = yield Zotero.DB.queryAsync(sql, [libraryID, objectType]); + var versions = {}; + for (let i = 0; i < rows.length; i++) { + let row = rows[i]; + versions[row.key] = row.version; + } + return versions; + }), + + + /** + * @return {Promise} - A promise for an array of object versions + */ + getCacheObjectVersions: function (objectType, libraryID, key) { + var sql = "SELECT version FROM syncCache WHERE libraryID=? AND key=? " + + "AND syncObjectTypeID IN (SELECT syncObjectTypeID FROM " + + "syncObjectTypes WHERE name=?) ORDER BY version"; + return Zotero.DB.columnQueryAsync(sql, [libraryID, key, objectType]); + }, + + + /** + * @return {Promise} - A promise for an object version + */ + getLatestCacheObjectVersion: function (objectType, libraryID, key) { + var sql = "SELECT version FROM syncCache WHERE libraryID=? AND key=? " + + "AND syncObjectTypeID IN (SELECT syncObjectTypeID FROM " + + "syncObjectTypes WHERE name=?) ORDER BY VERSION DESC LIMIT 1"; + return Zotero.DB.valueQueryAsync(sql, [libraryID, key, objectType]); + }, + + + /** + * @return {Promise} + */ + getCacheObject: Zotero.Promise.coroutine(function* (objectType, libraryID, key, version) { + var sql = "SELECT data FROM syncCache WHERE libraryID=? AND key=? AND version=? " + + "AND syncObjectTypeID IN (SELECT syncObjectTypeID FROM " + + "syncObjectTypes WHERE name=?)"; + var data = yield Zotero.DB.valueQueryAsync(sql, [libraryID, key, version, objectType]); + if (data) { + return JSON.parse(data); + } + return false; + }), + + + saveCacheObjects: Zotero.Promise.coroutine(function* (objectType, libraryID, jsonArray) { + if (!Array.isArray(jsonArray)) { + throw new Error("'json' must be an array"); + } + + Zotero.debug("Saving to sync cache:"); + Zotero.debug(jsonArray); + + var syncObjectTypeID = Zotero.Sync.Data.Utilities.getSyncObjectTypeID(objectType); + var sql = "INSERT OR REPLACE INTO syncCache " + + "(libraryID, key, syncObjectTypeID, version, data) VALUES "; + var chunkSize = Math.floor(Zotero.DB.MAX_BOUND_PARAMETERS / 5); + return Zotero.DB.executeTransaction(function* () { + return Zotero.Utilities.Internal.forEachChunkAsync( + jsonArray, + chunkSize, + Zotero.Promise.coroutine(function* (chunk) { + var params = []; + for (let i = 0; i < chunk.length; i++) { + let o = chunk[i]; + if (o.key === undefined) { + throw new Error("Missing 'key' property in JSON"); + } + if (o.version === undefined) { + throw new Error("Missing 'version' property in JSON"); + } + params.push(libraryID, o.key, syncObjectTypeID, o.version, JSON.stringify(o)); + } + return Zotero.DB.queryAsync( + sql + chunk.map(() => "(?, ?, ?, ?, ?)").join(", "), params + ); + }) + ); + }.bind(this)); + }), + + + processSyncCache: Zotero.Promise.coroutine(function* (libraryID, options) { + for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(libraryID)) { + yield this.processSyncCacheForObjectType(libraryID, objectType, options); + } + }), + + + processSyncCacheForObjectType: Zotero.Promise.coroutine(function* (libraryID, objectType, options) { + options = options || {}; + var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); + var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); + var ObjectType = Zotero.Utilities.capitalize(objectType); + var libraryName = Zotero.Libraries.getName(libraryID); + + Zotero.debug("Processing " + objectTypePlural + " in sync cache for " + libraryName); + + var numSaved = 0; + var numSkipped = 0; + + var data = yield this._getUnwrittenData(libraryID, objectType); + + if (!data.length) { + Zotero.debug("No unwritten " + objectTypePlural + " in sync cache"); + return; + } + + Zotero.debug("Processing " + data.length + " " + + (data.length == 1 ? objectType : objectTypePlural) + + " in sync cache"); + + if (options.setStatus) { + options.setStatus("Processing " + objectTypePlural); // TODO: localize + } + + // Sort parent objects first, to avoid retries due to unmet dependencies + if (objectType == 'item' || objectType == 'collection') { + let parentProp = 'parent' + objectType[0].toUpperCase() + objectType.substr(1); + data.sort(function (a, b) { + if (a[parentProp] && !b[parentProp]) return 1; + if (b[parentProp] && !a[parentProp]) return -1; + return 0; + }); + } + + var concurrentObjects = 5; + yield Zotero.Utilities.Internal.forEachChunkAsync( + data, + concurrentObjects, + function (chunk) { + return Zotero.DB.executeTransaction(function* () { + for (let i = 0; i < chunk.length; i++) { + let json = chunk[i]; + let jsonData = json.data; + let isNewObject; + let objectKey = json.key; + + Zotero.debug(json); + + if (!jsonData) { + Zotero.logError(new Error("Missing 'data' object in JSON in sync cache for " + + objectType + " " + libraryID + "/" + objectKey)); + continue; + } + + // Skip objects with unmet dependencies + if (objectType == 'item' || objectType == 'collection') { + let parentProp = 'parent' + objectType[0].toUpperCase() + objectType.substr(1); + let parentKey = jsonData[parentProp]; + if (parentKey) { + let parentObj = yield objectsClass.getByLibraryAndKeyAsync( + libraryID, parentKey, { noCache: true } + ); + if (!parentObj) { + Zotero.debug("Parent of " + objectType + " " + + libraryID + "/" + jsonData.key + " not found -- skipping"); + // TEMP: Add parent to a queue, in case it's somehow missing + // after retrieving all objects? + numSkipped++; + continue; + } + } + + /*if (objectType == 'item') { + for (let j = 0; j < jsonData.collections.length; i++) { + let parentKey = jsonData.collections[j]; + let parentCollection = Zotero.Collections.getByLibraryAndKey( + libraryID, parentKey, { noCache: true } + ); + if (!parentCollection) { + // ??? + } + } + }*/ + } + + let obj = yield objectsClass.getByLibraryAndKeyAsync( + libraryID, objectKey, { noCache: true } + ); + if (obj) { + Zotero.debug("Matching local " + objectType + " exists", 4); + isNewObject = false; + + // Local object has not been modified since last sync + if (obj.synced) { + // Overwrite local below + } + else { + Zotero.debug("Local " + objectType + " " + obj.libraryKey + + " has been modified since last sync", 4); + + let cachedJSON = yield this.getCacheObject( + objectType, obj.libraryID, obj.key, obj.version + ); + Zotero.debug("GOT CACHED"); + Zotero.debug(cachedJSON); + + let jsonDataLocal = yield obj.toJSON(); + + let result = this._reconcileChanges( + objectType, + cachedJSON.data, + jsonDataLocal, + jsonData, + ['dateAdded', 'dateModified'] + ); + + // If no changes, update local version and keep as unsynced + if (!result.changes.length && !result.conflicts.length) { + Zotero.debug("No remote changes to apply to local " + objectType + + " " + obj.libraryKey); + yield obj.updateVersion(json.version); + continue; + } + + // Ignore conflicts from Quick Start Guide, and just use remote version + /*if (objectType == 'item' + && jsonDataLocal.key == "ABCD2345" + && jsonDataLocal.url.indexOf('quick_start_guide') != -1 + && jsonData.url.indexOf('quick_start_guide') != -1) { + Zotero.debug("Ignoring conflict for item '" + jsonData.title + "' " + + "-- using remote version"); + let saved = yield this._saveObjectFromJSON(obj, jsonData, options); + if (saved) numSaved++; + continue; + }*/ + + // If no conflicts, apply remote changes automatically + if (!result.conflicts.length) { + Zotero.DataObjectUtilities.applyChanges( + jsonData, result.changes + ); + let saved = yield this._saveObjectFromJSON(obj, jsonData, options); + if (saved) numSaved++; + continue; + } + + Zotero.debug('======DIFF========'); + Zotero.debug(cachedJSON); + Zotero.debug(jsonDataLocal); + Zotero.debug(jsonData); + Zotero.debug(result); + throw new Error("Conflict"); + + + // TODO + + // reconcile changes automatically if we can + + // if we can't: + // if it's a search or collection, use most recent version + // if it's an item, + } + + let saved = yield this._saveObjectFromJSON(obj, jsonData, options); + if (saved) numSaved++; + } + // Object doesn't exist locally + else { + isNewObject = true; + + // Check if object has been deleted locally + if (yield this._objectInDeleteLog(objectType, libraryID, objectKey)) { + switch (objectType) { + case 'item': + throw new Error("Unimplemented"); + break; + + // Auto-restore some locally deleted objects that have changed remotely + case 'collection': + case 'search': + yield this._removeObjectFromDeleteLog( + objectType, + libraryID, + objectKey + ); + + throw new Error("Unimplemented"); + break; + + default: + throw new Error("Unknown object type '" + objectType + "'"); + } + } + + // Create new object + obj = new Zotero[ObjectType]; + obj.libraryID = libraryID; + obj.key = objectKey; + yield obj.loadPrimaryData(); + + let saved = yield this._saveObjectFromJSON(obj, jsonData, options, { + // Don't cache new items immediately, which skips reloading after save + skipCache: true + }); + if (saved) numSaved++; + } + } + }.bind(this)); + }.bind(this) + ); + + // Keep retrying if we skipped any, as long as we're still making progress + if (numSkipped && numSaved != 0) { + Zotero.debug("More " + objectTypePlural + " in cache -- continuing"); + yield this.processSyncCacheForObjectType(libraryID, objectType, options); + } + + data = yield this._getUnwrittenData(libraryID, objectType); + Zotero.debug("Skipping " + data.length + " " + + (data.length == 1 ? objectType : objectTypePlural) + + " in sync cache"); + return data; + }), + + + // + // Classic sync + // + getLastClassicSyncTime: function () { + if (_lastClassicSyncTime === null) { + throw new Error("Last classic sync time not yet loaded"); + } + return _lastClassicSyncTime; + }, + + _loadLastClassicSyncTime: Zotero.Promise.coroutine(function* () { + var sql = "SELECT version FROM version WHERE schema='lastlocalsync'"; + var lastsync = yield Zotero.DB.valueQueryAsync(sql); + _lastClassicSyncTime = (lastsync ? new Date(lastsync * 1000) : false); + }), + + _saveObjectFromJSON: Zotero.Promise.coroutine(function* (obj, json, options) { + try { + yield obj.fromJSON(json); + obj.version = json.version; + obj.synced = true; + Zotero.debug("SAVING " + json.key + " WITH SYNCED"); + Zotero.debug(obj.version); + yield obj.save({ + skipDateModifiedUpdate: true, + skipSelect: true, + errorHandler: function (e) { + // Don't log expected errors + if (e.name == 'ZoteroUnknownTypeError' + && e.name == 'ZoteroUnknownFieldError' + && e.name == 'ZoteroMissingObjectError') { + return; + } + Zotero.debug(e, 1); + } + }); + } + catch (e) { + if (e.name == 'ZoteroUnknownTypeError' + || e.name == 'ZoteroUnknownFieldError' + || e.name == 'ZoteroMissingObjectError') { + let desc = e.name + .replace(/^Zotero/, "") + // Convert "MissingObjectError" to "missing object error" + .split(/([a-z]+)/).join(' ').trim() + .replace(/([A-Z]) ([a-z]+)/g, "$1$2").toLowerCase(); + Zotero.logError("Ignoring " + desc + " for " + + obj.objectType + " " + obj.libraryKey, 2); + } + else if (options.stopOnError) { + throw e; + } + else { + Zotero.logError(e); + options.onError(e); + } + return false; + } + return true; + }), + + + /** + * Calculate a changeset to apply locally to resolve an object conflict, plus a list of + * conflicts where not possible + */ + _reconcileChanges: function (objectType, originalJSON, currentJSON, newJSON, ignoreFields) { + if (!originalJSON) { + return this._reconcileChangesWithoutCache(objectType, currentJSON, newJSON, ignoreFields); + } + + var changeset1 = Zotero.DataObjectUtilities.diff(originalJSON, currentJSON, ignoreFields); + var changeset2 = Zotero.DataObjectUtilities.diff(originalJSON, newJSON, ignoreFields); + + var conflicts = []; + + for (let i = 0; i < changeset1.length; i++) { + for (let j = 0; j < changeset2.length; j++) { + let c1 = changeset1[i]; + let c2 = changeset2[j]; + if (c1.field != c2.field) { + continue; + } + + // Disregard member additions/deletions for different values + if (c1.op.startsWith('member-') && c2.op.startsWith('member-')) { + switch (c1.field) { + case 'collections': + if (c1.value !== c2.value) { + continue; + } + break; + + case 'creators': + if (!Zotero.Creators.equals(c1.value, c2.value)) { + continue; + } + break; + + case 'tags': + if (!Zotero.Tags.equals(c1.value, c2.value)) { + // If just a type difference, treat as modify with type 0 if + // not type 0 in changeset1 + if (c1.op == 'member-add' && c2.op == 'member-add' + && c1.value.tag === c2.value.tag) { + changeset1.splice(i--, 1); + changeset2.splice(j--, 1); + if (c1.value.type > 0) { + changeset2.push({ + field: "tags", + op: "member-remove", + value: c1.value + }); + changeset2.push({ + field: "tags", + op: "member-add", + value: c2.value + }); + } + } + continue; + } + break; + } + } + + // Disregard member additions/deletions for different properties and values + if (c1.op.startsWith('property-member-') && c2.op.startsWith('property-member-')) { + if (c1.value.key !== c2.value.key || c1.value.value !== c2.value.value) { + continue; + } + } + + // Changes are equal or in conflict + + // Removed on both sides + if (c1.op == 'delete' && c2.op == 'delete') { + changeset2.splice(j--, 1); + continue; + } + + // Added or removed members on both sides + if ((c1.op == 'member-add' && c2.op == 'member-add') + || (c1.op == 'member-remove' && c2.op == 'member-remove') + || (c1.op == 'property-member-add' && c2.op == 'property-member-add') + || (c1.op == 'property-member-remove' && c2.op == 'property-member-remove')) { + changeset2.splice(j--, 1); + continue; + } + + // If both sides have values, see if they're the same, and if so remove the + // second one + if (c1.op != 'delete' && c2.op != 'delete' && c1.value === c2.value) { + changeset2.splice(j--, 1); + continue; + } + + // Automatically apply remote changes for non-items, even if in conflict + if (objectType != 'item') { + continue; + } + + // Conflict + changeset2.splice(j--, 1); + conflicts.push([c1, c2]); + } + } + + return { + changes: changeset2, + conflicts: conflicts + }; + }, + + + /** + * Calculate a changeset to apply locally to resolve an object conflict in absence of a + * cached version. Members and property members (e.g., collections, tags, relations) + * are combined, so any removals will be automatically undone. Field changes result in + * conflicts. + */ + _reconcileChangesWithoutCache: function (objectType, currentJSON, newJSON, ignoreFields) { + var changeset = Zotero.DataObjectUtilities.diff(currentJSON, newJSON, ignoreFields); + + var changes = []; + var conflicts = []; + + for (let i = 0; i < changeset.length; i++) { + let c = changeset[i]; + + // Member changes are additive only, so ignore removals + if (c.op.endsWith('-remove')) { + continue; + } + + // Record member changes + if (c.op.startsWith('member-') || c.op.startsWith('property-member-')) { + changes.push(c); + continue; + } + + // Automatically apply remote changes for non-items, even if in conflict + if (objectType != 'item') { + changes.push(c); + continue; + } + + // Field changes are conflicts + conflicts.push(c); + } + + return { changes, conflicts }; + }, + + + /** + * @return {Promise} A promise for an array of JSON objects + */ + _getUnwrittenData: function (libraryID, objectType, max) { + var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); + // The MAX(version) ensures we get the data from the most recent version of the object, + // thanks to SQLite 3.7.11 (http://www.sqlite.org/releaselog/3_7_11.html) + var sql = "SELECT data, MAX(SC.version) AS version FROM syncCache SC " + + "LEFT JOIN " + objectsClass.table + " O " + + "USING (libraryID, key) " + + "WHERE SC.libraryID=? AND " + + "syncObjectTypeID IN (SELECT syncObjectTypeID FROM " + + "syncObjectTypes WHERE name='" + objectType + "') " + // If saved version doesn't have a version or is less than the cache version + + "AND IFNULL(O.version, 0) < SC.version " + + "GROUP BY SC.libraryID, SC.key"; + return Zotero.DB.queryAsync(sql, [libraryID]).map(row => JSON.parse(row.data)); + }, + + + markObjectAsSynced: Zotero.Promise.method(function (obj) { + obj.synced = true; + return obj.saveTx({ + skipSyncedUpdate: true, + skipDateModifiedUpdate: true, + skipClientDateModifiedUpdate: true, + skipNotifier: true + }); + }), + + + markObjectAsUnsynced: Zotero.Promise.method(function (obj) { + obj.synced = false; + return obj.saveTx({ + skipSyncedUpdate: true, + skipDateModifiedUpdate: true, + skipClientDateModifiedUpdate: true, + skipNotifier: true + }); + }), + + + /** + * @return {Promise} + */ + _objectInDeleteLog: Zotero.Promise.coroutine(function* (objectType, libraryID, key) { + var syncObjectTypeID = Zotero.Sync.Data.Utilities.getSyncObjectTypeID(objectType); + var sql = "SELECT COUNT(*) FROM syncDeleteLog WHERE libraryID=? AND key=? " + + "AND syncObjectTypeID=?"; + var count = yield Zotero.DB.valueQueryAsync(sql, [libraryID, key, syncObjectTypeID]); + return !!count; + }), + + + /** + * @return {Promise} + */ + _removeObjectFromDeleteLog: function (objectType, libraryID, key) { + var syncObjectTypeID = Zotero.Sync.Data.Utilities.getSyncObjectTypeID(objectType); + var sql = "DELETE FROM syncDeleteLog WHERE libraryID=? AND key=? AND syncObjectTypeID=?"; + return Zotero.DB.queryAsync(sql, [libraryID, key, syncObjectTypeID]); + } +} diff --git a/chrome/content/zotero/xpcom/sync/syncRunner.js b/chrome/content/zotero/xpcom/sync/syncRunner.js new file mode 100644 index 000000000..290e25ea6 --- /dev/null +++ b/chrome/content/zotero/xpcom/sync/syncRunner.js @@ -0,0 +1,1056 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2009 Center for History and New Media + George Mason University, Fairfax, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +"use strict"; + +if (!Zotero.Sync) { + Zotero.Sync = {}; +} + +Zotero.Sync.Runner_Module = function () { + Zotero.defineProperty(this, 'background', { get: () => _background }); + Zotero.defineProperty(this, 'lastSyncStatus', { get: () => _lastSyncStatus }); + + const stopOnError = true; + + var _autoSyncTimer; + var _background; + var _firstInSession = true; + var _syncInProgress = false; + + var _lastSyncStatus; + var _currentSyncStatusLabel; + var _currentLastSyncLabel; + var _errors = []; + + + /** + * Begin a sync session + * + * @param {Object} [options] + * @param {String} [apiKey] + * @param {Boolean} [background=false] - Whether this is a background request, which prevents + * some alerts from being shown + * @param {String} [baseURL] + * @param {Integer[]} [libraries] - IDs of libraries to sync + * @param {Function} [onError] - Function to pass errors to instead of handling internally + * (used for testing) + */ + this.sync = Zotero.Promise.coroutine(function* (options = {}) { + // Clear message list + _errors = []; + + if (Zotero.HTTP.browserIsOffline()){ + this.clearSyncTimeout(); // DEBUG: necessary? + var msg = Zotero.getString('general.browserIsOffline', Zotero.appName); + var e = new Zotero.Error(msg, 0, { dialogButtonText: null }) + Components.utils.reportError(e); + Zotero.debug(e, 1); + this.updateIcons(e); + return false; + } + + // Shouldn't be possible + if (_syncInProgress) { + let msg = Zotero.getString('sync.error.syncInProgress'); + let e = new Zotero.Error(msg, 0, { dialogButtonText: null, frontWindowOnly: true }); + this.updateIcons(e); + return false; + } + _syncInProgress = true; + + // Purge deleted objects so they don't cause sync errors (e.g., long tags) + yield Zotero.purgeDataObjects(true); + + options.apiKey = options.apiKey || Zotero.Prefs.get('devAPIKey'); + if (!options.apiKey) { + let msg = "API key not provided"; + let e = new Zotero.Error(msg, 0, { dialogButtonText: null }) + this.updateIcons(e); + return false; + } + options.baseURL = options.baseURL || ZOTERO_CONFIG.API_URL; + if (_firstInSession) { + options.firstInSession = true; + _firstInSession = false; + } + + _background = !!options.background; + _syncInProgress = true; + this.updateIcons('animate'); + + try { + Components.utils.import("resource://zotero/concurrent-caller.js"); + var caller = new ConcurrentCaller(4); // TEMP: one for now + caller.setLogger(msg => Zotero.debug(msg)); + caller.stopOnError = stopOnError; + caller.onError = function (e) { + this.addError(e); + if (e.fatal) { + caller.stop(); + throw e; + } + }.bind(this); + + // TODO: Use a single client for all operations? + var client = new Zotero.Sync.APIClient({ + baseURL: options.baseURL, + apiVersion: ZOTERO_CONFIG.API_VERSION, + apiKey: options.apiKey, + concurrentCaller: caller, + background: options.background + }); + + var keyInfo = yield this.checkAccess(client, options); + if (!keyInfo) { + this.stop(); + Zotero.debug("Syncing cancelled"); + return false; + } + + var libraries = yield this.checkLibraries(client, options, keyInfo, libraries); + + for (let libraryID of libraries) { + try { + let engine = new Zotero.Sync.Data.Engine({ + libraryID: libraryID, + apiClient: client, + setStatus: this.setSyncStatus.bind(this), + stopOnError: stopOnError, + onError: this.addError.bind(this) + }); + yield engine.start(); + } + catch (e) { + Zotero.debug("Sync failed for library " + libraryID); + Zotero.debug(e, 1); + Components.utils.reportError(e); + this.checkError(e); + if (options.onError) { + options.onError(e); + } + else { + this.addError(e); + } + if (stopOnError || e.fatal) { + caller.stop(); + break; + } + } + } + + yield Zotero.Sync.Data.Local.updateLastSyncTime(); + } + catch (e) { + if (options.onError) { + options.onError(e); + } + else { + this.addError(e); + } + } + + this.stop(); + + Zotero.debug("Done syncing"); + + return; + + var storageSync = function () { + Zotero.Sync.Runner.setSyncStatus(Zotero.getString('sync.status.syncingFiles')); + + Zotero.Sync.Storage.sync(options) + .then(function (results) { + Zotero.debug("File sync is finished"); + + if (results.errors.length) { + Zotero.debug(results.errors, 1); + for each(var e in results.errors) { + Components.utils.reportError(e); + } + Zotero.Sync.Runner.setErrors(results.errors); + return; + } + + if (results.changesMade) { + Zotero.debug("Changes made during file sync " + + "-- performing additional data sync"); + Zotero.Sync.Server.sync(finalCallbacks); + } + else { + Zotero.Sync.Runner.stop(); + } + }) + .catch(function (e) { + Zotero.debug("File sync failed", 1); + Zotero.Sync.Runner.error(e); + }) + .done(); + }; + + Zotero.Sync.Server.sync({ + // Sync 1 success + onSuccess: storageSync, + + // Sync 1 skip + onSkip: storageSync, + + // Sync 1 stop + onStop: function () { + Zotero.Sync.Runner.stop(); + }, + + // Sync 1 error + onError: function (e) { + Zotero.Sync.Runner.error(e); + } + }); + }); + + + /** + * Check key for current user info and return access info + */ + this.checkAccess = Zotero.Promise.coroutine(function* (client, options) { + var json = yield client.getKeyInfo(); + Zotero.debug(json); + if (!json) { + // TODO: Nicer error message + throw new Error("Invalid API key"); + } + + // Sanity check + if (!json.userID) throw new Error("userID not found in response"); + if (!json.username) throw new Error("username not found in response"); + + // Make sure user hasn't changed, and prompt to update database if so + if (!(yield this.checkUser(json.userID, json.username))) { + return false; + } + + return json; + }); + + + this.checkLibraries = Zotero.Promise.coroutine(function* (client, options, keyInfo, libraries = []) { + var access = keyInfo.access; + +/* var libraries = [ + Zotero.Libraries.userLibraryID, + Zotero.Libraries.publicationsLibraryID, + // Groups sorted by name + ...(Zotero.Groups.getAll().map(x => x.libraryID)) + ]; +*/ + + var syncAllLibraries = !libraries.length; + + // TODO: Ability to remove or disable editing of user library? + + if (syncAllLibraries) { + if (access.user && access.user.library) { + libraries.push( + Zotero.Libraries.userLibraryID, Zotero.Libraries.publicationsLibraryID + ); + } + } + else { + // Check access to specified libraries + for (let libraryID of libraries) { + let type = Zotero.Libraries.getType(libraryID); + if (type == 'user' || type == 'publications') { + if (!access.user || !access.user.library) { + // TODO: Alert + throw new Error("Key does not have access to library " + libraryID); + } + } + } + } + + // + // Check group access + // + let remotelyMissingGroups = []; + let groupsToDownload = []; + + if (!Zotero.Utilities.isEmpty(access.groups)) { + // TEMP: Require all-group access for now + if (access.groups.all) { + + } + else { + throw new Error("Full group access is currently required"); + } + + let remoteGroupVersions = yield client.getGroupVersions(keyInfo.userID); + let remoteGroupIDs = Object.keys(remoteGroupVersions).map(id => parseInt(id)); + Zotero.debug(remoteGroupVersions); + + for (let id in remoteGroupVersions) { + id = parseInt(id); + let group = Zotero.Groups.get(id); + + if (syncAllLibraries) { + // If syncing all libraries, mark any that don't exist or are outdated + // locally for update. Group is added to the library list after downloading + if (!group || group.version < remoteGroupVersions[id]) { + groupsToDownload.push(id); + } + // If not outdated, just add to library list + else { + libraries.push(group.libraryID); + } + } + else { + // If specific libraries were provided, ignore remote groups that don't + // exist locally or aren't in the given list + if (!group || libraries.indexOf(group.libraryID) == -1) { + continue; + } + // If group metadata is outdated, mark for update + if (group.version < remoteGroupVersions[id]) { + groupsToDownload.push(id); + } + } + } + + // Get local groups (all if syncing all libraries or just selected ones) that don't + // exist remotely + // TODO: Use explicit removals? + remotelyMissingGroups = Zotero.Utilities.arrayDiff( + syncAllLibraries + ? Zotero.Groups.getAll().map(g => g.id) + : libraries.filter(id => Zotero.Libraries.getType(id) == 'group') + .map(id => Zotero.Groups.getGroupIDFromLibraryID(id)), + remoteGroupIDs + ).map(id => Zotero.Groups.get(id)); + } + // No group access + else { + remotelyMissingGroups = Zotero.Groups.getAll(); + } + + if (remotelyMissingGroups.length) { + // TODO: What about explicit deletions? + + let removedGroups = []; + + let ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"] + .getService(Components.interfaces.nsIPromptService); + let buttonFlags = (ps.BUTTON_POS_0) * (ps.BUTTON_TITLE_IS_STRING) + + (ps.BUTTON_POS_1) * (ps.BUTTON_TITLE_IS_STRING) + + (ps.BUTTON_POS_2) * (ps.BUTTON_TITLE_IS_STRING) + + ps.BUTTON_DELAY_ENABLE; + + // Prompt for each group + // + // TODO: Localize + for (let group of remotelyMissingGroups) { + let msg; + // If all-groups access but group is missing, user left it + if (access.groups && access.groups.all) { + msg = "You are no longer a member of the group \u2018" + group.name + "\u2019."; + } + // If not all-groups access, key might just not have access + else { + msg = "You no longer have access to the group \u2018" + group.name + "\u2019."; + } + + msg += "\n\n" + "Would you like to remove it from this computer or keep it " + + "as a read-only library?"; + + let index = ps.confirmEx( + null, + "Group Not Found", + msg, + buttonFlags, + "Remove Group", + // TODO: Any way to have Esc trigger extra1 instead so it doesn't + // have to be in this order? + "Cancel Sync", + "Keep Group", + null, {} + ); + + if (index == 0) { + removedGroups.push(group); + } + else if (index == 1) { + Zotero.debug("Cancelling sync"); + return []; + } + else if (index == 2) { + // TODO: Mark groups to be ignored + } + } + + let removedLibraryIDs = []; + for (let group of removedGroups) { + removedLibraryIDs.push(group.libraryID); + yield Zotero.DB.executeTransaction(function* () { + return group.erase(); + }); + } + libraries = Zotero.Utilities.arrayDiff(libraries, removedLibraryIDs); + } + + // Update metadata and permissions on missing or outdated groups + for (let groupID of groupsToDownload) { + let info = yield client.getGroupInfo(groupID); + if (!info) { + throw new Error("Group " + groupID + " not found"); + } + let group = Zotero.Groups.get(groupID); + if (!group) { + group = new Zotero.Group; + group.id = groupID; + } + group.version = info.version; + group.fromJSON(info.data, Zotero.Users.getCurrentUserID()); + yield group.save(); + + // Add group to library list + libraries.push(group.libraryID); + } + + return [...new Set(libraries)]; + }); + + + /** + * Make sure we're syncing with the same account we used last time, and prompt if not. + * If user accepts, change the current user, delete existing groups, and update relation + * URIs to point to the new user's library. + * + * @param {Integer} userID New userID + * @param {Integer} libraryID New libraryID + * @param {Integer} noServerData The server account is empty — this is + * the account after a server clear + * @return {Boolean} - True to continue, false to cancel + */ + this.checkUser = Zotero.Promise.coroutine(function* (userID, username) { + var lastUserID = Zotero.Users.getCurrentUserID(); + var lastUsername = Zotero.Users.getCurrentUsername(); + + // TEMP: Remove? No way to determine this quickly currently. + var noServerData = false; + + if (lastUserID && lastUserID != userID) { + var groups = Zotero.Groups.getAll(); + + var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"] + .getService(Components.interfaces.nsIPromptService); + var buttonFlags = (ps.BUTTON_POS_0) * (ps.BUTTON_TITLE_IS_STRING) + + (ps.BUTTON_POS_1) * (ps.BUTTON_TITLE_CANCEL) + + (ps.BUTTON_POS_2) * (ps.BUTTON_TITLE_IS_STRING) + + ps.BUTTON_POS_1_DEFAULT + + ps.BUTTON_DELAY_ENABLE; + + var msg = Zotero.getString('sync.lastSyncWithDifferentAccount', [lastUsername, username]); + + if (!noServerData) { + msg += " " + Zotero.getString('sync.localDataWillBeCombined', username); + // If there are local groups belonging to the previous user, + // we need to remove them + if (groups.length) { + msg += " " + Zotero.getString('sync.localGroupsWillBeRemoved1'); + } + msg += "\n\n" + Zotero.getString('sync.avoidCombiningData', lastUsername); + var syncButtonText = Zotero.getString('sync.sync'); + } + else if (groups.length) { + msg += " " + Zotero.getString('sync.localGroupsWillBeRemoved2', [username, lastUsername]); + var syncButtonText = Zotero.getString('sync.removeGroupsAndSync'); + } + // If there are no local groups and the server is empty, + // don't bother prompting + else { + var noPrompt = true; + } + + if (!noPrompt) { + var index = ps.confirmEx( + null, + Zotero.getString('general.warning'), + msg, + buttonFlags, + syncButtonText, + null, + Zotero.getString('sync.openSyncPreferences'), + null, {} + ); + + if (index > 0) { + if (index == 2) { + var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] + .getService(Components.interfaces.nsIWindowMediator); + var lastWin = wm.getMostRecentWindow("navigator:browser"); + lastWin.ZoteroPane.openPreferences('zotero-prefpane-sync'); + } + return false; + } + } + } + + yield Zotero.DB.executeTransaction(function* () { + if (lastUserID != userID) { + yield Zotero.Users.setCurrentUserID(userID); + + if (lastUserID) { + // Delete all local groups if changing users + for (let group of groups) { + yield group.erase(); + } + + // Update relations pointing to the old library to point to this one + yield Zotero.Relations.updateUser(lastUserID, userID); + } + // Replace local user key with libraryID, in case duplicates were + // merged before the first sync + else { + let repl = "local/" + Zotero.Users.getLocalUserKey(); + yield Zotero.Relations.updateUser(repl, userID); + } + } + + if (lastUsername != username) { + yield Zotero.Users.setCurrentUsername(username); + } + }) + + return true; + }); + + + this.stop = function () { + this.updateIcons(_errors); + _errors = []; + _syncInProgress = false; + } + + + /** + * Log a warning, but don't throw an error + */ + this.warning = function (e) { + Zotero.debug(e, 2); + Components.utils.reportError(e); + e.errorType = 'warning'; + _warning = e; + } + + + this.error = function (e) { + if (typeof e == 'string') { + e = new Error(e); + e.errorType = 'error'; + } + Zotero.debug(e, 1); + this.updateIcons(e); + throw (e); + } + + + /** + * @param {Integer} [timeout=15] Timeout in seconds + * @param {Boolean} [recurring=false] + * @param {Boolean} [background] Triggered sync is a background sync + */ + this.setSyncTimeout = function (timeout, recurring, background) { + // check if server/auto-sync are enabled? + + if (!timeout) { + var timeout = 15; + } + + if (_autoSyncTimer) { + Zotero.debug("Cancelling auto-sync timer"); + _autoSyncTimer.cancel(); + } + else { + _autoSyncTimer = Components.classes["@mozilla.org/timer;1"]. + createInstance(Components.interfaces.nsITimer); + } + + // Implements nsITimerCallback + var callback = { + notify: function (timer) { + if (!Zotero.Sync.Server.enabled) { + return; + } + + if (Zotero.locked) { + Zotero.debug('Zotero is locked -- skipping auto-sync', 4); + return; + } + + if (Zotero.Sync.Storage.syncInProgress) { + Zotero.debug('Storage sync already in progress -- skipping auto-sync', 4); + return; + } + + if (Zotero.Sync.Server.syncInProgress) { + Zotero.debug('Sync already in progress -- skipping auto-sync', 4); + return; + } + + if (Zotero.Sync.Server.manualSyncRequired) { + Zotero.debug('Manual sync required -- skipping auto-sync', 4); + return; + } + + this.sync({ + background: background + }); + } + } + + if (recurring) { + Zotero.debug('Setting auto-sync interval to ' + timeout + ' seconds'); + _autoSyncTimer.initWithCallback( + callback, timeout * 1000, Components.interfaces.nsITimer.TYPE_REPEATING_SLACK + ); + } + else { + if (Zotero.Sync.Storage.syncInProgress) { + Zotero.debug('Storage sync in progress -- not setting auto-sync timeout', 4); + return; + } + + if (Zotero.Sync.Server.syncInProgress) { + Zotero.debug('Sync in progress -- not setting auto-sync timeout', 4); + return; + } + + Zotero.debug('Setting auto-sync timeout to ' + timeout + ' seconds'); + _autoSyncTimer.initWithCallback( + callback, timeout * 1000, Components.interfaces.nsITimer.TYPE_ONE_SHOT + ); + } + } + + + this.clearSyncTimeout = function () { + if (_autoSyncTimer) { + _autoSyncTimer.cancel(); + } + } + + + /** + * Trigger updating of the main sync icon, the sync error icon, and + * library-specific sync error icons across all windows + */ + this.addError = function (e, libraryID) { + if (e.added) return; + e.added = true; + if (libraryID) { + e.libraryID = libraryID; + } + Zotero.logError(e); + _errors.push(this.parseError(e)); + } + + + this.getErrorsByLibrary = function (libraryID) { + return _errors.filter(e => e.libraryID === libraryID); + } + + + /** + * Get most severe error type from an array of parsed errors + */ + this.getPrimaryErrorType = function (errors) { + // Set highest priority error as the primary (sync error icon) + var errorTypes = { + info: 1, + warning: 2, + error: 3, + upgrade: 4, + + // Skip these + animate: -1 + }; + var state = false; + for (let i = 0; i < errors.length; i++) { + let e = errors[i]; + + let errorType = e.errorType; + + if (e.fatal) { + return 'error'; + } + + if (!errorType || errorTypes[errorType] < 0) { + continue; + } + if (!state || errorTypes[errorType] > errorTypes[state]) { + state = errorType; + } + } + return state; + } + + + this.checkError = function (e, background) { + if (e.name && e.name == 'Zotero Error') { + switch (e.error) { + case Zotero.Error.ERROR_SYNC_USERNAME_NOT_SET: + case Zotero.Error.ERROR_INVALID_SYNC_LOGIN: + // TODO: the setTimeout() call below should just simulate a click on the sync error icon + // instead of creating its own dialog, but updateIcons() doesn't yet provide full control + // over dialog title and primary button text/action, which is why this version of the + // dialog is a bit uglier than the manual click version + // TODO: localize (=>done) and combine with below (=>?) + var msg = Zotero.getString('sync.error.invalidLogin.text'); + e.message = msg; + e.data = {}; + e.data.dialogText = msg; + e.data.dialogButtonText = Zotero.getString('sync.openSyncPreferences'); + e.data.dialogButtonCallback = function () { + var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] + .getService(Components.interfaces.nsIWindowMediator); + var win = wm.getMostRecentWindow("navigator:browser"); + win.ZoteroPane.openPreferences("zotero-prefpane-sync"); + }; + + // Manual click + if (!background) { + setTimeout(function () { + var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] + .getService(Components.interfaces.nsIWindowMediator); + var win = wm.getMostRecentWindow("navigator:browser"); + + var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"] + .getService(Components.interfaces.nsIPromptService); + var buttonFlags = (ps.BUTTON_POS_0) * (ps.BUTTON_TITLE_IS_STRING) + + (ps.BUTTON_POS_1) * (ps.BUTTON_TITLE_CANCEL); + if (e.error == Zotero.Error.ERROR_SYNC_USERNAME_NOT_SET) { + var title = Zotero.getString('sync.error.usernameNotSet'); + var msg = Zotero.getString('sync.error.usernameNotSet.text'); + } + else { + var title = Zotero.getString('sync.error.invalidLogin'); + var msg = Zotero.getString('sync.error.invalidLogin.text'); + } + var index = ps.confirmEx( + win, + title, + msg, + buttonFlags, + Zotero.getString('sync.openSyncPreferences'), + null, null, null, {} + ); + + if (index == 0) { + win.ZoteroPane.openPreferences("zotero-prefpane-sync"); + return; + } + }, 1); + } + break; + } + } + + // TEMP + return; + + if (extraInfo) { + // Server errors will generally be HTML + extraInfo = Zotero.Utilities.unescapeHTML(extraInfo); + Components.utils.reportError(extraInfo); + } + + Zotero.debug(e, 1); + + if (!skipReload) { + Zotero.reloadDataObjects(); + } + Zotero.Sync.EventListener.resetIgnored(); + } + + + /** + * Set the sync icon and sync error icon across all windows + * + * @param {Error|Error[]|'animate'} errors - An error, an array of errors, or 'animate' to + * spin the icon. An empty array will reset the + * icons. + */ + this.updateIcons = function (errors) { + if (typeof errors == 'string') { + var state = errors; + errors = []; + } + else { + if (!Array.isArray(errors)) { + errors = [errors]; + } + var state = this.getPrimaryErrorType(errors); + } + + // Refresh source list + //yield Zotero.Notifier.trigger('redraw', 'collection', []); + + if (errors.length == 1 && errors[0].frontWindowOnly) { + // Fake an nsISimpleEnumerator with just the topmost window + var enumerator = { + _returned: false, + hasMoreElements: function () { + return !this._returned; + }, + getNext: function () { + if (this._returned) { + throw ("No more windows to return in Zotero.Sync.Runner.updateIcons()"); + } + this._returned = true; + var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] + .getService(Components.interfaces.nsIWindowMediator); + return wm.getMostRecentWindow("navigator:browser"); + } + }; + } + // Update all windows + else { + var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] + .getService(Components.interfaces.nsIWindowMediator); + var enumerator = wm.getEnumerator('navigator:browser'); + } + + while (enumerator.hasMoreElements()) { + var win = enumerator.getNext(); + if (!win.ZoteroPane) continue; + var doc = win.ZoteroPane.document; + + // Update sync error icon + var icon = doc.getElementById('zotero-tb-sync-error'); + Zotero.debug(icon + ""); + this.updateErrorIcon(icon, state, errors); + + // Update sync icon + var syncIcon = doc.getElementById('zotero-tb-sync'); + if (state == 'animate') { + syncIcon.setAttribute('status', state); + // Disable button while spinning + syncIcon.disabled = true; + } + else { + syncIcon.removeAttribute('status'); + syncIcon.disabled = false; + } + } + + // Clear status + this.setSyncStatus(); + } + + + /** + * Set the sync icon tooltip message + */ + this.setSyncStatus = function (msg) { + _lastSyncStatus = msg; + + // If a label is registered, update it + if (_currentSyncStatusLabel) { + _updateSyncStatusLabel(); + } + } + + + this.parseError = function (e) { + if (!e) { + return { parsed: true }; + } + + // Already parsed + if (e.parsed) { + return e; + } + + e.parsed = true; + e.errorType = e.errorType ? e.errorType : 'error'; + if (!e.data) { + e.data = {}; + } + + return e; + } + + + /** + * Set the state of the sync error icon and add an onclick to populate + * the error panel + */ + this.updateErrorIcon = function (icon, state, errors) { + if (!errors || !errors.length) { + icon.hidden = true; + icon.onclick = null; + return; + } + + icon.hidden = false; + icon.setAttribute('state', state); + var self = this; + icon.onclick = function () { + var panel = self.updateErrorPanel(this.ownerDocument, errors); + panel.openPopup(this, "after_end", 16, 0, false, false); + }; + } + + + this.updateErrorPanel = function (doc, errors) { + var panel = doc.getElementById('zotero-sync-error-panel'); + + // Clear existing panel content + while (panel.hasChildNodes()) { + panel.removeChild(panel.firstChild); + } + + for (let e of errors) { + var box = doc.createElement('vbox'); + var label = doc.createElement('label'); + if (e.libraryID !== undefined) { + label.className = "zotero-sync-error-panel-library-name"; + if (e.libraryID == 0) { + var libraryName = Zotero.getString('pane.collections.library'); + } + else { + let group = Zotero.Groups.getByLibraryID(e.libraryID); + var libraryName = group.name; + } + label.setAttribute('value', libraryName); + } + var content = doc.createElement('hbox'); + var buttons = doc.createElement('hbox'); + buttons.pack = 'end'; + box.appendChild(label); + box.appendChild(content); + box.appendChild(buttons); + + // Show our own error mesages directly + if (e instanceof Zotero.Error) { + var msg = e.message; + } + // For unexpected ones, just show a generic message + else { + // TODO: improve and localize + var msg = "An error occurred during syncing:\n\n" + e; + } + + var desc = doc.createElement('description'); + desc.textContent = msg; + // Make the text selectable + desc.setAttribute('style', '-moz-user-select: text; cursor: text'); + content.appendChild(desc); + + /*// If not an error and there's no explicit button text, don't show + // button to report errors + if (e.errorType != 'error' && e.data.dialogButtonText === undefined) { + e.data.dialogButtonText = null; + }*/ + + if (e.data && e.data.dialogButtonText !== null) { + if (e.data.dialogButtonText === undefined) { + var buttonText = Zotero.getString('errorReport.reportError'); + var buttonCallback = function () { + doc.defaultView.ZoteroPane.reportErrors(); + }; + } + else { + var buttonText = e.data.dialogButtonText; + var buttonCallback = e.data.dialogButtonCallback; + } + + var button = doc.createElement('button'); + button.setAttribute('label', buttonText); + button.onclick = buttonCallback; + buttons.appendChild(button); + } + + panel.appendChild(box) + break; + } + + return panel; + } + + + /** + * Register label in sync icon tooltip to receive updates + * + * If no label passed, unregister current label + * + * @param {Tooltip} [label] + */ + this.registerSyncStatusLabel = function (statusLabel, lastSyncLabel) { + _currentSyncStatusLabel = statusLabel; + _currentLastSyncLabel = lastSyncLabel; + if (_currentSyncStatusLabel) { + _updateSyncStatusLabel(); + } + } + + + function _updateSyncStatusLabel() { + if (_lastSyncStatus) { + _currentSyncStatusLabel.value = _lastSyncStatus; + _currentSyncStatusLabel.hidden = false; + } + else { + _currentSyncStatusLabel.hidden = true; + } + + // Always update last sync time + var lastSyncTime = Zotero.Sync.Data.Local.getLastSyncTime(); + if (!lastSyncTime) { + try { + lastSyncTime = Zotero.Sync.Data.Local.getLastClassicSyncTime() + } + catch (e) { + Zotero.debug(e, 2); + Components.utils.reportError(e); + _currentLastSyncLabel.hidden = true; + return; + } + } + if (lastSyncTime) { + var msg = Zotero.Date.toRelativeDate(lastSyncTime); + } + // Don't show "Not yet synced" if a sync is in progress + else if (_syncInProgress) { + _currentLastSyncLabel.hidden = true; + return; + } + else { + var msg = Zotero.getString('sync.status.notYetSynced'); + } + + _currentLastSyncLabel.value = Zotero.getString('sync.status.lastSync') + " " + msg; + _currentLastSyncLabel.hidden = false; + } +} + +Zotero.Sync.Runner = new Zotero.Sync.Runner_Module; diff --git a/chrome/content/zotero/xpcom/sync/syncUtilities.js b/chrome/content/zotero/xpcom/sync/syncUtilities.js new file mode 100644 index 000000000..d99a727e9 --- /dev/null +++ b/chrome/content/zotero/xpcom/sync/syncUtilities.js @@ -0,0 +1,49 @@ +/* + ***** BEGIN LICENSE BLOCK ***** + + Copyright © 2014 Center for History and New Media + George Mason University, Fairfax, Virginia, USA + http://zotero.org + + This file is part of Zotero. + + Zotero is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + Zotero is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with Zotero. If not, see . + + ***** END LICENSE BLOCK ***** +*/ + +if (!Zotero.Sync.Data) { + Zotero.Sync.Data = {}; +} + +Zotero.Sync.Data.Utilities = { + _syncObjectTypeIDs: {}, + + init: Zotero.Promise.coroutine(function* () { + // If not found, cache all + var sql = "SELECT name, syncObjectTypeID AS id FROM syncObjectTypes"; + var rows = yield Zotero.DB.queryAsync(sql); + for (let i = 0; i < rows.length; i++) { + row = rows[i]; + this._syncObjectTypeIDs[row.name] = row.id; + } + }), + + getSyncObjectTypeID: function (objectType) { + if (!this._syncObjectTypeIDs[objectType]) { + return false; + } + return this._syncObjectTypeIDs[objectType]; + }, +}; diff --git a/chrome/content/zotero/xpcom/syncedSettings.js b/chrome/content/zotero/xpcom/syncedSettings.js index 3a3e076f9..9ba3d0104 100644 --- a/chrome/content/zotero/xpcom/syncedSettings.js +++ b/chrome/content/zotero/xpcom/syncedSettings.js @@ -43,7 +43,24 @@ Zotero.SyncedSettings = (function () { return JSON.parse(json); }), - set: Zotero.Promise.coroutine(function* (libraryID, setting, value, version, synced) { + /** + * Used by sync and tests + * + * @return {Object} - Object with 'synced' and 'version' properties + */ + getMetadata: Zotero.Promise.coroutine(function* (libraryID, setting) { + var sql = "SELECT * FROM syncedSettings WHERE setting=? AND libraryID=?"; + var row = yield Zotero.DB.rowQueryAsync(sql, [setting, libraryID]); + if (!row) { + return false; + } + return { + synced: !!row.synced, + version: row.version + }; + }), + + set: Zotero.Promise.coroutine(function* (libraryID, setting, value, version = 0, synced) { if (typeof value == undefined) { throw new Error("Value not provided"); } @@ -87,13 +104,18 @@ Zotero.SyncedSettings = (function () { synced = synced ? 1 : 0; if (hasCurrentValue) { - var sql = "UPDATE syncedSettings SET value=?, synced=? WHERE setting=? AND libraryID=?"; - yield Zotero.DB.queryAsync(sql, [JSON.stringify(value), synced, setting, libraryID]); + var sql = "UPDATE syncedSettings SET value=?, version=?, synced=? " + + "WHERE setting=? AND libraryID=?"; + yield Zotero.DB.queryAsync( + sql, [JSON.stringify(value), version, synced, setting, libraryID] + ); } else { var sql = "INSERT INTO syncedSettings " - + "(setting, libraryID, value, synced) VALUES (?, ?, ?, ?)"; - yield Zotero.DB.queryAsync(sql, [setting, libraryID, JSON.stringify(value), synced]); + + "(setting, libraryID, value, version, synced) VALUES (?, ?, ?, ?, ?)"; + yield Zotero.DB.queryAsync( + sql, [setting, libraryID, JSON.stringify(value), version, synced] + ); } yield Zotero.Notifier.trigger(event, 'setting', [id], extraData); return true; diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index b3f835b80..b9e6220b6 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -604,8 +604,9 @@ Components.utils.import("resource://gre/modules/osfile.jsm"); Zotero.Notifier.registerObserver(Zotero.Tags, 'setting'); - Zotero.Sync.init(); - Zotero.Sync.Runner.init(); + yield Zotero.Sync.Data.Local.init(); + yield Zotero.Sync.Data.Utilities.init(); + Zotero.Sync.EventListeners.init(); Zotero.MIMETypeHandler.init(); yield Zotero.Proxies.init(); diff --git a/chrome/content/zotero/zoteroPane.js b/chrome/content/zotero/zoteroPane.js index b7a360802..4c846b612 100644 --- a/chrome/content/zotero/zoteroPane.js +++ b/chrome/content/zotero/zoteroPane.js @@ -183,7 +183,7 @@ var ZoteroPane = new function() if (index == 0) { Zotero.Sync.Server.sync({ onSuccess: function () { - Zotero.Sync.Runner.setSyncIcon(); + Zotero.Sync.Runner.updateIcons(); ps.alert( null, @@ -436,7 +436,7 @@ var ZoteroPane = new function() // // We don't bother setting an error state at open if (Zotero.Sync.Server.syncInProgress || Zotero.Sync.Storage.syncInProgress) { - Zotero.Sync.Runner.setSyncIcon('animate'); + Zotero.Sync.Runner.updateIcons('animate'); } return true; @@ -3969,7 +3969,7 @@ var ZoteroPane = new function() + ps.BUTTON_POS_1 * ps.BUTTON_TITLE_IS_STRING; // Warning - if (e.errorMode == 'warning') { + if (e.errorType == 'warning') { var title = Zotero.getString('general.warning'); // If secondary button not specified, just use an alert @@ -3996,7 +3996,7 @@ var ZoteroPane = new function() } } // Error - else if (e.errorMode == 'error') { + else if (e.errorType == 'error') { var title = Zotero.getString('general.error'); // If secondary button is explicitly null, just use an alert @@ -4031,7 +4031,7 @@ var ZoteroPane = new function() } } // Upgrade - else if (e.errorMode == 'upgrade') { + else if (e.errorType == 'upgrade') { ps.alert(null, "", e.message); } }; diff --git a/chrome/locale/en-US/zotero/zotero.properties b/chrome/locale/en-US/zotero/zotero.properties index 41580c56f..9c8c1ce08 100644 --- a/chrome/locale/en-US/zotero/zotero.properties +++ b/chrome/locale/en-US/zotero/zotero.properties @@ -822,7 +822,7 @@ sync.error.invalidCharsFilename = The filename '%S' contains invalid characters sync.lastSyncWithDifferentAccount = This Zotero database was last synced with a different zotero.org account ('%1$S') from the current one ('%2$S'). sync.localDataWillBeCombined = If you continue, local Zotero data will be combined with data from the '%S' account stored on the server. -sync.localGroupsWillBeRemoved1 = Local groups, including any with changed items, will also be removed. +sync.localGroupsWillBeRemoved1 = Local groups, including any with changed items, will also be removed from this computer. sync.avoidCombiningData = To avoid combining or losing data, revert to the '%S' account or use the Reset options in the Sync pane of the Zotero preferences. sync.localGroupsWillBeRemoved2 = If you continue, local groups, including any with changed items, will be removed and replaced with groups linked to the '%1$S' account.\n\nTo avoid losing local changes to groups, be sure you have synced with the '%2$S' account before syncing with the '%1$S' account. diff --git a/components/zotero-service.js b/components/zotero-service.js index c45b00894..3c7573e1d 100644 --- a/components/zotero-service.js +++ b/components/zotero-service.js @@ -95,6 +95,12 @@ const xpcomFilesLocal = [ 'server', 'style', 'sync', + 'sync/syncAPIClient', + 'sync/syncEngine', + 'sync/syncEventListeners', + 'sync/syncLocal', + 'sync/syncRunner', + 'sync/syncUtilities', 'storage', 'storage/streamListener', 'storage/queueManager', diff --git a/resource/config.js b/resource/config.js index e31d4a1b7..5cd95fe59 100644 --- a/resource/config.js +++ b/resource/config.js @@ -9,7 +9,7 @@ var ZOTERO_CONFIG = { PROXY_AUTH_URL: 'https://s3.amazonaws.com/zotero.org/proxy-auth', SYNC_URL: 'https://sync.zotero.org/', API_URL: 'https://api.zotero.org/', - API_VERSION: 2, + API_VERSION: 3, PREF_BRANCH: 'extensions.zotero.', BOOKMARKLET_ORIGIN: 'https://www.zotero.org', HTTP_BOOKMARKLET_ORIGIN: 'http://www.zotero.org', diff --git a/test/content/support.js b/test/content/support.js index 61a1b917f..4401b55ac 100644 --- a/test/content/support.js +++ b/test/content/support.js @@ -627,3 +627,40 @@ function importFileAttachment(filename) { filename.split('/').forEach((part) => testfile.append(part)); return Zotero.Attachments.importFromFile({file: testfile}); } + + +/** + * Sets the fake XHR server to response to a given response + * + * @param {Object} server - Sinon FakeXMLHttpRequest server + * @param {Object|String} response - Dot-separated path to predefined response in responses + * object (e.g., keyInfo.fullAccess) or a JSON object + * that defines the response + * @param {Object} responses - Predefined responses + */ +function setHTTPResponse(server, baseURL, response, responses) { + if (typeof response == 'string') { + let [topic, key] = response.split('.'); + if (!responses[topic]) { + throw new Error("Invalid topic"); + } + if (!responses[topic][key]) { + throw new Error("Invalid response key"); + } + response = responses[topic][key]; + } + + var responseArray = [response.status || 200, {}, ""]; + if (response.json) { + responseArray[1]["Content-Type"] = "application/json"; + responseArray[2] = JSON.stringify(response.json); + } + else { + responseArray[1]["Content-Type"] = "text/plain"; + responseArray[2] = response.text || ""; + } + for (let i in response.headers) { + responseArray[1][i] = response.headers[i]; + } + server.respondWith(response.method, baseURL + response.url, responseArray); +} diff --git a/test/tests/syncEngineTest.js b/test/tests/syncEngineTest.js new file mode 100644 index 000000000..1a954f566 --- /dev/null +++ b/test/tests/syncEngineTest.js @@ -0,0 +1,754 @@ +"use strict"; + +describe("Zotero.Sync.Data.Engine", function () { + var apiKey = Zotero.Utilities.randomString(24); + var baseURL = "http://local.zotero/"; + var engine, server, client, caller, stub, spy; + + var responses = {}; + + var setup = Zotero.Promise.coroutine(function* (options) { + options = options || {}; + + server = sinon.fakeServer.create(); + server.autoRespond = true; + + Components.utils.import("resource://zotero/concurrent-caller.js"); + var caller = new ConcurrentCaller(1); + caller.setLogger(msg => Zotero.debug(msg)); + caller.stopOnError = true; + caller.onError = function (e) { + Zotero.logError(e); + if (options.onError) { + options.onError(e); + } + if (e.fatal) { + caller.stop(); + throw e; + } + }; + + var client = new Zotero.Sync.APIClient({ + baseURL: baseURL, + apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION, + apiKey: apiKey, + concurrentCaller: caller, + background: options.background || true + }); + + var engine = new Zotero.Sync.Data.Engine({ + apiClient: client, + libraryID: options.libraryID || Zotero.Libraries.userLibraryID + }); + + return { engine, client, caller }; + }); + + function setResponse(response) { + setHTTPResponse(server, baseURL, response, responses); + } + + function makeCollectionJSON(options) { + return { + key: options.key, + version: options.version, + data: { + key: options.key, + version: options.version, + name: options.name + } + }; + } + + function makeSearchJSON(options) { + return { + key: options.key, + version: options.version, + data: { + key: options.key, + version: options.version, + name: options.name, + conditions: options.conditions ? options.conditions : [ + { + condition: 'title', + operator: 'contains', + value: 'test' + } + ] + } + }; + } + + function makeItemJSON(options) { + var json = { + key: options.key, + version: options.version, + data: { + key: options.key, + version: options.version, + itemType: options.itemType || 'book', + title: options.title || options.name + } + }; + Object.assign(json.data, options); + delete json.data.name; + return json; + } + + // Allow functions to be called programmatically + var makeJSONFunctions = { + collection: makeCollectionJSON, + search: makeSearchJSON, + item: makeItemJSON + }; + + // + // Tests + // + beforeEach(function* () { + this.timeout(60000); + yield resetDB({ + skipBundledFiles: true + }); + + Zotero.HTTP.mock = sinon.FakeXMLHttpRequest; + + yield Zotero.Users.setCurrentUserID(1); + yield Zotero.Users.setCurrentUsername("testuser"); + }) + after(function* () { + this.timeout(60000); + yield resetDB(); + }) + + describe("Syncing", function () { + it("should perform a sync for a new library", function* () { + ({ engine, client, caller } = yield setup()); + + server.respond(function (req) { + if (req.method == "POST" && req.url == baseURL + "users/1/items") { + let ifUnmodifiedSince = req.requestHeaders["If-Unmodified-Since-Version"]; + if (ifUnmodifiedSince == 0) { + req.respond(412, {}, "Library has been modified since specified version"); + return; + } + + if (ifUnmodifiedSince == 3) { + let json = JSON.parse(req.requestBody); + req.respond( + 200, + { + "Content-Type": "application/json", + "Last-Modified-Version": 3 + }, + JSON.stringify({ + success: { + "0": json[0].key, + "1": json[1].key + }, + unchanged: {}, + failed: {} + }) + ); + return; + } + } + }) + + var headers = { + "Last-Modified-Version": 3 + }; + setResponse({ + method: "GET", + url: "users/1/settings", + status: 200, + headers: headers, + json: { + tagColors: { + value: [ + { + name: "A", + color: "#CC66CC" + } + ], + version: 2 + } + } + }); + setResponse({ + method: "GET", + url: "users/1/collections?format=versions", + status: 200, + headers: headers, + json: { + "AAAAAAAA": 1 + } + }); + setResponse({ + method: "GET", + url: "users/1/searches?format=versions", + status: 200, + headers: headers, + json: { + "AAAAAAAA": 2 + } + }); + setResponse({ + method: "GET", + url: "users/1/items?format=versions&includeTrashed=1", + status: 200, + headers: headers, + json: { + "AAAAAAAA": 3 + } + }); + setResponse({ + method: "GET", + url: "users/1/collections?format=json&collectionKey=AAAAAAAA", + status: 200, + headers: headers, + json: [ + makeCollectionJSON({ + key: "AAAAAAAA", + version: 1, + name: "A" + }) + ] + }); + setResponse({ + method: "GET", + url: "users/1/searches?format=json&searchKey=AAAAAAAA", + status: 200, + headers: headers, + json: [ + makeSearchJSON({ + key: "AAAAAAAA", + version: 2, + name: "A" + }) + ] + }); + setResponse({ + method: "GET", + url: "users/1/items?format=json&itemKey=AAAAAAAA&includeTrashed=1", + status: 200, + headers: headers, + json: [ + makeItemJSON({ + key: "AAAAAAAA", + version: 3, + itemType: "book", + title: "A" + }) + ] + }); + setResponse({ + method: "GET", + url: "users/1/deleted?since=0", + status: 200, + headers: headers, + json: {} + }); + yield engine.start(); + + var userLibraryID = Zotero.Libraries.userLibraryID; + + // Check local library version + assert.equal(Zotero.Libraries.getVersion(userLibraryID), 3); + + // Make sure local objects exist + var setting = yield Zotero.SyncedSettings.get(userLibraryID, "tagColors"); + assert.lengthOf(setting, 1); + assert.equal(setting[0].name, 'A'); + var settingMetadata = yield Zotero.SyncedSettings.getMetadata(userLibraryID, "tagColors"); + assert.equal(settingMetadata.version, 2); + assert.isTrue(settingMetadata.synced); + + var obj = yield Zotero.Collections.getByLibraryAndKeyAsync(userLibraryID, "AAAAAAAA"); + assert.equal(obj.name, 'A'); + assert.equal(obj.version, 1); + assert.isTrue(obj.synced); + + obj = yield Zotero.Searches.getByLibraryAndKeyAsync(userLibraryID, "AAAAAAAA"); + assert.equal(obj.name, 'A'); + assert.equal(obj.version, 2); + assert.isTrue(obj.synced); + + obj = yield Zotero.Items.getByLibraryAndKeyAsync(userLibraryID, "AAAAAAAA"); + assert.equal(obj.getField('title'), 'A'); + assert.equal(obj.version, 3); + assert.isTrue(obj.synced); + }) + + it("should make only one request if in sync", function* () { + yield Zotero.Libraries.setVersion(Zotero.Libraries.userLibraryID, 5); + ({ engine, client, caller } = yield setup()); + + server.respond(function (req) { + if (req.method == "GET" && req.url == baseURL + "users/1/settings?since=5") { + let since = req.requestHeaders["If-Modified-Since-Version"]; + if (since == 5) { + req.respond(304); + return; + } + } + }); + yield engine.start(); + }) + }) + + describe("#_startDownload()", function () { + it("shouldn't redownload objects already in the cache", function* () { + var userLibraryID = Zotero.Libraries.userLibraryID; + //yield Zotero.Libraries.setVersion(userLibraryID, 5); + ({ engine, client, caller } = yield setup()); + + var objects = {}; + for (let type of Zotero.DataObjectUtilities.getTypes()) { + let obj = objects[type] = createUnsavedDataObject(type); + obj.version = 5; + obj.synced = true; + yield obj.saveTx({ skipSyncedUpdate: true }); + + yield Zotero.Sync.Data.Local.saveCacheObjects( + type, + userLibraryID, + [ + { + key: obj.key, + version: obj.version, + data: (yield obj.toJSON()) + } + ] + ); + } + + var json; + var headers = { + "Last-Modified-Version": 5 + }; + setResponse({ + method: "GET", + url: "users/1/settings", + status: 200, + headers: headers, + json: {} + }); + json = {}; + json[objects.collection.key] = 5; + setResponse({ + method: "GET", + url: "users/1/collections?format=versions", + status: 200, + headers: headers, + json: json + }); + json = {}; + json[objects.search.key] = 5; + setResponse({ + method: "GET", + url: "users/1/searches?format=versions", + status: 200, + headers: headers, + json: json + }); + json = {}; + json[objects.item.key] = 5; + setResponse({ + method: "GET", + url: "users/1/items?format=versions&includeTrashed=1", + status: 200, + headers: headers, + json: json + }); + setResponse({ + method: "GET", + url: "users/1/deleted?since=0", + status: 200, + headers: headers, + json: {} + }); + + yield engine._startDownload(); + }) + + it("should apply remote deletions", function* () { + var userLibraryID = Zotero.Libraries.userLibraryID; + yield Zotero.Libraries.setVersion(userLibraryID, 5); + ({ engine, client, caller } = yield setup()); + + // Create objects and mark them as synced + yield Zotero.SyncedSettings.set( + userLibraryID, 'tagColors', [{name: 'A', color: '#CC66CC'}], 1, true + ); + var collection = createUnsavedDataObject('collection'); + collection.synced = true; + var collectionID = yield collection.saveTx({ skipSyncedUpdate: true }); + var collectionKey = collection.key; + var search = createUnsavedDataObject('search'); + search.synced = true; + var searchID = yield search.saveTx({ skipSyncedUpdate: true }); + var searchKey = search.key; + var item = createUnsavedDataObject('item'); + item.synced = true; + var itemID = yield item.saveTx({ skipSyncedUpdate: true }); + var itemKey = item.key; + + var headers = { + "Last-Modified-Version": 6 + }; + setResponse({ + method: "GET", + url: "users/1/settings?since=5", + status: 200, + headers: headers, + json: {} + }); + setResponse({ + method: "GET", + url: "users/1/collections?format=versions&since=5", + status: 200, + headers: headers, + json: {} + }); + setResponse({ + method: "GET", + url: "users/1/searches?format=versions&since=5", + status: 200, + headers: headers, + json: {} + }); + setResponse({ + method: "GET", + url: "users/1/items?format=versions&since=5&includeTrashed=1", + status: 200, + headers: headers, + json: {} + }); + setResponse({ + method: "GET", + url: "users/1/deleted?since=5", + status: 200, + headers: headers, + json: { + settings: ['tagColors'], + collections: [collection.key], + searches: [search.key], + items: [item.key] + } + }); + yield engine._startDownload(); + + // Make sure objects were deleted + assert.isFalse(yield Zotero.SyncedSettings.get(userLibraryID, 'tagColors')); + assert.isFalse(Zotero.Collections.exists(collectionID)); + assert.isFalse(Zotero.Searches.exists(searchID)); + assert.isFalse(Zotero.Items.exists(itemID)); + + // Make sure objects weren't added to sync delete log + assert.isFalse(yield Zotero.Sync.Data.Local._objectInDeleteLog( + 'setting', userLibraryID, 'tagColors' + )); + assert.isFalse(yield Zotero.Sync.Data.Local._objectInDeleteLog( + 'collection', userLibraryID, collectionKey + )); + assert.isFalse(yield Zotero.Sync.Data.Local._objectInDeleteLog( + 'search', userLibraryID, searchKey + )); + assert.isFalse(yield Zotero.Sync.Data.Local._objectInDeleteLog( + 'item', userLibraryID, itemKey + )); + }) + + it("should ignore remote deletions for non-item objects if local objects changed", function* () { + var userLibraryID = Zotero.Libraries.userLibraryID; + yield Zotero.Libraries.setVersion(userLibraryID, 5); + ({ engine, client, caller } = yield setup()); + + // Create objects marked as unsynced + yield Zotero.SyncedSettings.set( + userLibraryID, 'tagColors', [{name: 'A', color: '#CC66CC'}] + ); + var collection = createUnsavedDataObject('collection'); + var collectionID = yield collection.saveTx(); + var collectionKey = collection.key; + var search = createUnsavedDataObject('search'); + var searchID = yield search.saveTx(); + var searchKey = search.key; + + var headers = { + "Last-Modified-Version": 6 + }; + setResponse({ + method: "GET", + url: "users/1/settings?since=5", + status: 200, + headers: headers, + json: {} + }); + setResponse({ + method: "GET", + url: "users/1/collections?format=versions&since=5", + status: 200, + headers: headers, + json: {} + }); + setResponse({ + method: "GET", + url: "users/1/searches?format=versions&since=5", + status: 200, + headers: headers, + json: {} + }); + setResponse({ + method: "GET", + url: "users/1/items?format=versions&since=5&includeTrashed=1", + status: 200, + headers: headers, + json: {} + }); + setResponse({ + method: "GET", + url: "users/1/deleted?since=5", + status: 200, + headers: headers, + json: { + settings: ['tagColors'], + collections: [collection.key], + searches: [search.key], + items: [] + } + }); + yield engine._startDownload(); + + // Make sure objects weren't deleted + assert.ok(yield Zotero.SyncedSettings.get(userLibraryID, 'tagColors')); + assert.ok(Zotero.Collections.exists(collectionID)); + assert.ok(Zotero.Searches.exists(searchID)); + }) + }) + + describe("#_upgradeCheck()", function () { + it("should upgrade a library last synced with the classic sync architecture", function* () { + var userLibraryID = Zotero.Libraries.userLibraryID; + ({ engine, client, caller } = yield setup()); + + yield Zotero.Items.erase([1, 2], { skipDeleteLog: true }); + var types = Zotero.DataObjectUtilities.getTypes(); + var objects = {}; + + // Create objects added before the last classic sync time, + // which should end up marked as synced + for (let type of types) { + objects[type] = [yield createDataObject(type)]; + } + + var time1 = "2015-05-01 01:23:45"; + yield Zotero.DB.queryAsync("UPDATE collections SET clientDateModified=?", time1); + yield Zotero.DB.queryAsync("UPDATE savedSearches SET clientDateModified=?", time1); + yield Zotero.DB.queryAsync("UPDATE items SET clientDateModified=?", time1); + + // Create objects added after the last sync time, which should be ignored and + // therefore end up marked as unsynced + for (let type of types) { + objects[type].push(yield createDataObject(type)); + } + + var objectJSON = {}; + for (let type of types) { + objectJSON[type] = []; + } + + // Create JSON for objects created remotely after the last sync time, + // which should be ignored + objectJSON.collection.push(makeCollectionJSON({ + key: Zotero.DataObjectUtilities.generateKey(), + version: 20, + name: Zotero.Utilities.randomString() + })); + objectJSON.search.push(makeSearchJSON({ + key: Zotero.DataObjectUtilities.generateKey(), + version: 20, + name: Zotero.Utilities.randomString() + })); + objectJSON.item.push(makeItemJSON({ + key: Zotero.DataObjectUtilities.generateKey(), + version: 20, + itemType: "book", + title: Zotero.Utilities.randomString() + })); + + var lastSyncTime = Zotero.Date.toUnixTimestamp( + Zotero.Date.sqlToDate("2015-05-02 00:00:00", true) + ); + yield Zotero.DB.queryAsync( + "INSERT INTO version VALUES ('lastlocalsync', ?1), ('lastremotesync', ?1)", + lastSyncTime + ); + + var headers = { + "Last-Modified-Version": 20 + } + for (let type of types) { + var suffix = type == 'item' ? '&includeTrashed=1' : ''; + + var json = {}; + json[objects[type][0].key] = 10; + json[objectJSON[type][0].key] = objectJSON[type][0].version; + setResponse({ + method: "GET", + url: "users/1/" + Zotero.DataObjectUtilities.getObjectTypePlural(type) + + "?format=versions" + suffix, + status: 200, + headers: headers, + json: json + }); + json = {}; + json[objectJSON[type][0].key] = objectJSON[type][0].version; + setResponse({ + method: "GET", + url: "users/1/" + Zotero.DataObjectUtilities.getObjectTypePlural(type) + + "?format=versions&sincetime=" + lastSyncTime + suffix, + status: 200, + headers: headers, + json: json + }); + } + var versionResults = yield engine._upgradeCheck(); + + // Objects 1 should be marked as synced, with versions from the server + // Objects 2 should be marked as unsynced + for (let type of types) { + var synced = yield Zotero.Sync.Data.Local.getSynced(userLibraryID, type); + assert.deepEqual(synced, [objects[type][0].key]); + assert.equal(objects[type][0].version, 10); + var unsynced = yield Zotero.Sync.Data.Local.getUnsynced(userLibraryID, type); + assert.deepEqual(unsynced, [objects[type][1].id]); + + assert.equal(versionResults[type].libraryVersion, headers["Last-Modified-Version"]); + assert.property(versionResults[type].versions, objectJSON[type][0].key); + } + + assert.equal(Zotero.Libraries.getVersion(userLibraryID), -1); + }) + }) + + describe("#_fullSync()", function () { + it("should download missing/updated local objects and flag remotely missing local objects for upload", function* () { + var userLibraryID = Zotero.Libraries.userLibraryID; + ({ engine, client, caller } = yield setup()); + + yield Zotero.Items.erase([1, 2], { skipDeleteLog: true }); + var types = Zotero.DataObjectUtilities.getTypes(); + var objects = {}; + var objectJSON = {}; + for (let type of types) { + objectJSON[type] = []; + } + + for (let type of types) { + // Create objects with outdated versions, which should be updated + let obj = createUnsavedDataObject(type); + obj.synced = true; + obj.version = 5; + yield obj.saveTx(); + objects[type] = [obj]; + + objectJSON[type].push(makeJSONFunctions[type]({ + key: obj.key, + version: 20, + name: Zotero.Utilities.randomString() + })); + + // Create JSON for objects that exist remotely and not locally, + // which should be downloaded + objectJSON[type].push(makeJSONFunctions[type]({ + key: Zotero.DataObjectUtilities.generateKey(), + version: 20, + name: Zotero.Utilities.randomString() + })); + + // Create objects marked as synced that don't exist remotely, + // which should be flagged for upload + obj = createUnsavedDataObject(type); + obj.synced = true; + obj.version = 10; + yield obj.saveTx(); + objects[type].push(obj); + } + + var headers = { + "Last-Modified-Version": 20 + } + setResponse({ + method: "GET", + url: "users/1/settings", + status: 200, + headers: headers, + json: { + tagColors: { + value: [ + { + name: "A", + color: "#CC66CC" + } + ], + version: 2 + } + } + }); + for (let type of types) { + var suffix = type == 'item' ? '&includeTrashed=1' : ''; + + var json = {}; + json[objectJSON[type][0].key] = objectJSON[type][0].version; + json[objectJSON[type][1].key] = objectJSON[type][1].version; + setResponse({ + method: "GET", + url: "users/1/" + Zotero.DataObjectUtilities.getObjectTypePlural(type) + + "?format=versions" + suffix, + status: 200, + headers: headers, + json: json + }); + + setResponse({ + method: "GET", + url: "users/1/" + Zotero.DataObjectUtilities.getObjectTypePlural(type) + + "?format=json" + + "&" + type + "Key=" + objectJSON[type][0].key + "%2C" + objectJSON[type][1].key + + suffix, + status: 200, + headers: headers, + json: objectJSON[type] + }); + } + yield engine._fullSync(); + + // Check settings + var setting = yield Zotero.SyncedSettings.get(userLibraryID, "tagColors"); + assert.lengthOf(setting, 1); + assert.equal(setting[0].name, 'A'); + var settingMetadata = yield Zotero.SyncedSettings.getMetadata(userLibraryID, "tagColors"); + assert.equal(settingMetadata.version, 2); + assert.isTrue(settingMetadata.synced); + + // Check objects + for (let type of types) { + // Objects 1 should be updated with version from server + assert.equal(objects[type][0].version, 20); + assert.isTrue(objects[type][0].synced); + + // JSON objects 1 should be created locally with version from server + let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(type); + let obj = objectsClass.getByLibraryAndKey(userLibraryID, objectJSON[type][0].key); + assert.equal(obj.version, 20); + assert.isTrue(obj.synced); + + // JSON objects 2 should be marked as unsynced, with their version reset to 0 + assert.equal(objects[type][1].version, 0); + assert.isFalse(objects[type][1].synced); + } + }) + }) +}) diff --git a/test/tests/syncLocalTest.js b/test/tests/syncLocalTest.js new file mode 100644 index 000000000..e8db1738c --- /dev/null +++ b/test/tests/syncLocalTest.js @@ -0,0 +1,984 @@ +"use strict"; + +describe("Zotero.Sync.Data.Local", function() { + describe("#processSyncCacheForObjectType()", function () { + var types = Zotero.DataObjectUtilities.getTypes(); + + it("should update local version number if remote version is identical", function* () { + var libraryID = Zotero.Libraries.userLibraryID; + + for (let type of types) { + let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(type); + let obj = yield createDataObject(type); + let data = yield obj.toJSON(); + data.key = obj.key; + data.version = 10; + let json = { + key: obj.key, + version: 10, + data: data + }; + yield Zotero.Sync.Data.Local.saveCacheObjects( + type, libraryID, [json] + ); + yield Zotero.Sync.Data.Local.processSyncCacheForObjectType( + libraryID, type, { stopOnError: true } + ); + assert.equal( + objectsClass.getByLibraryAndKey(libraryID, obj.key).version, 10 + ); + } + }) + }) + + describe("#_reconcileChanges()", function () { + describe("items", function () { + it("should ignore non-conflicting local changes and return remote changes", function () { + var cacheJSON = { + key: "AAAAAAAA", + version: 1234, + itemType: "book", + title: "Title 1", + url: "http://zotero.org/", + publicationTitle: "Publisher", // Remove locally + extra: "Extra", // Removed on both + dateModified: "2015-05-14 12:34:56", + collections: [ + 'AAAAAAAA', // Removed locally + 'DDDDDDDD', // Removed remotely, + 'EEEEEEEE' // Removed from both + ], + relations: { + a: 'A', // Unchanged string + c: ['C1', 'C2'], // Unchanged array + d: 'D', // String removed locally + e: ['E'], // Array removed locally + f: 'F1', // String changed locally + g: [ + 'G1', // Unchanged + 'G2', // Removed remotely + 'G3' // Removed from both + ], + h: 'H', // String removed remotely + i: ['I'], // Array removed remotely + }, + tags: [ + { tag: 'A' }, // Removed locally + { tag: 'D' }, // Removed remotely + { tag: 'E' } // Removed from both + ] + }; + var json1 = { + key: "AAAAAAAA", + version: 1234, + itemType: "book", + title: "Title 2", // Changed locally + url: "https://www.zotero.org/", // Same change on local and remote + place: "Place", // Added locally + dateModified: "2015-05-14 14:12:34", // Changed locally and remotely, but ignored + collections: [ + 'BBBBBBBB', // Added locally + 'DDDDDDDD', + 'FFFFFFFF' // Added on both + ], + relations: { + 'a': 'A', + 'b': 'B', // String added locally + 'f': 'F2', + 'g': [ + 'G1', + 'G2', + 'G6' // Added locally and remotely + ], + h: 'H', // String removed remotely + i: ['I'], // Array removed remotely + + }, + tags: [ + { tag: 'B' }, + { tag: 'D' }, + { tag: 'F', type: 1 }, // Added on both + { tag: 'G' }, // Added on both, but with different types + { tag: 'H', type: 1 } // Added on both, but with different types + ] + }; + var json2 = { + key: "AAAAAAAA", + version: 1235, + itemType: "book", + title: "Title 1", + url: "https://www.zotero.org/", + publicationTitle: "Publisher", + date: "2015-05-15", // Added remotely + dateModified: "2015-05-14 13:45:12", + collections: [ + 'AAAAAAAA', + 'CCCCCCCC', // Added remotely + 'FFFFFFFF' + ], + relations: { + 'a': 'A', + 'd': 'D', + 'e': ['E'], + 'f': 'F1', + 'g': [ + 'G1', + 'G4', // Added remotely + 'G6' + ], + }, + tags: [ + { tag: 'A' }, + { tag: 'C' }, + { tag: 'F', type: 1 }, + { tag: 'G', type: 1 }, + { tag: 'H' } + ] + }; + var ignoreFields = ['dateAdded', 'dateModified']; + var result = Zotero.Sync.Data.Local._reconcileChanges( + 'item', cacheJSON, json1, json2, ignoreFields + ); + assert.sameDeepMembers( + result.changes, + [ + { + field: "date", + op: "add", + value: "2015-05-15" + }, + { + field: "collections", + op: "member-add", + value: "CCCCCCCC" + }, + { + field: "collections", + op: "member-remove", + value: "DDDDDDDD" + }, + // Relations + { + field: "relations", + op: "property-member-remove", + value: { + key: 'g', + value: 'G2' + } + }, + { + field: "relations", + op: "property-member-add", + value: { + key: 'g', + value: 'G4' + } + }, + { + field: "relations", + op: "property-member-remove", + value: { + key: 'h', + value: 'H' + } + }, + { + field: "relations", + op: "property-member-remove", + value: { + key: 'i', + value: 'I' + } + }, + // Tags + { + field: "tags", + op: "member-add", + value: { + tag: 'C' + } + }, + { + field: "tags", + op: "member-remove", + value: { + tag: 'D' + } + }, + { + field: "tags", + op: "member-remove", + value: { + tag: 'H', + type: 1 + } + }, + { + field: "tags", + op: "member-add", + value: { + tag: 'H' + } + } + ] + ); + assert.lengthOf(result.conflicts, 0); + }) + + it("should return empty arrays when no remote changes to apply", function () { + // Similar to above but without differing remote changes + var cacheJSON = { + key: "AAAAAAAA", + version: 1234, + itemType: "book", + title: "Title 1", + url: "http://zotero.org/", + publicationTitle: "Publisher", // Remove locally + extra: "Extra", // Removed on both + dateModified: "2015-05-14 12:34:56", + collections: [ + 'AAAAAAAA', // Removed locally + 'DDDDDDDD', + 'EEEEEEEE' // Removed from both + ], + tags: [ + { + tag: 'A' // Removed locally + }, + { + tag: 'D' // Removed remotely + }, + { + tag: 'E' // Removed from both + } + ] + }; + var json1 = { + key: "AAAAAAAA", + version: 1234, + itemType: "book", + title: "Title 2", // Changed locally + url: "https://www.zotero.org/", // Same change on local and remote + place: "Place", // Added locally + dateModified: "2015-05-14 14:12:34", // Changed locally and remotely, but ignored + collections: [ + 'BBBBBBBB', // Added locally + 'DDDDDDDD', + 'FFFFFFFF' // Added on both + ], + tags: [ + { + tag: 'B' + }, + { + tag: 'D' + }, + { + tag: 'F', // Added on both + type: 1 + }, + { + tag: 'G' // Added on both, but with different types + } + ] + }; + var json2 = { + key: "AAAAAAAA", + version: 1235, + itemType: "book", + title: "Title 1", + url: "https://www.zotero.org/", + publicationTitle: "Publisher", + dateModified: "2015-05-14 13:45:12", + collections: [ + 'AAAAAAAA', + 'DDDDDDDD', + 'FFFFFFFF' + ], + tags: [ + { + tag: 'A' + }, + { + tag: 'D' + }, + { + tag: 'F', + type: 1 + }, + { + tag: 'G', + type: 1 + } + ] + }; + var ignoreFields = ['dateAdded', 'dateModified']; + var result = Zotero.Sync.Data.Local._reconcileChanges( + 'item', cacheJSON, json1, json2, ignoreFields + ); + assert.lengthOf(result.changes, 0); + assert.lengthOf(result.conflicts, 0); + }) + + it("should return conflict when changes can't be automatically resolved", function () { + var cacheJSON = { + key: "AAAAAAAA", + version: 1234, + title: "Title 1", + dateModified: "2015-05-14 12:34:56" + }; + var json1 = { + key: "AAAAAAAA", + version: 1234, + title: "Title 2", + dateModified: "2015-05-14 14:12:34" + }; + var json2 = { + key: "AAAAAAAA", + version: 1235, + title: "Title 3", + dateModified: "2015-05-14 13:45:12" + }; + var ignoreFields = ['dateAdded', 'dateModified']; + var result = Zotero.Sync.Data.Local._reconcileChanges( + 'item', cacheJSON, json1, json2, ignoreFields + ); + Zotero.debug('=-=-=-='); + Zotero.debug(result); + assert.lengthOf(result.changes, 0); + assert.sameDeepMembers( + result.conflicts, + [ + [ + { + field: "title", + op: "modify", + value: "Title 2" + }, + { + field: "title", + op: "modify", + value: "Title 3" + } + ] + ] + ); + }) + + it("should automatically merge array/object members and generate conflicts for field changes in absence of cached version", function () { + var json1 = { + key: "AAAAAAAA", + version: 1234, + itemType: "book", + title: "Title", + creators: [ + { + name: "Center for History and New Media", + creatorType: "author" + } + ], + place: "Place", // Local + dateModified: "2015-05-14 14:12:34", // Changed on both, but ignored + collections: [ + 'AAAAAAAA' // Local + ], + relations: { + 'a': 'A', + 'b': 'B', // Local + 'e': 'E1', + 'f': [ + 'F1', + 'F2' // Local + ], + h: 'H', // String removed remotely + i: ['I'], // Array removed remotely + }, + tags: [ + { tag: 'A' }, // Local + { tag: 'C' }, + { tag: 'F', type: 1 }, + { tag: 'G' }, // Different types + { tag: 'H', type: 1 } // Different types + ] + }; + var json2 = { + key: "AAAAAAAA", + version: 1235, + itemType: "book", + title: "Title", + creators: [ + { + creatorType: "author", // Different property order shouldn't matter + name: "Center for History and New Media" + } + ], + date: "2015-05-15", // Remote + dateModified: "2015-05-14 13:45:12", + collections: [ + 'BBBBBBBB' // Remote + ], + relations: { + 'a': 'A', + 'c': 'C', // Remote + 'd': ['D'], // Remote + 'e': 'E2', + 'f': [ + 'F1', + 'F3' // Remote + ], + }, + tags: [ + { tag: 'B' }, // Remote + { tag: 'C' }, + { tag: 'F', type: 1 }, + { tag: 'G', type: 1 }, // Different types + { tag: 'H' } // Different types + ] + }; + var ignoreFields = ['dateAdded', 'dateModified']; + var result = Zotero.Sync.Data.Local._reconcileChanges( + 'item', false, json1, json2, ignoreFields + ); + Zotero.debug(result); + assert.sameDeepMembers( + result.changes, + [ + // Collections + { + field: "collections", + op: "member-add", + value: "BBBBBBBB" + }, + // Relations + { + field: "relations", + op: "property-member-add", + value: { + key: 'c', + value: 'C' + } + }, + { + field: "relations", + op: "property-member-add", + value: { + key: 'd', + value: 'D' + } + }, + { + field: "relations", + op: "property-member-add", + value: { + key: 'e', + value: 'E2' + } + }, + { + field: "relations", + op: "property-member-add", + value: { + key: 'f', + value: 'F3' + } + }, + // Tags + { + field: "tags", + op: "member-add", + value: { + tag: 'B' + } + }, + { + field: "tags", + op: "member-add", + value: { + tag: 'G', + type: 1 + } + }, + { + field: "tags", + op: "member-add", + value: { + tag: 'H' + } + } + ] + ); + assert.sameDeepMembers( + result.conflicts, + [ + { + field: "place", + op: "delete" + }, + { + field: "date", + op: "add", + value: "2015-05-15" + } + ] + ); + }) + }) + + + describe("collections", function () { + it("should ignore non-conflicting local changes and return remote changes", function () { + var cacheJSON = { + key: "AAAAAAAA", + version: 1234, + name: "Name 1", + parentCollection: null, + relations: { + A: "A", // Removed locally + C: "C" // Removed on both + } + }; + var json1 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 2", // Changed locally + parentCollection: null, + relations: {} + }; + var json2 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 1", + parentCollection: "BBBBBBBB", // Added remotely + relations: { + A: "A", + B: "B" // Added remotely + } + }; + var result = Zotero.Sync.Data.Local._reconcileChanges( + 'collection', cacheJSON, json1, json2 + ); + assert.sameDeepMembers( + result.changes, + [ + { + field: "parentCollection", + op: "add", + value: "BBBBBBBB" + }, + { + field: "relations", + op: "property-member-add", + value: { + key: "B", + value: "B" + } + } + ] + ); + assert.lengthOf(result.conflicts, 0); + }) + + it("should return empty arrays when no remote changes to apply", function () { + // Similar to above but without differing remote changes + var cacheJSON = { + key: "AAAAAAAA", + version: 1234, + name: "Name 1", + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + { + condition: "place", + operator: "is", + value: "Chicago" + } + ] + }; + var json1 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 2", // Changed locally + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + // Added locally + { + condition: "place", + operator: "is", + value: "New York" + }, + { + condition: "place", + operator: "is", + value: "Chicago" + } + ] + }; + var json2 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 1", + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + { + condition: "place", + operator: "is", + value: "Chicago" + } + ] + }; + var result = Zotero.Sync.Data.Local._reconcileChanges( + 'search', cacheJSON, json1, json2 + ); + assert.lengthOf(result.changes, 0); + assert.lengthOf(result.conflicts, 0); + }) + + it("should automatically resolve conflicts with remote version", function () { + var cacheJSON = { + key: "AAAAAAAA", + version: 1234, + name: "Name 1" + }; + var json1 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 2" + }; + var json2 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 3" + }; + var result = Zotero.Sync.Data.Local._reconcileChanges( + 'search', cacheJSON, json1, json2 + ); + assert.sameDeepMembers( + result.changes, + [ + { + field: "name", + op: "modify", + value: "Name 3" + } + ] + ); + assert.lengthOf(result.conflicts, 0); + }) + + it("should automatically resolve conflicts in absence of cached version", function () { + var json1 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 1", + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + { + condition: "place", + operator: "is", + value: "New York" + } + ] + }; + var json2 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 2", + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + { + condition: "place", + operator: "is", + value: "Chicago" + } + ] + }; + var result = Zotero.Sync.Data.Local._reconcileChanges( + 'search', false, json1, json2 + ); + assert.sameDeepMembers( + result.changes, + [ + { + field: "name", + op: "modify", + value: "Name 2" + }, + { + field: "conditions", + op: "member-add", + value: { + condition: "place", + operator: "is", + value: "Chicago" + } + } + ] + ); + assert.lengthOf(result.conflicts, 0); + }) + }) + + + describe("searches", function () { + it("should ignore non-conflicting local changes and return remote changes", function () { + var cacheJSON = { + key: "AAAAAAAA", + version: 1234, + name: "Name 1", + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + { + condition: "place", + operator: "is", + value: "Chicago" + } + ] + }; + var json1 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 2", // Changed locally + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + // Removed remotely + { + condition: "place", + operator: "is", + value: "Chicago" + } + ] + }; + var json2 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 1", + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + // Added remotely + { + condition: "place", + operator: "is", + value: "New York" + } + ] + }; + var result = Zotero.Sync.Data.Local._reconcileChanges( + 'search', cacheJSON, json1, json2 + ); + assert.sameDeepMembers( + result.changes, + [ + { + field: "conditions", + op: "member-add", + value: { + condition: "place", + operator: "is", + value: "New York" + } + }, + { + field: "conditions", + op: "member-remove", + value: { + condition: "place", + operator: "is", + value: "Chicago" + } + } + ] + ); + assert.lengthOf(result.conflicts, 0); + }) + + it("should return empty arrays when no remote changes to apply", function () { + // Similar to above but without differing remote changes + var cacheJSON = { + key: "AAAAAAAA", + version: 1234, + name: "Name 1", + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + { + condition: "place", + operator: "is", + value: "Chicago" + } + ] + }; + var json1 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 2", // Changed locally + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + // Added locally + { + condition: "place", + operator: "is", + value: "New York" + }, + { + condition: "place", + operator: "is", + value: "Chicago" + } + ] + }; + var json2 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 1", + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + { + condition: "place", + operator: "is", + value: "Chicago" + } + ] + }; + var result = Zotero.Sync.Data.Local._reconcileChanges( + 'search', cacheJSON, json1, json2 + ); + assert.lengthOf(result.changes, 0); + assert.lengthOf(result.conflicts, 0); + }) + + it("should automatically resolve conflicts with remote version", function () { + var cacheJSON = { + key: "AAAAAAAA", + version: 1234, + name: "Name 1" + }; + var json1 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 2" + }; + var json2 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 3" + }; + var result = Zotero.Sync.Data.Local._reconcileChanges( + 'search', cacheJSON, json1, json2 + ); + assert.sameDeepMembers( + result.changes, + [ + { + field: "name", + op: "modify", + value: "Name 3" + } + ] + ); + assert.lengthOf(result.conflicts, 0); + }) + + it("should automatically resolve conflicts in absence of cached version", function () { + var json1 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 1", + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + { + condition: "place", + operator: "is", + value: "New York" + } + ] + }; + var json2 = { + key: "AAAAAAAA", + version: 1234, + name: "Name 2", + conditions: [ + { + condition: "title", + operator: "contains", + value: "A" + }, + { + condition: "place", + operator: "is", + value: "Chicago" + } + ] + }; + var result = Zotero.Sync.Data.Local._reconcileChanges( + 'search', false, json1, json2 + ); + assert.sameDeepMembers( + result.changes, + [ + { + field: "name", + op: "modify", + value: "Name 2" + }, + { + field: "conditions", + op: "member-add", + value: { + condition: "place", + operator: "is", + value: "Chicago" + } + } + ] + ); + assert.lengthOf(result.conflicts, 0); + }) + }) + }) +}) diff --git a/test/tests/syncRunnerTest.js b/test/tests/syncRunnerTest.js new file mode 100644 index 000000000..bbc1932ef --- /dev/null +++ b/test/tests/syncRunnerTest.js @@ -0,0 +1,653 @@ +"use strict"; + +describe("Zotero.Sync.Runner", function () { + var apiKey = Zotero.Utilities.randomString(24); + var baseURL = "http://local.zotero/"; + var userLibraryID, publicationsLibraryID, runner, caller, server, client, stub, spy; + + var responses = { + keyInfo: { + fullAccess: { + method: "GET", + url: "keys/" + apiKey, + status: 200, + json: { + key: apiKey, + userID: 1, + username: "Username", + access: { + user: { + library: true, + files: true, + notes: true, + write: true + }, + groups: { + all: { + library: true, + write: true + } + } + } + } + } + }, + userGroups: { + groupVersions: { + method: "GET", + url: "users/1/groups?format=versions", + json: { + "1623562": 10, + "2694172": 11 + } + }, + groupVersionsEmpty: { + method: "GET", + url: "users/1/groups?format=versions", + json: {} + }, + groupVersionsOnlyMemberGroup: { + method: "GET", + url: "users/1/groups?format=versions", + json: { + "2694172": 11 + } + } + }, + groups: { + ownerGroup: { + method: "GET", + url: "groups/1623562", + json: { + id: 1623562, + version: 10, + data: { + id: 1623562, + version: 10, + name: "Group Name", + description: "

Test group

", + owner: 1, + type: "Private", + libraryEditing: "members", + libraryReading: "all", + fileEditing: "members", + admins: [], + members: [] + } + } + }, + memberGroup: { + method: "GET", + url: "groups/2694172", + json: { + id: 2694172, + version: 11, + data: { + id: 2694172, + version: 11, + name: "Group Name 2", + description: "

Test group

", + owner: 123456, + type: "Private", + libraryEditing: "admins", + libraryReading: "all", + fileEditing: "admins", + admins: [], + members: [1] + } + } + } + } + }; + + // + // Helper functions + // + var setup = Zotero.Promise.coroutine(function* (options = {}) { + yield Zotero.DB.queryAsync("DELETE FROM settings WHERE setting='account'"); + yield Zotero.Users.init(); + + var runner = new Zotero.Sync.Runner_Module({ + baseURL: baseURL, + apiKey: apiKey + }); + + Components.utils.import("resource://zotero/concurrent-caller.js"); + var caller = new ConcurrentCaller(1); + caller.setLogger(msg => Zotero.debug(msg)); + caller.stopOnError = true; + caller.onError = function (e) { + Zotero.logError(e); + if (options.onError) { + options.onError(e); + } + if (e.fatal) { + caller.stop(); + throw e; + } + }; + + var client = new Zotero.Sync.APIClient({ + baseURL: baseURL, + apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION, + apiKey: apiKey, + concurrentCaller: caller, + background: options.background || true + }); + + return { runner, caller, client }; + }) + + function setResponse(response) { + setHTTPResponse(server, baseURL, response, responses); + } + + + // + // Tests + // + before(function () { + userLibraryID = Zotero.Libraries.userLibraryID; + publicationsLibraryID = Zotero.Libraries.publicationsLibraryID; + }) + beforeEach(function* () { + Zotero.HTTP.mock = sinon.FakeXMLHttpRequest; + + server = sinon.fakeServer.create(); + server.autoRespond = true; + + ({ runner, caller, client } = yield setup()); + + yield Zotero.Users.setCurrentUserID(1); + yield Zotero.Users.setCurrentUsername("A"); + }) + afterEach(function () { + if (stub) stub.restore(); + if (spy) spy.restore(); + }) + after(function () { + Zotero.HTTP.mock = null; + }) + + describe("#checkAccess()", function () { + it("should check key access", function* () { + spy = sinon.spy(runner, "checkUser"); + setResponse('keyInfo.fullAccess'); + var json = yield runner.checkAccess(client); + sinon.assert.calledWith(spy, 1, "Username"); + var compare = {}; + Object.assign(compare, responses.keyInfo.fullAccess.json); + delete compare.key; + assert.deepEqual(json, compare); + }) + }) + + describe("#checkLibraries()", function () { + afterEach(function* () { + var group = Zotero.Groups.get(responses.groups.ownerGroup.json.id); + if (group) { + yield group.eraseTx(); + } + group = Zotero.Groups.get(responses.groups.memberGroup.json.id); + if (group) { + yield group.eraseTx(); + } + }) + + it("should check library access and versions without library list", function* () { + // Create group with same id and version as groups response + var groupData = responses.groups.ownerGroup; + var group1 = yield createGroup({ + id: groupData.json.id, + version: groupData.json.version + }); + groupData = responses.groups.memberGroup; + var group2 = yield createGroup({ + id: groupData.json.id, + version: groupData.json.version + }); + + setResponse('userGroups.groupVersions'); + var libraries = yield runner.checkLibraries( + client, false, responses.keyInfo.fullAccess.json + ); + assert.lengthOf(libraries, 4); + assert.sameMembers( + libraries, + [userLibraryID, publicationsLibraryID, group1.libraryID, group2.libraryID] + ); + }) + + it("should check library access and versions with library list", function* () { + // Create groups with same id and version as groups response + var groupData = responses.groups.ownerGroup; + var group1 = yield createGroup({ + id: groupData.json.id, + version: groupData.json.version + }); + groupData = responses.groups.memberGroup; + var group2 = yield createGroup({ + id: groupData.json.id, + version: groupData.json.version + }); + + setResponse('userGroups.groupVersions'); + var libraries = yield runner.checkLibraries( + client, false, responses.keyInfo.fullAccess.json, [userLibraryID] + ); + assert.lengthOf(libraries, 1); + assert.sameMembers(libraries, [userLibraryID]); + + var libraries = yield runner.checkLibraries( + client, false, responses.keyInfo.fullAccess.json, [userLibraryID, publicationsLibraryID] + ); + assert.lengthOf(libraries, 2); + assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID]); + + var libraries = yield runner.checkLibraries( + client, false, responses.keyInfo.fullAccess.json, [group1.libraryID] + ); + assert.lengthOf(libraries, 1); + assert.sameMembers(libraries, [group1.libraryID]); + }) + + it("should update outdated group metadata", function* () { + // Create groups with same id as groups response but earlier versions + var groupData1 = responses.groups.ownerGroup; + var group1 = yield createGroup({ + id: groupData1.json.id, + version: groupData1.json.version - 1, + editable: false + }); + var groupData2 = responses.groups.memberGroup; + var group2 = yield createGroup({ + id: groupData2.json.id, + version: groupData2.json.version - 1, + editable: true + }); + + setResponse('userGroups.groupVersions'); + setResponse('groups.ownerGroup'); + setResponse('groups.memberGroup'); + var libraries = yield runner.checkLibraries( + client, false, responses.keyInfo.fullAccess.json + ); + assert.lengthOf(libraries, 4); + assert.sameMembers( + libraries, + [userLibraryID, publicationsLibraryID, group1.libraryID, group2.libraryID] + ); + + assert.equal(group1.name, groupData1.json.data.name); + assert.equal(group1.version, groupData1.json.version); + assert.isTrue(group1.editable); + assert.equal(group2.name, groupData2.json.data.name); + assert.equal(group2.version, groupData2.json.version); + assert.isFalse(group2.editable); + }) + + it("should update outdated group metadata for group created with classic sync", function* () { + var groupData1 = responses.groups.ownerGroup; + var group1 = yield createGroup({ + id: groupData1.json.id, + version: 0, + editable: false + }); + var groupData2 = responses.groups.memberGroup; + var group2 = yield createGroup({ + id: groupData2.json.id, + version: 0, + editable: true + }); + + yield Zotero.DB.queryAsync( + "UPDATE groups SET version=0 WHERE groupID IN (?, ?)", [group1.id, group2.id] + ); + yield Zotero.Groups.init(); + group1 = Zotero.Groups.get(group1.id); + group2 = Zotero.Groups.get(group2.id); + + setResponse('userGroups.groupVersions'); + setResponse('groups.ownerGroup'); + setResponse('groups.memberGroup'); + var libraries = yield runner.checkLibraries( + client, + false, + responses.keyInfo.fullAccess.json, + [group1.libraryID, group2.libraryID] + ); + assert.lengthOf(libraries, 2); + assert.sameMembers(libraries, [group1.libraryID, group2.libraryID]); + + assert.equal(group1.name, groupData1.json.data.name); + assert.equal(group1.version, groupData1.json.version); + assert.isTrue(group1.editable); + assert.equal(group2.name, groupData2.json.data.name); + assert.equal(group2.version, groupData2.json.version); + assert.isFalse(group2.editable); + }) + + it("should create locally missing groups", function* () { + setResponse('userGroups.groupVersions'); + setResponse('groups.ownerGroup'); + setResponse('groups.memberGroup'); + var libraries = yield runner.checkLibraries( + client, false, responses.keyInfo.fullAccess.json + ); + assert.lengthOf(libraries, 4); + var groupData1 = responses.groups.ownerGroup; + var group1 = Zotero.Groups.get(groupData1.json.id); + var groupData2 = responses.groups.memberGroup; + var group2 = Zotero.Groups.get(groupData2.json.id); + assert.ok(group1); + assert.ok(group2); + assert.sameMembers( + libraries, + [userLibraryID, publicationsLibraryID, group1.libraryID, group2.libraryID] + ); + assert.equal(group1.name, groupData1.json.data.name); + assert.isTrue(group1.editable); + assert.equal(group2.name, groupData2.json.data.name); + assert.isFalse(group2.editable); + }) + + it("should delete remotely missing groups", function* () { + var groupData1 = responses.groups.ownerGroup; + var group1 = yield createGroup({ id: groupData1.json.id, version: groupData1.json.version }); + var groupData2 = responses.groups.memberGroup; + var group2 = yield createGroup({ id: groupData2.json.id, version: groupData2.json.version }); + + setResponse('userGroups.groupVersionsOnlyMemberGroup'); + waitForDialog(function (dialog) { + var text = dialog.document.documentElement.textContent; + assert.include(text, group1.name); + }); + var libraries = yield runner.checkLibraries( + client, false, responses.keyInfo.fullAccess.json + ); + assert.lengthOf(libraries, 3); + assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID, group2.libraryID]); + assert.isFalse(Zotero.Groups.exists(groupData1.json.id)); + assert.isTrue(Zotero.Groups.exists(groupData2.json.id)); + }) + + it.skip("should keep remotely missing groups", function* () { + var groupData = responses.groups.ownerGroup; + var group = yield createGroup({ id: groupData.json.id, version: groupData.json.version }); + + setResponse('userGroups.groupVersionsEmpty'); + waitForDialog(function (dialog) { + var text = dialog.document.documentElement.textContent; + assert.include(text, group.name); + }, "extra1"); + var libraries = yield runner.checkLibraries( + client, false, responses.keyInfo.fullAccess.json + ); + assert.lengthOf(libraries, 3); + assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID, group.libraryID]); + assert.isTrue(Zotero.Groups.exists(groupData.json.id)); + }) + + it("should cancel sync with remotely missing groups", function* () { + var groupData = responses.groups.ownerGroup; + var group = yield createGroup({ id: groupData.json.id, version: groupData.json.version }); + + setResponse('userGroups.groupVersionsEmpty'); + waitForDialog(function (dialog) { + var text = dialog.document.documentElement.textContent; + assert.include(text, group.name); + }, "cancel"); + var libraries = yield runner.checkLibraries( + client, false, responses.keyInfo.fullAccess.json + ); + assert.lengthOf(libraries, 0); + assert.isTrue(Zotero.Groups.exists(groupData.json.id)); + }) + }) + + describe("#checkUser()", function () { + it("should prompt for user update and perform on accept", function* () { + waitForDialog(function (dialog) { + var text = dialog.document.documentElement.textContent; + var matches = text.match(/'[^']*'/g); + assert.equal(matches.length, 4); + assert.equal(matches[0], "'A'"); + assert.equal(matches[1], "'B'"); + assert.equal(matches[2], "'B'"); + assert.equal(matches[3], "'A'"); + }); + var cont = yield runner.checkUser(2, "B"); + assert.isTrue(cont); + + assert.equal(Zotero.Users.getCurrentUserID(), 2); + assert.equal(Zotero.Users.getCurrentUsername(), "B"); + }) + + it("should prompt for user update and cancel", function* () { + yield Zotero.Users.setCurrentUserID(1); + yield Zotero.Users.setCurrentUsername("A"); + + waitForDialog(false, 'cancel'); + var cont = yield runner.checkUser(2, "B"); + assert.isFalse(cont); + + assert.equal(Zotero.Users.getCurrentUserID(), 1); + assert.equal(Zotero.Users.getCurrentUsername(), "A"); + }) + }) + + describe("#sync()", function () { + before(function* () { + this.timeout(60000); + yield resetDB({ + skipBundledFiles: true + }); + + yield Zotero.Groups.init(); + }) + after(function* () { + this.timeout(60000); + yield resetDB(); + }) + + it("should perform a sync across all libraries", function* () { + yield Zotero.Users.setCurrentUserID(1); + yield Zotero.Users.setCurrentUsername("A"); + + setResponse('keyInfo.fullAccess'); + setResponse('userGroups.groupVersions'); + setResponse('groups.ownerGroup'); + setResponse('groups.memberGroup'); + // My Library + setResponse({ + method: "GET", + url: "users/1/settings", + status: 200, + headers: { + "Last-Modified-Version": 5 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "users/1/collections?format=versions", + status: 200, + headers: { + "Last-Modified-Version": 5 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "users/1/searches?format=versions", + status: 200, + headers: { + "Last-Modified-Version": 5 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "users/1/items?format=versions&includeTrashed=1", + status: 200, + headers: { + "Last-Modified-Version": 5 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "users/1/deleted?since=0", + status: 200, + headers: { + "Last-Modified-Version": 5 + }, + json: [] + }); + // My Publications + setResponse({ + method: "GET", + url: "users/1/publications/settings", + status: 200, + headers: { + "Last-Modified-Version": 10 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "users/1/publications/items?format=versions&includeTrashed=1", + status: 200, + headers: { + "Last-Modified-Version": 10 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "users/1/publications/deleted?since=0", + status: 200, + headers: { + "Last-Modified-Version": 10 + }, + json: [] + }); + // Group library 1 + setResponse({ + method: "GET", + url: "groups/1623562/settings", + status: 200, + headers: { + "Last-Modified-Version": 15 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "groups/1623562/collections?format=versions", + status: 200, + headers: { + "Last-Modified-Version": 15 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "groups/1623562/searches?format=versions", + status: 200, + headers: { + "Last-Modified-Version": 15 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "groups/1623562/items?format=versions&includeTrashed=1", + status: 200, + headers: { + "Last-Modified-Version": 15 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "groups/1623562/deleted?since=0", + status: 200, + headers: { + "Last-Modified-Version": 15 + }, + json: [] + }); + // Group library 2 + setResponse({ + method: "GET", + url: "groups/2694172/settings", + status: 200, + headers: { + "Last-Modified-Version": 20 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "groups/2694172/collections?format=versions", + status: 200, + headers: { + "Last-Modified-Version": 20 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "groups/2694172/searches?format=versions", + status: 200, + headers: { + "Last-Modified-Version": 20 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "groups/2694172/items?format=versions&includeTrashed=1", + status: 200, + headers: { + "Last-Modified-Version": 20 + }, + json: [] + }); + setResponse({ + method: "GET", + url: "groups/2694172/deleted?since=0", + status: 200, + headers: { + "Last-Modified-Version": 20 + }, + json: [] + }); + + yield runner.sync({ + baseURL: baseURL, + apiKey: apiKey, + onError: e => { throw e }, + }); + + // Check local library versions + assert.equal( + Zotero.Libraries.getVersion(Zotero.Libraries.userLibraryID), + 5 + ); + assert.equal( + Zotero.Libraries.getVersion(Zotero.Libraries.publicationsLibraryID), + 10 + ); + assert.equal( + Zotero.Libraries.getVersion(Zotero.Groups.getLibraryIDFromGroupID(1623562)), + 15 + ); + assert.equal( + Zotero.Libraries.getVersion(Zotero.Groups.getLibraryIDFromGroupID(2694172)), + 20 + ); + }) + }) +})