From 73f4d28ab26de537efc8e7d1a3e38d2ecdc01e58 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Thu, 29 Oct 2015 03:41:54 -0400 Subject: [PATCH] ZFS file sync overhaul for API syncing This mostly gets ZFS file syncing and file conflict resolution working with the API sync process. WebDAV will need to be updated separately. Known issues: - File sync progress is temporarily gone - File uploads can result in an unnecessary 412 loop on the next data sync - This causes Firefox to crash on one of my computers during tests, which would be easier to debug if it produced a crash log. Also: - Adds httpd.js for use in tests when FakeXMLHttpRequest can't be used (e.g., saveURI()). - Adds some additional test data files for attachment tests --- .../content/zotero/bindings/attachmentbox.xml | 371 +- chrome/content/zotero/bindings/itembox.xml | 9 +- chrome/content/zotero/bindings/merge.xml | 22 +- chrome/content/zotero/bindings/noteeditor.xml | 1 + chrome/content/zotero/bindings/relatedbox.xml | 1 + chrome/content/zotero/merge.js | 32 +- chrome/content/zotero/xpcom/data/item.js | 28 +- chrome/content/zotero/xpcom/data/items.js | 3 +- chrome/content/zotero/xpcom/file.js | 90 +- chrome/content/zotero/xpcom/storage.js | 1661 +---- chrome/content/zotero/xpcom/storage/mode.js | 87 - chrome/content/zotero/xpcom/storage/queue.js | 427 -- .../zotero/xpcom/storage/queueManager.js | 370 -- .../zotero/xpcom/storage/storageEngine.js | 307 + .../zotero/xpcom/storage/storageLocal.js | 1088 ++++ .../storage/{request.js => storageRequest.js} | 184 +- .../zotero/xpcom/storage/storageResult.js | 47 + .../zotero/xpcom/storage/storageUtilities.js | 67 + .../zotero/xpcom/storage/streamListener.js | 79 +- chrome/content/zotero/xpcom/storage/webdav.js | 1579 +++-- chrome/content/zotero/xpcom/storage/zfs.js | 2003 +++--- chrome/content/zotero/xpcom/sync.js | 37 - .../zotero/xpcom/sync/syncAPIClient.js | 105 +- .../content/zotero/xpcom/sync/syncEngine.js | 70 +- .../zotero/xpcom/sync/syncEventListeners.js | 56 +- chrome/content/zotero/xpcom/sync/syncLocal.js | 333 +- .../content/zotero/xpcom/sync/syncRunner.js | 351 +- chrome/content/zotero/xpcom/zotero.js | 4 + chrome/content/zotero/zoteroPane.js | 54 +- chrome/content/zotero/zoteroPane.xul | 5 +- chrome/locale/en-US/zotero/zotero.properties | 12 +- components/zotero-service.js | 9 +- test/resource/httpd.js | 5356 +++++++++++++++++ test/tests/data/snapshot/img.gif | Bin 0 -> 42 bytes test/tests/data/test.html | 8 + test/tests/data/test.txt | 1 + test/tests/itemTest.js | 10 +- test/tests/storageEngineTest.js | 822 +++ test/tests/storageLocalTest.js | 329 + test/tests/storageRequestTest.js | 22 + test/tests/syncEngineTest.js | 20 +- test/tests/syncLocalTest.js | 264 +- test/tests/syncRunnerTest.js | 47 +- test/tests/zoteroPaneTest.js | 94 + 44 files changed, 11226 insertions(+), 5239 deletions(-) delete mode 100644 chrome/content/zotero/xpcom/storage/mode.js delete mode 100644 chrome/content/zotero/xpcom/storage/queue.js delete mode 100644 chrome/content/zotero/xpcom/storage/queueManager.js create mode 100644 chrome/content/zotero/xpcom/storage/storageEngine.js create mode 100644 chrome/content/zotero/xpcom/storage/storageLocal.js rename chrome/content/zotero/xpcom/storage/{request.js => storageRequest.js} (60%) create mode 100644 chrome/content/zotero/xpcom/storage/storageResult.js create mode 100644 chrome/content/zotero/xpcom/storage/storageUtilities.js create mode 100644 test/resource/httpd.js create mode 100644 test/tests/data/snapshot/img.gif create mode 100644 test/tests/data/test.html create mode 100644 test/tests/data/test.txt create mode 100644 test/tests/storageEngineTest.js create mode 100644 test/tests/storageLocalTest.js create mode 100644 test/tests/storageRequestTest.js diff --git a/chrome/content/zotero/bindings/attachmentbox.xml b/chrome/content/zotero/bindings/attachmentbox.xml index 6d98e7a5f..fd53b1a06 100644 --- a/chrome/content/zotero/bindings/attachmentbox.xml +++ b/chrome/content/zotero/bindings/attachmentbox.xml @@ -57,6 +57,7 @@ Zotero.debug("Setting mode to '" + val + "'"); this.editable = false; + this.synchronous = false; this.displayURL = false; this.displayFileName = false; this.clickableLink = false; @@ -93,6 +94,7 @@ break; case 'merge': + this.synchronous = true; this.displayURL = true; this.displayFileName = true; this.displayAccessed = true; @@ -102,6 +104,7 @@ break; case 'mergeedit': + this.synchronous = true; this.editable = true; this.displayURL = true; this.displayFileName = true; @@ -112,6 +115,13 @@ this.displayDateModified = true; break; + case 'filemerge': + this.synchronous = true; + this.displayURL = true; + this.displayFileName = true; + this.displayDateModified = true; + break; + default: throw ("Invalid mode '" + val + "' in attachmentbox.xml"); } @@ -123,18 +133,16 @@ - + + - - - - - @@ -167,125 +175,122 @@ 29 ) || firstSpace > 29) { + title.setAttribute('crop', 'end'); + title.setAttribute('value', val); + } + // Create a element, essentially + else { + title.removeAttribute('value'); + title.appendChild(document.createTextNode(val)); + } + + if (this.editable) { + title.className = 'zotero-clicky'; - yield Zotero.Promise.all([this.item.loadItemData(), this.item.loadNote()]) - .tap(() => Zotero.Promise.check(this.item)); + // For the time being, use a silly little popup + title.addEventListener('click', this.editTitle, false); + } + + var isImportedURL = this.item.attachmentLinkMode == + Zotero.Attachments.LINK_MODE_IMPORTED_URL; + + // Metadata for URL's + if (this.item.attachmentLinkMode == Zotero.Attachments.LINK_MODE_LINKED_URL + || isImportedURL) { - var attachmentBox = document.getAnonymousNodes(this)[0]; - var title = this._id('title'); - var fileNameRow = this._id('fileNameRow'); - var urlField = this._id('url'); - var accessed = this._id('accessedRow'); - var pagesRow = this._id('pagesRow'); - var dateModifiedRow = this._id('dateModifiedRow'); - var indexStatusRow = this._id('indexStatusRow'); - var selectButton = this._id('select-button'); - - // DEBUG: this is annoying -- we really want to use an abstracted - // version of createValueElement() from itemPane.js - // (ideally in an XBL binding) - - // Wrap title to multiple lines if necessary - while (title.hasChildNodes()) { - title.removeChild(title.firstChild); - } - var val = this.item.getField('title'); - - if (typeof val != 'string') { - val += ""; - } - - var firstSpace = val.indexOf(" "); - // Crop long uninterrupted text - if ((firstSpace == -1 && val.length > 29 ) || firstSpace > 29) { - title.setAttribute('crop', 'end'); - title.setAttribute('value', val); - } - // Create a element, essentially - else { - title.removeAttribute('value'); - title.appendChild(document.createTextNode(val)); - } - - if (this.editable) { - title.className = 'zotero-clicky'; - - // For the time being, use a silly little popup - title.addEventListener('click', this.editTitle, false); - } - - var isImportedURL = this.item.attachmentLinkMode == - Zotero.Attachments.LINK_MODE_IMPORTED_URL; - - // Metadata for URL's - if (this.item.attachmentLinkMode == Zotero.Attachments.LINK_MODE_LINKED_URL - || isImportedURL) { - - // URL - if (this.displayURL) { - var urlSpec = this.item.getField('url'); - urlField.setAttribute('value', urlSpec); - urlField.setAttribute('hidden', false); - if (this.clickableLink) { - urlField.onclick = function (event) { - ZoteroPane_Local.loadURI(this.value, event) - }; - urlField.className = 'zotero-text-link'; - } - else { - urlField.className = ''; - } - urlField.hidden = false; + // URL + if (this.displayURL) { + var urlSpec = this.item.getField('url'); + urlField.setAttribute('value', urlSpec); + urlField.setAttribute('hidden', false); + if (this.clickableLink) { + urlField.onclick = function (event) { + ZoteroPane_Local.loadURI(this.value, event) + }; + urlField.className = 'zotero-text-link'; } else { - urlField.hidden = true; - } - - // Access date - if (this.displayAccessed) { - this._id("accessed-label").value = Zotero.getString('itemFields.accessDate') - + Zotero.getString('punctuation.colon'); - this._id("accessed").value = Zotero.Date.sqlToDate( - this.item.getField('accessDate'), true - ).toLocaleString(); - accessed.hidden = false; - } - else { - accessed.hidden = true; + urlField.className = ''; } + urlField.hidden = false; } - // Metadata for files else { urlField.hidden = true; - accessed.hidden = true; } - if (this.item.attachmentLinkMode - != Zotero.Attachments.LINK_MODE_LINKED_URL - && this.displayFileName) { - var fileName = this.item.getFilename(); - - if (fileName) { - this._id("fileName-label").value = Zotero.getString('pane.item.attachments.filename') - + Zotero.getString('punctuation.colon'); - this._id("fileName").value = fileName; - fileNameRow.hidden = false; - } - else { - fileNameRow.hidden = true; - } + // Access date + if (this.displayAccessed) { + this._id("accessed-label").value = Zotero.getString('itemFields.accessDate') + + Zotero.getString('punctuation.colon'); + this._id("accessed").value = Zotero.Date.sqlToDate( + this.item.getField('accessDate'), true + ).toLocaleString(); + accessed.hidden = false; + } + else { + accessed.hidden = true; + } + } + // Metadata for files + else { + urlField.hidden = true; + accessed.hidden = true; + } + + if (this.item.attachmentLinkMode + != Zotero.Attachments.LINK_MODE_LINKED_URL + && this.displayFileName) { + var fileName = this.item.attachmentFilename; + + if (fileName) { + this._id("fileName-label").value = Zotero.getString('pane.item.attachments.filename') + + Zotero.getString('punctuation.colon'); + this._id("fileName").value = fileName; + fileNameRow.hidden = false; } else { fileNameRow.hidden = true; } - - // Page count - if (this.displayPages) { - var pages = yield Zotero.Fulltext.getPages(this.item.id) - .tap(() => Zotero.Promise.check(this.item)); - var pages = pages ? pages.total : null; + } + else { + fileNameRow.hidden = true; + } + + // Page count + if (this.displayPages) { + Zotero.Fulltext.getPages(this.item.id) + .tap(() => Zotero.Promise.check(this.item)) + .then(function (pages) { + pages = pages ? pages.total : null; if (pages) { this._id("pages-label").value = Zotero.getString('itemFields.pages') + Zotero.getString('punctuation.colon'); @@ -295,77 +300,85 @@ else { pagesRow.hidden = true; } - } - else { - pagesRow.hidden = true; - } - - if (this.displayDateModified) { - this._id("dateModified-label").value = Zotero.getString('itemFields.dateModified') - + Zotero.getString('punctuation.colon'); - var mtime = yield this.item.attachmentModificationTime - .tap(() => Zotero.Promise.check(this.item)); - if (mtime) { - this._id("dateModified").value = new Date(mtime).toLocaleString(); - } - // Use the item's mod time as a backup (e.g., when sync - // passes in the mod time for the nonexistent remote file) - else { - this._id("dateModified").value = Zotero.Date.sqlToDate( - this.item.getField('dateModified'), true - ).toLocaleString(); - } + }); + } + else { + pagesRow.hidden = true; + } + + if (this.displayDateModified) { + this._id("dateModified-label").value = Zotero.getString('itemFields.dateModified') + + Zotero.getString('punctuation.colon'); + // Conflict resolution uses a modal window, so promises won't work, but + // the sync process passes in the file mod time as dateModified + if (this.synchronous) { + this._id("dateModified").value = Zotero.Date.sqlToDate( + this.item.getField('dateModified'), true + ).toLocaleString(); dateModifiedRow.hidden = false; } else { - dateModifiedRow.hidden = true; + this.item.attachmentModificationTime + .tap(() => Zotero.Promise.check(this._id)) + .then(function (mtime) { + if (!this._id) return; + if (mtime) { + this._id("dateModified").value = new Date(mtime).toLocaleString(); + } + dateModifiedRow.hidden = false; + }); } - - // Full-text index information - if (this.displayIndexed) { - yield this.updateItemIndexedState() - .tap(() => Zotero.Promise.check(this.item)); + } + else { + dateModifiedRow.hidden = true; + } + + // Full-text index information + if (this.displayIndexed) { + this.updateItemIndexedState() + .tap(() => Zotero.Promise.check(this.item)) + .then(function () { indexStatusRow.hidden = false; - } - else { - indexStatusRow.hidden = true; - } - - // Note editor - var noteEditor = this._id('attachment-note-editor'); - if (this.displayNote) { - if (this.displayNoteIfEmpty || this.item.getNote() != '') { - Zotero.debug("setting links on top"); - noteEditor.linksOnTop = true; - noteEditor.hidden = false; - - // Don't make note editable (at least for now) - if (this.mode == 'merge' || this.mode == 'mergeedit') { - noteEditor.mode = 'merge'; - noteEditor.displayButton = false; - } - else { - noteEditor.mode = this.mode; - } - noteEditor.parent = null; - noteEditor.item = this.item; - } - } - else { - noteEditor.hidden = true; - } + }); + } + else { + indexStatusRow.hidden = true; + } + + // Note editor + var noteEditor = this._id('attachment-note-editor'); + if (this.displayNote) { + if (this.displayNoteIfEmpty || this.item.getNote() != '') { + Zotero.debug("setting links on top"); + noteEditor.linksOnTop = true; + noteEditor.hidden = false; + // Don't make note editable (at least for now) + if (this.mode == 'merge' || this.mode == 'mergeedit') { + noteEditor.mode = 'merge'; + noteEditor.displayButton = false; + } + else { + noteEditor.mode = this.mode; + } + noteEditor.parent = null; + noteEditor.item = this.item; + } + } + else { + noteEditor.hidden = true; + } - if (this.displayButton) { - selectButton.label = this.buttonCaption; - selectButton.hidden = false; - selectButton.setAttribute('oncommand', - 'document.getBindingParent(this).clickHandler(this)'); - } - else { - selectButton.hidden = true; - } - }, this); + + if (this.displayButton) { + selectButton.label = this.buttonCaption; + selectButton.hidden = false; + selectButton.setAttribute('oncommand', + 'document.getBindingParent(this).clickHandler(this)'); + } + else { + selectButton.hidden = true; + } ]]> diff --git a/chrome/content/zotero/bindings/itembox.xml b/chrome/content/zotero/bindings/itembox.xml index c1ec439b3..622fd0b66 100644 --- a/chrome/content/zotero/bindings/itembox.xml +++ b/chrome/content/zotero/bindings/itembox.xml @@ -71,6 +71,7 @@ switch (val) { case 'view': + case 'merge': break; case 'edit': @@ -99,10 +100,9 @@ - - .item must be a Zotero.Item"); + throw new Error("'item' must be a Zotero.Item"); } // When changing items, reset truncation of creator list @@ -112,8 +112,7 @@ this._item = val; this.refresh(); - ]]> - + ]]> diff --git a/chrome/content/zotero/bindings/merge.xml b/chrome/content/zotero/bindings/merge.xml index 045fcca39..3cf96f722 100644 --- a/chrome/content/zotero/bindings/merge.xml +++ b/chrome/content/zotero/bindings/merge.xml @@ -99,9 +99,11 @@ } // Check for note or attachment - this.type = this._getTypeFromObject( - this._data.left.deleted ? this._data.right : this._data.left - ); + if (!this.type) { + this.type = this._getTypeFromObject( + this._data.left.deleted ? this._data.right : this._data.left + ); + } var showButton = this.type != 'item'; @@ -109,7 +111,6 @@ this._rightpane.showButton = showButton; this._leftpane.data = this._data.left; this._rightpane.data = this._data.right; - this._mergepane.type = this.type; this._mergepane.data = this._data.merge; if (this._data.selected == 'left') { @@ -313,6 +314,7 @@ break; case 'attachment': + case 'file': elementName = 'zoteroattachmentbox'; break; @@ -320,13 +322,8 @@ elementName = 'zoteronoteeditor'; break; - case 'file': - elementName = 'zoterostoragefilebox'; - break; - default: - throw ("Object type '" + this.type - + "' not supported in .ref"); + throw new Error("Object type '" + this.type + "' not supported"); } var objbox = document.createElement(elementName); @@ -342,8 +339,7 @@ objbox.setAttribute("anonid", "objectbox"); objbox.setAttribute("flex", "1"); - - objbox.mode = 'view'; + objbox.mode = this.type == 'file' ? 'filemerge' : 'merge'; var button = this._id('choose-button'); if (this.showButton) { @@ -363,7 +359,7 @@ // Create item from JSON for metadata box var item = new Zotero.Item(val.itemType); item.fromJSON(val); - objbox.ref = item; + objbox.item = item; ]]> diff --git a/chrome/content/zotero/bindings/noteeditor.xml b/chrome/content/zotero/bindings/noteeditor.xml index a0a13a43b..d9b17c03e 100644 --- a/chrome/content/zotero/bindings/noteeditor.xml +++ b/chrome/content/zotero/bindings/noteeditor.xml @@ -64,6 +64,7 @@ switch (val) { case 'view': + case 'merge': break; case 'edit': diff --git a/chrome/content/zotero/bindings/relatedbox.xml b/chrome/content/zotero/bindings/relatedbox.xml index 8ecbfba43..aa8cc30ef 100644 --- a/chrome/content/zotero/bindings/relatedbox.xml +++ b/chrome/content/zotero/bindings/relatedbox.xml @@ -109,6 +109,7 @@ ]]> + diff --git a/chrome/content/zotero/merge.js b/chrome/content/zotero/merge.js index f35acc5ce..ee527c4d4 100644 --- a/chrome/content/zotero/merge.js +++ b/chrome/content/zotero/merge.js @@ -47,13 +47,20 @@ var Zotero_Merge_Window = new function () { _wizard.getButton('cancel').setAttribute('label', Zotero.getString('sync.cancel')); - _io = window.arguments[0].wrappedJSObject; + _io = window.arguments[0]; + // Not totally clear when this is necessary + if (window.arguments[0].wrappedJSObject) { + _io = window.arguments[0].wrappedJSObject; + } _conflicts = _io.dataIn.conflicts; if (!_conflicts.length) { // TODO: handle no conflicts return; } + if (_io.dataIn.type) { + _mergeGroup.type = _io.dataIn.type; + } _mergeGroup.leftCaption = _io.dataIn.captions[0]; _mergeGroup.rightCaption = _io.dataIn.captions[1]; _mergeGroup.mergeCaption = _io.dataIn.captions[2]; @@ -240,7 +247,7 @@ var Zotero_Merge_Window = new function () { } // Apply changes from each side and pick most recent version for conflicting fields var mergeInfo = { - data: {} + data: {} }; Object.assign(mergeInfo.data, _conflicts[pos].left) Zotero.DataObjectUtilities.applyChanges(mergeInfo.data, _conflicts[pos].changes); @@ -251,7 +258,9 @@ var Zotero_Merge_Window = new function () { else { var side = 1; } - Zotero.DataObjectUtilities.applyChanges(mergeInfo.data, _conflicts[pos].conflicts.map(x => x[side])); + Zotero.DataObjectUtilities.applyChanges( + mergeInfo.data, _conflicts[pos].conflicts.map(x => x[side]) + ); mergeInfo.selected = side ? 'right' : 'left'; return mergeInfo; } @@ -284,13 +293,22 @@ var Zotero_Merge_Window = new function () { function _updateResolveAllCheckbox() { - if (_mergeGroup.rightpane.getAttribute("selected") == 'true') { - var label = 'resolveAllRemoteFields'; + if (_mergeGroup.type == 'file') { + if (_mergeGroup.rightpane.getAttribute("selected") == 'true') { + var label = 'resolveAllRemote'; + } + else { + var label = 'resolveAllLocal'; + } } else { - var label = 'resolveAllLocalFields'; + if (_mergeGroup.rightpane.getAttribute("selected") == 'true') { + var label = 'resolveAllRemoteFields'; + } + else { + var label = 'resolveAllLocalFields'; + } } - // TODO: files _resolveAllCheckbox.label = Zotero.getString('sync.conflict.' + label); } diff --git a/chrome/content/zotero/xpcom/data/item.js b/chrome/content/zotero/xpcom/data/item.js index c8d4ee8cf..b5565ac35 100644 --- a/chrome/content/zotero/xpcom/data/item.js +++ b/chrome/content/zotero/xpcom/data/item.js @@ -50,7 +50,6 @@ Zotero.Item = function(itemTypeOrID) { this._attachmentLinkMode = null; this._attachmentContentType = null; this._attachmentPath = null; - this._attachmentSyncState = null; // loadCreators this._creators = []; @@ -1453,14 +1452,13 @@ Zotero.Item.prototype._saveData = Zotero.Promise.coroutine(function* (env) { if (this._changed.attachmentData) { let sql = "REPLACE INTO itemAttachments (itemID, parentItemID, linkMode, " - + "contentType, charsetID, path, syncState) VALUES (?,?,?,?,?,?,?)"; + + "contentType, charsetID, path) VALUES (?,?,?,?,?,?)"; let linkMode = this.attachmentLinkMode; let contentType = this.attachmentContentType; let charsetID = this.attachmentCharset ? Zotero.CharacterSets.getID(this.attachmentCharset) : null; let path = this.attachmentPath; - let syncState = this.attachmentSyncState; if (linkMode == Zotero.Attachments.LINK_MODE_LINKED_FILE && libraryType != 'user') { throw new Error("Linked files can only be added to user library"); @@ -1472,8 +1470,7 @@ Zotero.Item.prototype._saveData = Zotero.Promise.coroutine(function* (env) { { int: linkMode }, contentType ? { string: contentType } : null, charsetID ? { int: charsetID } : null, - path ? { string: path } : null, - syncState ? { int: syncState } : 0 + path ? { string: path } : null ]; yield Zotero.DB.queryAsync(sql, params); @@ -2295,8 +2292,10 @@ Zotero.Item.prototype.renameAttachmentFile = Zotero.Promise.coroutine(function* yield this.relinkAttachmentFile(destPath); yield Zotero.DB.executeTransaction(function* () { - yield Zotero.Sync.Storage.setSyncedHash(this.id, null, false); - yield Zotero.Sync.Storage.setSyncState(this.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD); + yield Zotero.Sync.Storage.Local.setSyncedHash(this.id, null, false); + yield Zotero.Sync.Storage.Local.setSyncState( + this.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD + ); }.bind(this)); return true; @@ -2317,11 +2316,10 @@ Zotero.Item.prototype.renameAttachmentFile = Zotero.Promise.coroutine(function* /** * @param {string} path File path - * @param {Boolean} [skipItemUpdate] Don't update attachment item mod time, - * so that item doesn't sync. Used when a file - * needs to be renamed to be accessible but the - * user doesn't have access to modify the - * attachment metadata + * @param {Boolean} [skipItemUpdate] Don't update attachment item mod time, so that item doesn't + * sync. Used when a file needs to be renamed to be accessible but the user doesn't have + * access to modify the attachment metadata. This also allows a save when the library is + * read-only. */ Zotero.Item.prototype.relinkAttachmentFile = Zotero.Promise.coroutine(function* (path, skipItemUpdate) { if (path instanceof Components.interfaces.nsIFile) { @@ -2382,7 +2380,8 @@ Zotero.Item.prototype.relinkAttachmentFile = Zotero.Promise.coroutine(function* yield this.saveTx({ skipDateModifiedUpdate: true, - skipClientDateModifiedUpdate: skipItemUpdate + skipClientDateModifiedUpdate: skipItemUpdate, + skipEditCheck: skipItemUpdate }); return true; @@ -3606,9 +3605,6 @@ Zotero.Item.prototype.clone = Zotero.Promise.coroutine(function* (libraryID, ski if (this.attachmentPath) { newItem.attachmentPath = this.attachmentPath; } - if (this.attachmentSyncState) { - newItem.attachmentSyncState = this.attachmentSyncState; - } } } } diff --git a/chrome/content/zotero/xpcom/data/items.js b/chrome/content/zotero/xpcom/data/items.js index dacd06967..8733dd03c 100644 --- a/chrome/content/zotero/xpcom/data/items.js +++ b/chrome/content/zotero/xpcom/data/items.js @@ -84,8 +84,7 @@ Zotero.Items = function() { attachmentCharset: "CS.charset AS attachmentCharset", attachmentLinkMode: "IA.linkMode AS attachmentLinkMode", attachmentContentType: "IA.contentType AS attachmentContentType", - attachmentPath: "IA.path AS attachmentPath", - attachmentSyncState: "IA.syncState AS attachmentSyncState" + attachmentPath: "IA.path AS attachmentPath" }; } }, {lazy: true}); diff --git a/chrome/content/zotero/xpcom/file.js b/chrome/content/zotero/xpcom/file.js index b289dab54..82976e9da 100644 --- a/chrome/content/zotero/xpcom/file.js +++ b/chrome/content/zotero/xpcom/file.js @@ -48,7 +48,7 @@ Zotero.File = new function(){ else if (pathOrFile instanceof Ci.nsIFile) { return pathOrFile; } - throw new Error('Unexpected value provided to Zotero.File.pathToFile() (' + pathOrFile + ')'); + throw new Error("Unexpected value '" + pathOrFile + "'"); } @@ -348,7 +348,7 @@ Zotero.File = new function(){ * @param {String} [charset] - The character set; defaults to UTF-8 * @return {Promise} - A promise that is resolved when the file has been written */ - this.putContentsAsync = function putContentsAsync(path, data, charset) { + this.putContentsAsync = function (path, data, charset) { if (path instanceof Ci.nsIFile) { path = path.path; } @@ -424,18 +424,17 @@ Zotero.File = new function(){ * iterator when done * * The DirectoryInterator is passed as the first parameter to the generator. - * A StopIteration error will be caught automatically. * * Zotero.File.iterateDirectory(path, function* (iterator) { * while (true) { * var entry = yield iterator.next(); * [...] * } - * }).done() + * }) * * @return {Promise} */ - this.iterateDirectory = function iterateDirectory(path, generator) { + this.iterateDirectory = function (path, generator) { var iterator = new OS.File.DirectoryIterator(path); return Zotero.Promise.coroutine(generator)(iterator) .catch(function (e) { @@ -470,6 +469,8 @@ Zotero.File = new function(){ this.createShortened = function (file, type, mode, maxBytes) { + file = this.pathToFile(file); + if (!maxBytes) { maxBytes = 255; } @@ -575,6 +576,8 @@ Zotero.File = new function(){ } break; } + + return file.leafName; } @@ -902,29 +905,28 @@ Zotero.File = new function(){ this.checkFileAccessError = function (e, file, operation) { + var str = 'file.accessError.'; if (file) { - var str = Zotero.getString('file.accessError.theFile', file.path); + str += 'theFile' } else { - var str = Zotero.getString('file.accessError.aFile'); + str += 'aFile' } + str += 'CannotBe'; switch (operation) { case 'create': - var opWord = Zotero.getString('file.accessError.created'); - break; - - case 'update': - var opWord = Zotero.getString('file.accessError.updated'); + str += 'Created'; break; case 'delete': - var opWord = Zotero.getString('file.accessError.deleted'); + str += 'Deleted'; break; default: - var opWord = Zotero.getString('file.accessError.updated'); + str += 'Updated'; } + str = Zotero.getString(str, file.path ? file.path : undefined); Zotero.debug(file.path); Zotero.debug(e, 1); @@ -962,4 +964,64 @@ Zotero.File = new function(){ throw (e); } + + + this.checkPathAccessError = function (e, path, operation) { + var str = 'file.accessError.'; + if (path) { + str += 'theFile' + } + else { + str += 'aFile' + } + str += 'CannotBe'; + + switch (operation) { + case 'create': + str += 'Created'; + break; + + case 'delete': + str += 'Deleted'; + break; + + default: + str += 'Updated'; + } + str = Zotero.getString(str, path ? path : undefined); + + Zotero.debug(path); + Zotero.debug(e, 1); + Components.utils.reportError(e); + + // TODO: Check for specific errors? + if (e instanceof OS.File.Error) { + let checkFileWindows = Zotero.getString('file.accessError.message.windows'); + let checkFileOther = Zotero.getString('file.accessError.message.other'); + var msg = str + "\n\n" + + (Zotero.isWin ? checkFileWindows : checkFileOther) + + "\n\n" + + Zotero.getString('file.accessError.restart'); + + var e = new Zotero.Error( + msg, + 0, + { + dialogButtonText: Zotero.getString('file.accessError.showParentDir'), + dialogButtonCallback: function () { + try { + file.parent.QueryInterface(Components.interfaces.nsILocalFile); + file.parent.reveal(); + } + // Unsupported on some platforms + catch (e2) { + Zotero.launchFile(file.parent); + } + } + } + ); + } + + throw e; + } } diff --git a/chrome/content/zotero/xpcom/storage.js b/chrome/content/zotero/xpcom/storage.js index f382bcfb1..719bb47ed 100644 --- a/chrome/content/zotero/xpcom/storage.js +++ b/chrome/content/zotero/xpcom/storage.js @@ -33,6 +33,7 @@ Zotero.Sync.Storage = new function () { this.SYNC_STATE_IN_SYNC = 2; this.SYNC_STATE_FORCE_UPLOAD = 3; this.SYNC_STATE_FORCE_DOWNLOAD = 4; + this.SYNC_STATE_IN_CONFLICT = 5; this.SUCCESS = 1; this.ERROR_NO_URL = -1; @@ -57,14 +58,11 @@ Zotero.Sync.Storage = new function () { this.__defineGetter__("defaultError", function () Zotero.getString('sync.storage.error.default', Zotero.appName)); this.__defineGetter__("defaultErrorRestart", function () Zotero.getString('sync.storage.error.defaultRestart', Zotero.appName)); + var _itemDownloadPercentages = {}; + // // Public properties // - - - this.__defineGetter__("syncInProgress", function () _syncInProgress); - this.__defineGetter__("updatesInProgress", function () _updatesInProgress); - this.compressionTracker = { compressed: 0, uncompressed: 0, @@ -75,539 +73,6 @@ Zotero.Sync.Storage = new function () { } } - Zotero.Notifier.registerObserver(this, ['file']); - - - // - // Private properties - // - var _maxCheckAgeInSeconds = 10800; // maximum age for upload modification check (3 hours) - var _syncInProgress; - var _updatesInProgress; - var _itemDownloadPercentages = {}; - var _uploadCheckFiles = []; - var _lastFullFileCheck = {}; - - - this.sync = function (options) { - if (options.libraries) { - Zotero.debug("Starting file sync for libraries " + options.libraries); - } - else { - Zotero.debug("Starting file sync"); - } - - var self = this; - - var libraryModes = {}; - var librarySyncTimes = {}; - - // Get personal library file sync mode - return Zotero.Promise.try(function () { - // TODO: Make sure modes are active - - if (options.libraries && options.libraries.indexOf(0) == -1) { - return; - } - - if (Zotero.Sync.Storage.ZFS.includeUserFiles) { - libraryModes[0] = Zotero.Sync.Storage.ZFS; - } - else if (Zotero.Sync.Storage.WebDAV.includeUserFiles) { - libraryModes[0] = Zotero.Sync.Storage.WebDAV; - } - }) - .then(function () { - // Get group library file sync modes - if (Zotero.Sync.Storage.ZFS.includeGroupFiles) { - var groups = Zotero.Groups.getAll(); - for each(var group in groups) { - if (options.libraries && options.libraries.indexOf(group.libraryID) == -1) { - continue; - } - // TODO: if library file syncing enabled - libraryModes[group.libraryID] = Zotero.Sync.Storage.ZFS; - } - } - - // Cache auth credentials for each mode - var modes = []; - var promises = []; - for each(var mode in libraryModes) { - if (modes.indexOf(mode) == -1) { - modes.push(mode); - - // Try to verify WebDAV server first if it hasn't been - if (mode == Zotero.Sync.Storage.WebDAV - && !Zotero.Sync.Storage.WebDAV.verified) { - Zotero.debug("WebDAV file sync is not active"); - var promise = Zotero.Sync.Storage.checkServerPromise(Zotero.Sync.Storage.WebDAV) - .then(function () { - return mode.cacheCredentials(); - }); - } - else { - var promise = mode.cacheCredentials(); - } - promises.push(Zotero.Promise.allSettled([mode, promise])); - } - } - - return Zotero.Promise.all(promises) - // Get library last-sync times - .then(function (cacheCredentialsPromises) { - var promises = []; - - // Mark WebDAV verification failure as user library error. - // We ignore credentials-caching errors for ZFS and let the - // later requests fail. - cacheCredentialsPromises.forEach(function (results) { - let mode = results[0].value; - if (mode == Zotero.Sync.Storage.WebDAV) { - if (results[1].state == "rejected") { - promises.push(Zotero.Promise.allSettled( - [0, Zotero.Promise.reject(results[1].reason)] - )); - // Skip further syncing of user library - delete libraryModes[0]; - } - } - }); - - for (var libraryID in libraryModes) { - libraryID = parseInt(libraryID); - - // Get the last sync time for each library - if (self.downloadOnSync(libraryID)) { - promises.push(Zotero.Promise.allSettled( - [libraryID, libraryModes[libraryID].getLastSyncTime(libraryID)] - )); - } - // If download-as-needed, we don't need the last sync time - else { - promises.push(Zotero.Promise.allSettled([libraryID, null])); - } - } - return Zotero.Promise.all(promises); - }); - }) - .then(function (promises) { - if (!promises.length) { - Zotero.debug("No libraries are active for file sync"); - return []; - } - - var libraryQueues = []; - - // Get the libraries we have sync times for - promises.forEach(function (results) { - let libraryID = results[0].value; - let lastSyncTime = results[1].value; - if (results[1].state == "fulfilled") { - librarySyncTimes[libraryID] = lastSyncTime; - } - else { - Zotero.debug(lastSyncTime.reason); - Components.utils.reportError(lastSyncTime.reason); - // Pass rejected promise through - libraryQueues.push(results); - } - }); - - // Check for updated files to upload in each library - var promises = []; - for (let libraryID in librarySyncTimes) { - let promise; - libraryID = parseInt(libraryID); - - if (!Zotero.Libraries.isFilesEditable(libraryID)) { - Zotero.debug("No file editing access -- skipping file " - + "modification check for library " + libraryID); - continue; - } - // If this is a background sync, it's not the first sync of - // the session, the library has had at least one full check - // this session, and it's been less than _maxCheckAgeInSeconds - // since the last full check of this library, check only files - // that were previously modified or opened recently - else if (options.background - && !options.firstInSession - && _lastFullFileCheck[libraryID] - && (_lastFullFileCheck[libraryID] + (_maxCheckAgeInSeconds * 1000)) - > new Date().getTime()) { - let itemIDs = _getFilesToCheck(libraryID); - promise = self.checkForUpdatedFiles(libraryID, itemIDs); - } - // Otherwise check all files in the library - else { - _lastFullFileCheck[libraryID] = new Date().getTime(); - promise = self.checkForUpdatedFiles(libraryID); - } - promises.push(promise); - } - return Zotero.Promise.all(promises) - .then(function () { - // Queue files to download and upload from each library - for (let libraryID in librarySyncTimes) { - libraryID = parseInt(libraryID); - - var downloadAll = self.downloadOnSync(libraryID); - - // Forced downloads happen even in on-demand mode - var sql = "SELECT COUNT(*) FROM items " - + "JOIN itemAttachments USING (itemID) " - + "WHERE libraryID=? AND syncState=?"; - var downloadForced = !!Zotero.DB.valueQuery( - sql, - [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD] - ); - - // If we don't have any forced downloads, we can skip - // downloads if the last sync time hasn't changed - // or doesn't exist on the server (meaning there are no files) - if (downloadAll && !downloadForced) { - let lastSyncTime = librarySyncTimes[libraryID]; - if (lastSyncTime) { - var version = self.getStoredLastSyncTime( - libraryModes[libraryID], libraryID - ); - if (version == lastSyncTime) { - Zotero.debug("Last " + libraryModes[libraryID].name - + " sync id hasn't changed for library " - + libraryID + " -- skipping file downloads"); - downloadAll = false; - } - } - else { - Zotero.debug("No last " + libraryModes[libraryID].name - + " sync time for library " + libraryID - + " -- skipping file downloads"); - downloadAll = false; - } - } - - if (downloadAll || downloadForced) { - for each(var itemID in _getFilesToDownload(libraryID, !downloadAll)) { - var item = Zotero.Items.get(itemID); - self.queueItem(item); - } - } - - // Get files to upload - if (Zotero.Libraries.isFilesEditable(libraryID)) { - for each(var itemID in _getFilesToUpload(libraryID)) { - var item = Zotero.Items.get(itemID); - self.queueItem(item); - } - } - else { - Zotero.debug("No file editing access -- skipping file uploads for library " + libraryID); - } - } - - // Start queues for each library - for (let libraryID in librarySyncTimes) { - libraryID = parseInt(libraryID); - libraryQueues.push(Zotero.Promise.allSettled( - [libraryID, Zotero.Sync.Storage.QueueManager.start(libraryID)] - )); - } - - // The promise is done when all libraries are done - return Zotero.Promise.all(libraryQueues); - }); - }) - .then(function (promises) { - Zotero.debug('Queue manager is finished'); - - var changedLibraries = []; - var finalPromises = []; - - promises.forEach(function (results) { - var libraryID = results[0].value; - var libraryQueues = results[1].value; - - if (results[1].state == "fulfilled") { - libraryQueues.forEach(function (queuePromise) { - if (queueZotero.Promise.isFulfilled()) { - let result = queueZotero.Promise.value(); - Zotero.debug("File " + result.type + " sync finished " - + "for library " + libraryID); - if (result.localChanges) { - changedLibraries.push(libraryID); - } - finalPromises.push(Zotero.Promise.allSettled([ - libraryID, - libraryModes[libraryID].setLastSyncTime( - libraryID, - result.remoteChanges ? false : librarySyncTimes[libraryID] - ) - ])); - } - else { - let e = queueZotero.Promise.reason(); - Zotero.debug("File " + e.type + " sync failed " - + "for library " + libraryID); - finalPromises.push(Zotero.Promise.allSettled( - [libraryID, Zotero.Promise.reject(e)] - )); - } - }); - } - else { - Zotero.debug("File sync failed for library " + libraryID); - finalPromises.push([libraryID, libraryQueues]); - } - - // If WebDAV sync enabled, purge deleted and orphaned files - if (libraryID == Zotero.Libraries.userLibraryID - && Zotero.Sync.Storage.WebDAV.includeUserFiles) { - Zotero.Sync.Storage.WebDAV.purgeDeletedStorageFiles() - .then(function () { - return Zotero.Sync.Storage.WebDAV.purgeOrphanedStorageFiles(); - }) - .catch(function (e) { - Zotero.debug(e, 1); - Components.utils.reportError(e); - }); - } - }); - - Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles() - .catch(function (e) { - Zotero.debug(e, 1); - Components.utils.reportError(e); - }); - - if (promises.length && !changedLibraries.length) { - Zotero.debug("No local changes made during file sync"); - } - - return Zotero.Promise.all(finalPromises) - .then(function (promises) { - var results = { - changesMade: !!changedLibraries.length, - errors: [] - }; - - promises.forEach(function (promiseResults) { - var libraryID = promiseResults[0].value; - if (promiseResults[1].state == "rejected") { - let e = promiseResults[1].reason; - if (typeof e == 'string') { - e = new Error(e); - } - e.libraryID = libraryID; - results.errors.push(e); - } - }); - - return results; - }); - }); - } - - - // - // Public methods - // - this.queueItem = function (item, highPriority) { - var library = item.libraryID; - if (libraryID) { - var mode = Zotero.Sync.Storage.ZFS; - } - else { - var mode = Zotero.Sync.Storage.ZFS.includeUserFiles - ? Zotero.Sync.Storage.ZFS : Zotero.Sync.Storage.WebDAV; - } - switch (Zotero.Sync.Storage.getSyncState(item.id)) { - case this.SYNC_STATE_TO_DOWNLOAD: - case this.SYNC_STATE_FORCE_DOWNLOAD: - var queue = 'download'; - var callbacks = { - onStart: function (request) { - return mode.downloadFile(request); - } - }; - break; - - case this.SYNC_STATE_TO_UPLOAD: - case this.SYNC_STATE_FORCE_UPLOAD: - var queue = 'upload'; - var callbacks = { - onStart: function (request) { - return mode.uploadFile(request); - } - }; - break; - - case false: - Zotero.debug("Sync state for item " + item.id + " not found", 2); - return; - } - - var queue = Zotero.Sync.Storage.QueueManager.get(queue, library); - var request = new Zotero.Sync.Storage.Request( - item.libraryID + '/' + item.key, callbacks - ); - if (queue.type == 'upload') { - try { - request.setMaxSize(Zotero.Attachments.getTotalFileSize(item)); - } - // If this fails, ignore it, though we might fail later - catch (e) { - // But if the file doesn't exist yet, don't try to upload it - // - // This isn't a perfect test, because the file could still be - // in the process of being downloaded. It'd be better to - // download files to a temp directory and move them into place. - if (!item.getFile()) { - Zotero.debug("File " + item.libraryKey + " not yet available to upload -- skipping"); - return; - } - - Components.utils.reportError(e); - Zotero.debug(e, 1); - } - } - queue.addRequest(request, highPriority); - }; - - - this.getStoredLastSyncTime = function (mode, libraryID) { - var sql = "SELECT version FROM version WHERE schema=?"; - return Zotero.DB.valueQuery( - sql, "storage_" + mode.name.toLowerCase() + "_" + libraryID - ); - }; - - - this.setStoredLastSyncTime = function (mode, libraryID, time) { - var sql = "REPLACE INTO version SET version=? WHERE schema=?"; - Zotero.DB.query( - sql, - [ - time, - "storage_" + mode.name.toLowerCase() + "_" + libraryID - ] - ); - }; - - - /** - * @param {Integer} itemID - */ - this.getSyncState = function (itemID) { - var sql = "SELECT syncState FROM itemAttachments WHERE itemID=?"; - return Zotero.DB.valueQueryAsync(sql, itemID); - } - - - /** - * @param {Integer} itemID - * @param {Integer} syncState Constant from Zotero.Sync.Storage - */ - this.setSyncState = Zotero.Promise.method(function (itemID, syncState) { - switch (syncState) { - case this.SYNC_STATE_TO_UPLOAD: - case this.SYNC_STATE_TO_DOWNLOAD: - case this.SYNC_STATE_IN_SYNC: - case this.SYNC_STATE_FORCE_UPLOAD: - case this.SYNC_STATE_FORCE_DOWNLOAD: - break; - - default: - throw new Error("Invalid sync state '" + syncState); - } - - var sql = "UPDATE itemAttachments SET syncState=? WHERE itemID=?"; - return Zotero.DB.valueQueryAsync(sql, [syncState, itemID]); - }); - - - /** - * @param {Integer} itemID - * @return {Integer|NULL} Mod time as timestamp in ms, - * or NULL if never synced - */ - this.getSyncedModificationTime = function (itemID) { - var sql = "SELECT storageModTime FROM itemAttachments WHERE itemID=?"; - var mtime = Zotero.DB.valueQuery(sql, itemID); - if (mtime === false) { - throw "Item " + itemID + " not found in " - + "Zotero.Sync.Storage.getSyncedModificationTime()"; - } - return mtime; - } - - - /** - * @param {Integer} itemID - * @param {Integer} mtime File modification time as - * timestamp in ms - * @param {Boolean} [updateItem=FALSE] Update dateModified field of - * attachment item - */ - this.setSyncedModificationTime = function (itemID, mtime, updateItem) { - if (mtime < 0) { - Components.utils.reportError("Invalid file mod time " + mtime - + " in Zotero.Storage.setSyncedModificationTime()"); - mtime = 0; - } - - Zotero.DB.beginTransaction(); - - var sql = "UPDATE itemAttachments SET storageModTime=? WHERE itemID=?"; - Zotero.DB.valueQuery(sql, [mtime, itemID]); - - if (updateItem) { - // Update item date modified so the new mod time will be synced - var sql = "UPDATE items SET clientDateModified=? WHERE itemID=?"; - Zotero.DB.query(sql, [Zotero.DB.transactionDateTime, itemID]); - } - - Zotero.DB.commitTransaction(); - } - - - /** - * @param {Integer} itemID - * @return {Promise} - File hash, null if never synced, if false if - * file doesn't exist - */ - this.getSyncedHash = Zotero.Promise.coroutine(function* (itemID) { - var sql = "SELECT storageHash FROM itemAttachments WHERE itemID=?"; - var hash = yield Zotero.DB.valueQueryAsync(sql, itemID); - if (hash === false) { - throw new Error("Item " + itemID + " not found"); - } - return hash; - }) - - - /** - * @param {Integer} itemID - * @param {String} hash File hash - * @param {Boolean} [updateItem=FALSE] Update dateModified field of - * attachment item - */ - this.setSyncedHash = Zotero.Promise.coroutine(function* (itemID, hash, updateItem) { - if (hash !== null && hash.length != 32) { - throw ("Invalid file hash '" + hash + "' in Zotero.Storage.setSyncedHash()"); - } - - Zotero.DB.requireTransaction(); - - var sql = "UPDATE itemAttachments SET storageHash=? WHERE itemID=?"; - yield Zotero.DB.queryAsync(sql, [hash, itemID]); - - if (updateItem) { - // Update item date modified so the new mod time will be synced - var sql = "UPDATE items SET clientDateModified=? WHERE itemID=?"; - yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]); - } - }); - /** * Check if modification time of file on disk matches the mod time @@ -646,518 +111,6 @@ Zotero.Sync.Storage = new function () { } - /** - * @param {Integer|'groups'} [libraryID] - */ - this.downloadAsNeeded = function (libraryID) { - // Personal library - if (!libraryID) { - return Zotero.Prefs.get('sync.storage.downloadMode.personal') == 'on-demand'; - } - // Group library (groupID or 'groups') - else { - return Zotero.Prefs.get('sync.storage.downloadMode.groups') == 'on-demand'; - } - } - - - /** - * @param {Integer|'groups'} [libraryID] - */ - this.downloadOnSync = function (libraryID) { - // Personal library - if (!libraryID) { - return Zotero.Prefs.get('sync.storage.downloadMode.personal') == 'on-sync'; - } - // Group library (groupID or 'groups') - else { - return Zotero.Prefs.get('sync.storage.downloadMode.groups') == 'on-sync'; - } - } - - - - /** - * Scans local files and marks any that have changed for uploading - * and any that are missing for downloading - * - * @param {Integer} [libraryID] - * @param {Integer[]} [itemIDs] - * @param {Object} [itemModTimes] Item mod times indexed by item ids; - * items with stored mod times - * that differ from the provided - * time but file mod times - * matching the stored time will - * be marked for download - * @return {Promise} Promise resolving to TRUE if any items changed state, - * FALSE otherwise - */ - this.checkForUpdatedFiles = function (libraryID, itemIDs, itemModTimes) { - return Zotero.Promise.try(function () { - libraryID = parseInt(libraryID); - if (isNaN(libraryID)) { - libraryID = false; - } - - var msg = "Checking for locally changed attachment files"; - - var memmgr = Components.classes["@mozilla.org/memory-reporter-manager;1"] - .getService(Components.interfaces.nsIMemoryReporterManager); - memmgr.init(); - //Zotero.debug("Memory usage: " + memmgr.resident); - - if (libraryID !== false) { - if (itemIDs) { - if (!itemIDs.length) { - var msg = "No files to check for local changes in library " + libraryID; - Zotero.debug(msg); - return false; - } - } - if (itemModTimes) { - throw new Error("itemModTimes is not allowed when libraryID is set"); - } - - msg += " in library " + libraryID; - } - else if (itemIDs) { - throw new Error("libraryID not provided"); - } - else if (itemModTimes) { - if (!Object.keys(itemModTimes).length) { - return false; - } - msg += " in download-marking mode"; - } - else { - throw new Error("libraryID, itemIDs, or itemModTimes must be provided"); - } - Zotero.debug(msg); - - var changed = false; - - if (!itemIDs) { - itemIDs = Object.keys(itemModTimes ? itemModTimes : {}); - } - - // Can only handle a certain number of bound parameters at a time - var numIDs = itemIDs.length; - var maxIDs = Zotero.DB.MAX_BOUND_PARAMETERS - 10; - var done = 0; - var rows = []; - - Zotero.DB.beginTransaction(); - - do { - var chunk = itemIDs.splice(0, maxIDs); - var sql = "SELECT itemID, linkMode, path, storageModTime, storageHash, syncState " - + "FROM itemAttachments JOIN items USING (itemID) " - + "WHERE linkMode IN (?,?) AND syncState IN (?,?)"; - var params = []; - params.push( - Zotero.Attachments.LINK_MODE_IMPORTED_FILE, - Zotero.Attachments.LINK_MODE_IMPORTED_URL, - Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD, - Zotero.Sync.Storage.SYNC_STATE_IN_SYNC - ); - if (libraryID !== false) { - sql += " AND libraryID=?"; - params.push(libraryID); - } - if (chunk.length) { - sql += " AND itemID IN (" + chunk.map(function () '?').join() + ")"; - params = params.concat(chunk); - } - var chunkRows = Zotero.DB.query(sql, params); - if (chunkRows) { - rows = rows.concat(chunkRows); - } - done += chunk.length; - } - while (done < numIDs); - - Zotero.DB.commitTransaction(); - - // If no files, or everything is already marked for download, - // we don't need to do anything - if (!rows.length) { - var msg = "No in-sync or to-upload files found"; - if (libraryID !== false) { - msg += " in library " + libraryID; - } - Zotero.debug(msg); - return false; - } - - // Index attachment data by item id - itemIDs = []; - var attachmentData = {}; - for each(let row in rows) { - var id = row.itemID; - itemIDs.push(id); - attachmentData[id] = { - linkMode: row.linkMode, - path: row.path, - mtime: row.storageModTime, - hash: row.storageHash, - state: row.syncState - }; - } - rows = null; - - var t = new Date(); - var items = Zotero.Items.get(itemIDs); - var numItems = items.length; - var updatedStates = {}; - - let checkItems = function () { - if (!items.length) return Zotero.Promise.resolve(); - - //Zotero.debug("Memory usage: " + memmgr.resident); - - let item = items.shift(); - let row = attachmentData[item.id]; - let lk = item.libraryKey; - Zotero.debug("Checking attachment file for item " + lk); - - let nsIFile = item.getFile(row, true); - if (!nsIFile) { - Zotero.debug("Marking pathless attachment " + lk + " as in-sync"); - updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_IN_SYNC; - return checkItems(); - } - let file = null; - return Zotero.Promise.resolve(OS.File.open(nsIFile.path)) - .then(function (promisedFile) { - file = promisedFile; - return file.stat() - .then(function (info) { - //Zotero.debug("Memory usage: " + memmgr.resident); - - var fmtime = info.lastModificationDate.getTime(); - //Zotero.debug("File modification time for item " + lk + " is " + fmtime); - - if (fmtime < 1) { - Zotero.debug("File mod time " + fmtime + " is less than 1 -- interpreting as 1", 2); - fmtime = 1; - } - - // If file is already marked for upload, skip check. Even if this - // is download-marking mode (itemModTimes) and the file was - // changed remotely, conflicts are checked at upload time, so we - // don't need to worry about it here. - if (row.state == Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD) { - return; - } - - //Zotero.debug("Stored mtime is " + row.mtime); - //Zotero.debug("File mtime is " + fmtime); - - // Download-marking mode - if (itemModTimes) { - Zotero.debug("Remote mod time for item " + lk + " is " + itemModTimes[item.id]); - - // Ignore attachments whose stored mod times haven't changed - if (row.storageModTime == itemModTimes[item.id]) { - Zotero.debug("Storage mod time (" + row.storageModTime + ") " - + "hasn't changed for item " + lk); - return; - } - - Zotero.debug("Marking attachment " + lk + " for download " - + "(stored mtime: " + itemModTimes[item.id] + ")"); - updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD; - } - - var mtime = row.mtime; - - // If stored time matches file, it hasn't changed locally - if (mtime == fmtime) { - return; - } - - // Allow floored timestamps for filesystems that don't support - // millisecond precision (e.g., HFS+) - if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) { - Zotero.debug("File mod times are within one-second precision " - + "(" + fmtime + " ≅ " + mtime + ") for " + file.leafName - + " for item " + lk + " -- ignoring"); - return; - } - - // Allow timestamp to be exactly one hour off to get around - // time zone issues -- there may be a proper way to fix this - if (Math.abs(fmtime - mtime) == 3600000 - // And check with one-second precision as well - || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000 - || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) { - Zotero.debug("File mod time (" + fmtime + ") is exactly one " - + "hour off remote file (" + mtime + ") for item " + lk - + "-- assuming time zone issue and skipping upload"); - return; - } - - // If file hash matches stored hash, only the mod time changed, so skip - return Zotero.Utilities.Internal.md5Async(file) - .then(function (fileHash) { - if (row.hash && row.hash == fileHash) { - // We have to close the file before modifying it from the main - // thread (at least on Windows, where assigning lastModifiedTime - // throws an NS_ERROR_FILE_IS_LOCKED otherwise) - return Zotero.Promise.resolve(file.close()) - .then(function () { - Zotero.debug("Mod time didn't match (" + fmtime + "!=" + mtime + ") " - + "but hash did for " + nsIFile.leafName + " for item " + lk - + " -- updating file mod time"); - try { - nsIFile.lastModifiedTime = row.mtime; - } - catch (e) { - Zotero.File.checkFileAccessError(e, nsIFile, 'update'); - } - }); - } - - // Mark file for upload - Zotero.debug("Marking attachment " + lk + " as changed " - + "(" + mtime + " != " + fmtime + ")"); - updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD; - }); - }); - }) - .finally(function () { - if (file) { - //Zotero.debug("Closing file for item " + lk); - file.close(); - } - }) - .catch(function (e) { - if (e instanceof OS.File.Error && - (e.becauseNoSuchFile - // This can happen if a path is too long on Windows, - // e.g. a file is being accessed on a VM through a share - // (and probably in other cases). - || (e.winLastError && e.winLastError == 3) - // Handle long filenames on OS X/Linux - || (e.unixErrno && e.unixErrno == 63))) { - Zotero.debug("Marking attachment " + lk + " as missing"); - updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD; - return; - } - - if (e instanceof OS.File.Error) { - if (e.becauseClosed) { - Zotero.debug("File was closed", 2); - } - Zotero.debug(e); - Zotero.debug(e.toString()); - throw new Error("Error for operation '" + e.operation + "' for " + nsIFile.path); - } - - throw e; - }) - .then(function () { - return checkItems(); - }); - }; - - return checkItems() - .then(function () { - for (let itemID in updatedStates) { - Zotero.Sync.Storage.setSyncState(itemID, updatedStates[itemID]); - changed = true; - } - - if (!changed) { - Zotero.debug("No synced files have changed locally"); - } - - let msg = "Checked " + numItems + " files in "; - if (libraryID !== false) { - msg += "library " + libraryID + " in "; - } - msg += (new Date() - t) + "ms"; - Zotero.debug(msg); - - return changed; - }); - }); - }; - - - /** - * Download a single file - * - * If no queue is active, start one. Otherwise, add to existing queue. - */ - this.downloadFile = function (item, requestCallbacks) { - var itemID = item.id; - var mode = getModeFromLibrary(item.libraryID); - - // TODO: verify WebDAV on-demand? - if (!mode || !mode.verified) { - Zotero.debug("File syncing is not active for item's library -- skipping download"); - return false; - } - - if (!item.isImportedAttachment()) { - throw new Error("Not an imported attachment"); - } - - if (item.getFile()) { - Zotero.debug("File already exists -- replacing"); - } - - // TODO: start sync icon in cacheCredentials - return Zotero.Promise.try(function () { - return mode.cacheCredentials(); - }) - .then(function () { - // TODO: start sync icon - var library = item.libraryID; - var queue = Zotero.Sync.Storage.QueueManager.get( - 'download', library - ); - - if (!requestCallbacks) { - requestCallbacks = {}; - } - var onStart = function (request) { - return mode.downloadFile(request); - }; - requestCallbacks.onStart = requestCallbacks.onStart - ? [onStart, requestCallbacks.onStart] - : onStart; - - var request = new Zotero.Sync.Storage.Request( - library + '/' + item.key, requestCallbacks - ); - - queue.addRequest(request, true); - queue.start(); - - return request.promise; - }); - } - - - /** - * Extract a downloaded file and update the database metadata - * - * This is called from Zotero.Sync.Server.StreamListener.onStopRequest() - * - * @return {Promise} data - Promise for object with properties 'request', 'item', - * 'compressed', 'syncModTime', 'syncHash' - */ - this.processDownload = Zotero.Promise.coroutine(function* (data) { - var funcName = "Zotero.Sync.Storage.processDownload()"; - - if (!data) { - throw "'data' not set in " + funcName; - } - - if (!data.item) { - throw "'data.item' not set in " + funcName; - } - - if (!data.syncModTime) { - throw "'data.syncModTime' not set in " + funcName; - } - - if (!data.compressed && !data.syncHash) { - throw "'data.syncHash' is required if 'data.compressed' is false in " + funcName; - } - - var item = data.item; - var syncModTime = data.syncModTime; - var syncHash = data.syncHash; - - // TODO: Test file hash - - if (data.compressed) { - var newFile = yield _processZipDownload(item); - } - else { - var newFile = yield _processDownload(item); - } - - // If |newFile| is set, the file was renamed, so set item filename to that - // and mark for updated - var file = item.getFile(); - if (newFile && file.leafName != newFile.leafName) { - // Bypass library access check - _updatesInProgress = true; - - // If library isn't editable but filename was changed, update - // database without updating the item's mod time, which would result - // in a library access error - if (!Zotero.Items.isEditable(item)) { - Zotero.debug("File renamed without library access -- " - + "updating itemAttachments path", 3); - item.relinkAttachmentFile(newFile, true); - var useCurrentModTime = false; - } - else { - item.relinkAttachmentFile(newFile); - - // TODO: use an integer counter instead of mod time for change detection - var useCurrentModTime = true; - } - - file = item.getFile(); - _updatesInProgress = false; - } - else { - var useCurrentModTime = false; - } - - if (!file) { - // This can happen if an HTML snapshot filename was changed and synced - // elsewhere but the renamed file wasn't synced, so the ZIP doesn't - // contain a file with the known name - var missingFile = item.getFile(null, true); - Components.utils.reportError("File '" + missingFile.leafName - + "' not found after processing download " - + item.libraryID + "/" + item.key + " in " + funcName); - return false; - } - - Zotero.DB.beginTransaction(); - - //var syncState = Zotero.Sync.Storage.getSyncState(item.id); - //var updateItem = syncState != this.SYNC_STATE_TO_DOWNLOAD; - var updateItem = false; - - try { - if (useCurrentModTime) { - file.lastModifiedTime = new Date(); - - // Reset hash and sync state - Zotero.Sync.Storage.setSyncedHash(item.id, null); - Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD); - this.queueItem(item); - } - else { - file.lastModifiedTime = syncModTime; - // If hash not provided (e.g., WebDAV), calculate it now - if (!syncHash) { - syncHash = item.attachmentHash; - } - Zotero.Sync.Storage.setSyncedHash(item.id, syncHash); - Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); - } - } - catch (e) { - Zotero.File.checkFileAccessError(e, file, 'update'); - } - - Zotero.Sync.Storage.setSyncedModificationTime(item.id, syncModTime, updateItem); - Zotero.DB.commitTransaction(); - - return true; - }); - - this.checkServerPromise = function (mode) { return mode.checkServer() .spread(function (uri, status) { @@ -1277,570 +230,16 @@ Zotero.Sync.Storage = new function () { } - this.getItemFromRequestName = function (name) { - var [libraryID, key] = name.split('/'); - return Zotero.Items.getByLibraryAndKey(libraryID, key); - } - - - this.notify = function(event, type, ids, extraData) { - if (event == 'open' && type == 'file') { - let timestamp = new Date().getTime(); - - for each(let id in ids) { - _uploadCheckFiles.push({ - itemID: id, - timestamp: timestamp - }); - } - } - } - - - // - // Private methods - // - function getModeFromLibrary(libraryID) { - if (libraryID === undefined) { - throw new Error("libraryID not provided"); - } - - // Personal library - if (!libraryID) { - if (Zotero.Sync.Storage.ZFS.includeUserFiles) { - return Zotero.Sync.Storage.ZFS; - } - if (Zotero.Sync.Storage.WebDAV.includeUserFiles) { - return Zotero.Sync.Storage.WebDAV; - } - return false; - } - - // Group library - else { - if (Zotero.Sync.Storage.ZFS.includeGroupFiles) { - return Zotero.Sync.Storage.ZFS; - } - return false; - } - } - - - var _processDownload = Zotero.Promise.coroutine(function* (item) { - var funcName = "Zotero.Sync.Storage._processDownload()"; - - var tempFile = Zotero.getTempDirectory(); - tempFile.append(item.key + '.tmp'); - - if (!tempFile.exists()) { - Zotero.debug(tempFile.path); - throw ("Downloaded file not found in " + funcName); - } - - var parentDir = Zotero.Attachments.getStorageDirectory(item); - if (!parentDir.exists()) { - yield Zotero.Attachments.createDirectoryForItem(item); - } - - _deleteExistingAttachmentFiles(item); - - var file = item.getFile(null, true); - if (!file) { - throw ("Empty path for item " + item.key + " in " + funcName); - } - // Don't save Windows aliases - if (file.leafName.endsWith('.lnk')) { - return false; - } - - var fileName = file.leafName; - var renamed = false; - - // Make sure the new filename is valid, in case an invalid character made it over - // (e.g., from before we checked for them) - var filteredName = Zotero.File.getValidFileName(fileName); - if (filteredName != fileName) { - Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'"); - fileName = filteredName; - file.leafName = fileName; - renamed = true; - } - - Zotero.debug("Moving download file " + tempFile.leafName + " into attachment directory as '" + fileName + "'"); - try { - var destFile = parentDir.clone(); - destFile.append(fileName); - Zotero.File.createShortened(destFile, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644); - } - catch (e) { - Zotero.File.checkFileAccessError(e, destFile, 'create'); - } - - if (destFile.leafName != fileName) { - Zotero.debug("Changed filename '" + fileName + "' to '" + destFile.leafName + "'"); - - // Abort if Windows path limitation would cause filenames to be overly truncated - if (Zotero.isWin && destFile.leafName.length < 40) { - try { - destFile.remove(false); - } - catch (e) {} - // TODO: localize - var msg = "Due to a Windows path length limitation, your Zotero data directory " - + "is too deep in the filesystem for syncing to work reliably. " - + "Please relocate your Zotero data to a higher directory."; - Zotero.debug(msg, 1); - throw new Error(msg); - } - - renamed = true; - } - - try { - tempFile.moveTo(parentDir, destFile.leafName); - } - catch (e) { - try { - destFile.remove(false); - } - catch (e) {} - - Zotero.File.checkFileAccessError(e, destFile, 'create'); - } - - var returnFile = null; - // processDownload() needs to know that we're renaming the file - if (renamed) { - returnFile = destFile.clone(); - } - return returnFile; - }); - - - var _processZipDownload = Zotero.Promise.coroutine(function* (item) { - var funcName = "Zotero.Sync.Storage._processDownloadedZip()"; - - var zipFile = Zotero.getTempDirectory(); - zipFile.append(item.key + '.zip.tmp'); - - if (!zipFile.exists()) { - throw ("Downloaded ZIP file not found in " + funcName); - } - - var zipReader = Components.classes["@mozilla.org/libjar/zip-reader;1"]. - createInstance(Components.interfaces.nsIZipReader); - try { - zipReader.open(zipFile); - zipReader.test(null); - - Zotero.debug("ZIP file is OK"); - } - catch (e) { - Zotero.debug(zipFile.leafName + " is not a valid ZIP file", 2); - zipReader.close(); - - try { - zipFile.remove(false); - } - catch (e) { - Zotero.File.checkFileAccessError(e, zipFile, 'delete'); - } - - // TODO: Remove prop file to trigger reuploading, in case it was an upload error? - - return false; - } - - var parentDir = Zotero.Attachments.getStorageDirectory(item); - if (!parentDir.exists()) { - yield Zotero.Attachments.createDirectoryForItem(item); - } - - try { - _deleteExistingAttachmentFiles(item); - } - catch (e) { - zipReader.close(); - throw (e); - } - - var returnFile = null; - var count = 0; - - var entries = zipReader.findEntries(null); - while (entries.hasMore()) { - count++; - var entryName = entries.getNext(); - var b64re = /%ZB64$/; - if (entryName.match(b64re)) { - var fileName = Zotero.Utilities.Internal.Base64.decode( - entryName.replace(b64re, '') - ); - } - else { - var fileName = entryName; - } - - if (fileName.startsWith('.zotero')) { - Zotero.debug("Skipping " + fileName); - continue; - } - - Zotero.debug("Extracting " + fileName); - - var primaryFile = false; - var filtered = false; - var renamed = false; - - // Get the old filename - var itemFileName = item.getFilename(); - - // Make sure the new filename is valid, in case an invalid character - // somehow make it into the ZIP (e.g., from before we checked for them) - // - // Do this before trying to use the relative descriptor, since otherwise - // it might fail silently and select the parent directory - var filteredName = Zotero.File.getValidFileName(fileName); - if (filteredName != fileName) { - Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'"); - fileName = filteredName; - filtered = true; - } - - // Name in ZIP is a relative descriptor, so file has to be reconstructed - // using setRelativeDescriptor() - var destFile = parentDir.clone(); - destFile.QueryInterface(Components.interfaces.nsILocalFile); - destFile.setRelativeDescriptor(parentDir, fileName); - - fileName = destFile.leafName; - - // If only one file in zip and it doesn't match the known filename, - // take our chances and use that name - if (count == 1 && !entries.hasMore() && itemFileName) { - // May not be necessary, but let's be safe - itemFileName = Zotero.File.getValidFileName(itemFileName); - if (itemFileName != fileName) { - Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'", 2); - Components.utils.reportError("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'"); - fileName = itemFileName; - destFile.leafName = fileName; - renamed = true; - } - } - - var primaryFile = itemFileName == fileName; - if (primaryFile && filtered) { - renamed = true; - } - - if (destFile.exists()) { - var msg = "ZIP entry '" + fileName + "' " + "already exists"; - Zotero.debug(msg, 2); - Components.utils.reportError(msg + " in " + funcName); - continue; - } - - try { - Zotero.File.createShortened(destFile, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644); - } - catch (e) { - Zotero.debug(e, 1); - Components.utils.reportError(e); - - zipReader.close(); - - Zotero.File.checkFileAccessError(e, destFile, 'create'); - } - - if (destFile.leafName != fileName) { - Zotero.debug("Changed filename '" + fileName + "' to '" + destFile.leafName + "'"); - - // Abort if Windows path limitation would cause filenames to be overly truncated - if (Zotero.isWin && destFile.leafName.length < 40) { - try { - destFile.remove(false); - } - catch (e) {} - zipReader.close(); - // TODO: localize - var msg = "Due to a Windows path length limitation, your Zotero data directory " - + "is too deep in the filesystem for syncing to work reliably. " - + "Please relocate your Zotero data to a higher directory."; - Zotero.debug(msg, 1); - throw new Error(msg); - } - - if (primaryFile) { - renamed = true; - } - } - - try { - zipReader.extract(entryName, destFile); - } - catch (e) { - try { - destFile.remove(false); - } - catch (e) {} - - // For advertising junk files, ignore a bug on Windows where - // destFile.create() works but zipReader.extract() doesn't - // when the path length is close to 255. - if (destFile.leafName.match(/[a-zA-Z0-9+=]{130,}/)) { - var msg = "Ignoring error extracting '" + destFile.path + "'"; - Zotero.debug(msg, 2); - Zotero.debug(e, 2); - Components.utils.reportError(msg + " in " + funcName); - continue; - } - - zipReader.close(); - - Zotero.File.checkFileAccessError(e, destFile, 'create'); - } - - destFile.permissions = 0644; - - // If we're renaming the main file, processDownload() needs to know - if (renamed) { - returnFile = destFile; - } - } - zipReader.close(); - zipFile.remove(false); - - return returnFile; - }); - - - function _deleteExistingAttachmentFiles(item) { - var funcName = "Zotero.Sync.Storage._deleteExistingAttachmentFiles()"; - - var parentDir = Zotero.Attachments.getStorageDirectory(item); - - // Delete existing files - var otherFiles = parentDir.directoryEntries; - otherFiles.QueryInterface(Components.interfaces.nsIDirectoryEnumerator); - var filesToDelete = []; - var file; - while (file = otherFiles.nextFile) { - if (file.leafName.startsWith('.zotero')) { - continue; - } - - // Check symlink awareness, just to be safe - if (!parentDir.contains(file, false)) { - var msg = "Storage directory doesn't contain '" + file.leafName + "'"; - Zotero.debug(msg, 2); - Components.utils.reportError(msg + " in " + funcName); - continue; - } - - filesToDelete.push(file); - } - otherFiles.close(); - - // Do deletes outside of the enumerator to avoid an access error on Windows - for each(var file in filesToDelete) { - try { - if (file.isFile()) { - Zotero.debug("Deleting existing file " + file.leafName); - file.remove(false); - } - else if (file.isDirectory()) { - Zotero.debug("Deleting existing directory " + file.leafName); - file.remove(true); - } - } - catch (e) { - Zotero.File.checkFileAccessError(e, file, 'delete'); - } - } - } - - - /** - * Create zip file of attachment directory - * - * @param {Zotero.Sync.Storage.Request} request - * @param {Function} callback - * @return {Boolean} TRUE if zip process started, - * FALSE if storage was empty - */ - this.createUploadFile = function (request, callback) { - var item = Zotero.Sync.Storage.getItemFromRequestName(request.name); - Zotero.debug("Creating zip file for item " + item.libraryID + "/" + item.key); - - try { - switch (item.attachmentLinkMode) { - case Zotero.Attachments.LINK_MODE_LINKED_FILE: - case Zotero.Attachments.LINK_MODE_LINKED_URL: - throw (new Error( - "Upload file must be an imported snapshot or file in " - + "Zotero.Sync.Storage.createUploadFile()" - )); - } - - var dir = Zotero.Attachments.getStorageDirectory(item); - - var tmpFile = Zotero.getTempDirectory(); - tmpFile.append(item.key + '.zip'); - - var zw = Components.classes["@mozilla.org/zipwriter;1"] - .createInstance(Components.interfaces.nsIZipWriter); - zw.open(tmpFile, 0x04 | 0x08 | 0x20); // open rw, create, truncate - var fileList = _zipDirectory(dir, dir, zw); - if (fileList.length == 0) { - Zotero.debug('No files to add -- removing zip file'); - zw.close(); - tmpFile.remove(null); - return false; - } - - Zotero.debug('Creating ' + tmpFile.leafName + ' with ' + fileList.length + ' file(s)'); - - var observer = new Zotero.Sync.Storage.ZipWriterObserver( - zw, callback, { request: request, files: fileList } - ); - zw.processQueue(observer, null); - return true; - } - // DEBUG: Do we want to catch this? - catch (e) { - Zotero.debug(e, 1); - Components.utils.reportError(e); - return false; - } - } - - function _zipDirectory(rootDir, dir, zipWriter) { - var fileList = []; - dir = dir.directoryEntries; - while (dir.hasMoreElements()) { - var file = dir.getNext(); - file.QueryInterface(Components.interfaces.nsILocalFile); - if (file.isDirectory()) { - //Zotero.debug("Recursing into directory " + file.leafName); - fileList.concat(_zipDirectory(rootDir, file, zipWriter)); - continue; - } - var fileName = file.getRelativeDescriptor(rootDir); - if (fileName.startsWith('.zotero')) { - Zotero.debug('Skipping file ' + fileName); - continue; - } - - //Zotero.debug("Adding file " + fileName); - - zipWriter.addEntryFile( - fileName, - Components.interfaces.nsIZipWriter.COMPRESSION_DEFAULT, - file, - true - ); - fileList.push(fileName); - } - return fileList; - } - /** - * Get files marked as ready to download - * - * @inner - * @return {Number[]} Array of attachment itemIDs - */ - function _getFilesToDownload(libraryID, forcedOnly) { - var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) " - + "WHERE libraryID=? AND syncState IN (?"; - var params = [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD]; - if (!forcedOnly) { - sql += ",?"; - params.push(Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD); - } - sql += ") " - // Skip attachments with empty path, which can't be saved, and files with .zotero* - // paths, which have somehow ended up in some users' libraries - + "AND path!='' AND path NOT LIKE 'storage:.zotero%'"; - var itemIDs = Zotero.DB.columnQuery(sql, params); - if (!itemIDs) { - return []; - } - return itemIDs; - } - /** - * Get files marked as ready to upload - * - * @inner - * @return {Number[]} Array of attachment itemIDs - */ - function _getFilesToUpload(libraryID) { - var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) " - + "WHERE syncState IN (?,?) AND linkMode IN (?,?)"; - var params = [ - Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD, - Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD, - Zotero.Attachments.LINK_MODE_IMPORTED_FILE, - Zotero.Attachments.LINK_MODE_IMPORTED_URL - ]; - if (typeof libraryID != 'undefined') { - sql += " AND libraryID=?"; - params.push(libraryID); - } - else { - throw new Error("libraryID not specified"); - } - var itemIDs = Zotero.DB.columnQuery(sql, params); - if (!itemIDs) { - return []; - } - return itemIDs; - } - /** - * Get files to check for local modifications for uploading - * - * This includes files previously modified and files opened externally - * via Zotero within _maxCheckAgeInSeconds. - */ - function _getFilesToCheck(libraryID) { - var minTime = new Date().getTime() - (_maxCheckAgeInSeconds * 1000); - - // Get files by modification time - var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) " - + "WHERE libraryID=? AND linkMode IN (?,?) AND syncState IN (?) AND " - + "storageModTime>=?"; - var params = [ - libraryID, - Zotero.Attachments.LINK_MODE_IMPORTED_FILE, - Zotero.Attachments.LINK_MODE_IMPORTED_URL, - Zotero.Sync.Storage.SYNC_STATE_IN_SYNC, - minTime - ]; - var itemIDs = Zotero.DB.columnQuery(sql, params) || []; - - // Get files by open time - _uploadCheckFiles.filter(function (x) x.timestamp >= minTime); - itemIDs = itemIDs.concat([x.itemID for each(x in _uploadCheckFiles)]) - - return Zotero.Utilities.arrayUnique(itemIDs); - } - /** - * @inner - * @return {String[]|FALSE} Array of keys, or FALSE if none - */ - this.getDeletedFiles = function () { - var sql = "SELECT key FROM storageDeleteLog"; - return Zotero.DB.columnQuery(sql); - } + function error(e) { @@ -1892,56 +291,4 @@ Zotero.Sync.Storage = new function () { } -/** - * Request observer for zip writing - * - * Implements nsIRequestObserver - * - * @param {nsIZipWriter} zipWriter - * @param {Function} callback - * @param {Object} data - */ -Zotero.Sync.Storage.ZipWriterObserver = function (zipWriter, callback, data) { - this._zipWriter = zipWriter; - this._callback = callback; - this._data = data; -} -Zotero.Sync.Storage.ZipWriterObserver.prototype = { - onStartRequest: function () {}, - - onStopRequest: function(req, context, status) { - var zipFileName = this._zipWriter.file.leafName; - - var originalSize = 0; - for each(var fileName in this._data.files) { - var entry = this._zipWriter.getEntry(fileName); - if (!entry) { - var msg = "ZIP entry '" + fileName + "' not found for request '" + this._data.request.name + "'"; - Components.utils.reportError(msg); - Zotero.debug(msg, 1); - this._zipWriter.close(); - this._callback(false); - return; - } - originalSize += entry.realSize; - } - delete this._data.files; - - this._zipWriter.close(); - - Zotero.debug("Zip of " + zipFileName + " finished with status " + status - + " (original " + Math.round(originalSize / 1024) + "KB, " - + "compressed " + Math.round(this._zipWriter.file.fileSize / 1024) + "KB, " - + Math.round( - ((originalSize - this._zipWriter.file.fileSize) / originalSize) * 100 - ) + "% reduction)"); - - Zotero.Sync.Storage.compressionTracker.compressed += this._zipWriter.file.fileSize; - Zotero.Sync.Storage.compressionTracker.uncompressed += originalSize; - Zotero.debug("Average compression so far: " - + Math.round(Zotero.Sync.Storage.compressionTracker.ratio * 100) + "%"); - - this._callback(this._data); - } -} diff --git a/chrome/content/zotero/xpcom/storage/mode.js b/chrome/content/zotero/xpcom/storage/mode.js deleted file mode 100644 index 472b6a101..000000000 --- a/chrome/content/zotero/xpcom/storage/mode.js +++ /dev/null @@ -1,87 +0,0 @@ -/* - ***** BEGIN LICENSE BLOCK ***** - - Copyright © 2009 Center for History and New Media - George Mason University, Fairfax, Virginia, USA - http://zotero.org - - This file is part of Zotero. - - Zotero is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Zotero is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with Zotero. If not, see . - - ***** END LICENSE BLOCK ***** -*/ - - -Zotero.Sync.Storage.Mode = function () {}; - -Zotero.Sync.Storage.Mode.prototype.__defineGetter__('verified', function () { - return this._verified; -}); - -Zotero.Sync.Storage.Mode.prototype.__defineGetter__('username', function () { - return this._username; -}); - -Zotero.Sync.Storage.Mode.prototype.__defineGetter__('password', function () { - return this._password; -}); - -Zotero.Sync.Storage.Mode.prototype.__defineSetter__('password', function (val) { - this._password = val; -}); - -Zotero.Sync.Storage.Mode.prototype.init = function () { - return this._init(); -} - -Zotero.Sync.Storage.Mode.prototype.sync = function (observer) { - return Zotero.Sync.Storage.sync(this.name, observer); -} - -Zotero.Sync.Storage.Mode.prototype.downloadFile = function (request) { - return this._downloadFile(request); -} - -Zotero.Sync.Storage.Mode.prototype.uploadFile = function (request) { - return this._uploadFile(request); -} - -Zotero.Sync.Storage.Mode.prototype.getLastSyncTime = function (libraryID) { - return this._getLastSyncTime(libraryID); -} - -Zotero.Sync.Storage.Mode.prototype.setLastSyncTime = function (callback, useLastSyncTime) { - return this._setLastSyncTime(callback, useLastSyncTime); -} - -Zotero.Sync.Storage.Mode.prototype.checkServer = function (callback) { - return this._checkServer(callback); -} - -Zotero.Sync.Storage.Mode.prototype.checkServerCallback = function (uri, status, window, skipSuccessMessage) { - return this._checkServerCallback(uri, status, window, skipSuccessMessage); -} - -Zotero.Sync.Storage.Mode.prototype.cacheCredentials = function () { - return this._cacheCredentials(); -} - -Zotero.Sync.Storage.Mode.prototype.purgeDeletedStorageFiles = function (callback) { - return this._purgeDeletedStorageFiles(callback); -} - -Zotero.Sync.Storage.Mode.prototype.purgeOrphanedStorageFiles = function (callback) { - return this._purgeOrphanedStorageFiles(callback); -} diff --git a/chrome/content/zotero/xpcom/storage/queue.js b/chrome/content/zotero/xpcom/storage/queue.js deleted file mode 100644 index 25a399d72..000000000 --- a/chrome/content/zotero/xpcom/storage/queue.js +++ /dev/null @@ -1,427 +0,0 @@ -/* - ***** BEGIN LICENSE BLOCK ***** - - Copyright © 2009 Center for History and New Media - George Mason University, Fairfax, Virginia, USA - http://zotero.org - - This file is part of Zotero. - - Zotero is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Zotero is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with Zotero. If not, see . - - ***** END LICENSE BLOCK ***** -*/ - -/** - * Queue for storage sync transfer requests - * - * @param {String} type Queue type (e.g., 'download' or 'upload') - */ -Zotero.Sync.Storage.Queue = function (type, libraryID) { - Zotero.debug("Initializing " + type + " queue for library " + libraryID); - - // Public properties - this.type = type; - this.libraryID = libraryID; - this.maxConcurrentRequests = 1; - this.activeRequests = 0; - this.totalRequests = 0; - - // Private properties - this._requests = {}; - this._highPriority = []; - this._running = false; - this._stopping = false; - this._finished = false; - this._error = false; - this._finishedReqs = 0; - this._localChanges = false; - this._remoteChanges = false; - this._conflicts = []; - this._cachedPercentage; - this._cachedPercentageTime; -} - -Zotero.Sync.Storage.Queue.prototype.__defineGetter__('name', function () { - return this.type + "/" + this.libraryID; -}); - -Zotero.Sync.Storage.Queue.prototype.__defineGetter__('Type', function () { - return this.type[0].toUpperCase() + this.type.substr(1); -}); - -Zotero.Sync.Storage.Queue.prototype.__defineGetter__('running', function () this._running); -Zotero.Sync.Storage.Queue.prototype.__defineGetter__('stopping', function () this._stopping); -Zotero.Sync.Storage.Queue.prototype.__defineGetter__('finished', function () this._finished); - -Zotero.Sync.Storage.Queue.prototype.__defineGetter__('unfinishedRequests', function () { - return this.totalRequests - this.finishedRequests; -}); - -Zotero.Sync.Storage.Queue.prototype.__defineGetter__('finishedRequests', function () { - return this._finishedReqs; -}); - -Zotero.Sync.Storage.Queue.prototype.__defineSetter__('finishedRequests', function (val) { - Zotero.debug("Finished requests: " + val); - Zotero.debug("Total requests: " + this.totalRequests); - - this._finishedReqs = val; - - if (val == 0) { - return; - } - - // Last request - if (val == this.totalRequests) { - Zotero.debug(this.Type + " queue is done for library " + this.libraryID); - - // DEBUG info - Zotero.debug("Active requests: " + this.activeRequests); - - if (this.activeRequests) { - throw new Error(this.Type + " queue for library " + this.libraryID - + " can't be done if there are active requests"); - } - - this._running = false; - this._stopping = false; - this._finished = true; - this._requests = {}; - this._highPriority = []; - - var localChanges = this._localChanges; - var remoteChanges = this._remoteChanges; - var conflicts = this._conflicts.concat(); - var deferred = this._deferred; - this._localChanges = false; - this._remoteChanges = false; - this._conflicts = []; - this._deferred = null; - - if (!this._error) { - Zotero.debug("Resolving promise for queue " + this.name); - Zotero.debug(this._localChanges); - Zotero.debug(this._remoteChanges); - Zotero.debug(this._conflicts); - - deferred.resolve({ - libraryID: this.libraryID, - type: this.type, - localChanges: localChanges, - remoteChanges: remoteChanges, - conflicts: conflicts - }); - } - else { - Zotero.debug("Rejecting promise for queue " + this.name); - var e = this._error; - this._error = false; - e.libraryID = this.libraryID; - e.type = this.type; - deferred.reject(e); - } - - return; - } - - if (this._stopping) { - return; - } - this.advance(); -}); - -Zotero.Sync.Storage.Queue.prototype.__defineGetter__('queuedRequests', function () { - return this.unfinishedRequests - this.activeRequests; -}); - -Zotero.Sync.Storage.Queue.prototype.__defineGetter__('remaining', function () { - var remaining = 0; - for each(var request in this._requests) { - remaining += request.remaining; - } - return remaining; -}); - -Zotero.Sync.Storage.Queue.prototype.__defineGetter__('percentage', function () { - if (this.totalRequests == 0) { - return 0; - } - if (this._finished) { - return 100; - } - - // Cache percentage for a second - if (this._cachedPercentage && (new Date() - this._cachedPercentageTime) < 1000) { - return this._cachedPercentage; - } - - var completedRequests = 0; - for each(var request in this._requests) { - completedRequests += request.percentage / 100; - } - this._cachedPercentage = Math.round((completedRequests / this.totalRequests) * 100); - this._cachedPercentageTime = new Date(); - return this._cachedPercentage; -}); - - -Zotero.Sync.Storage.Queue.prototype.isRunning = function () { - return this._running; -} - -Zotero.Sync.Storage.Queue.prototype.isStopping = function () { - return this._stopping; -} - - -/** - * Add a request to this queue - * - * @param {Zotero.Sync.Storage.Request} request - * @param {Boolean} highPriority Add or move request to high priority queue - */ -Zotero.Sync.Storage.Queue.prototype.addRequest = function (request, highPriority) { - if (this._finished) { - this.reset(); - } - - request.queue = this; - var name = request.name; - Zotero.debug("Queuing " + this.type + " request '" + name + "' for library " + this.libraryID); - - if (this._requests[name]) { - if (highPriority) { - Zotero.debug("Moving " + name + " to high-priority queue"); - this._requests[name].importCallbacks(request); - this._highPriority.push(name); - return; - } - - Zotero.debug("Request '" + name + "' already exists"); - return; - } - - this._requests[name] = request; - this.totalRequests++; - - if (highPriority) { - this._highPriority.push(name); - } -} - - -Zotero.Sync.Storage.Queue.prototype.start = function () { - if (!this._deferred || this._deferred.promise.isFulfilled()) { - Zotero.debug("Creating deferred for queue " + this.name); - this._deferred = Zotero.Promise.defer(); - } - // The queue manager needs to know what queues were running in the - // current session - Zotero.Sync.Storage.QueueManager.addCurrentQueue(this); - - var self = this; - setTimeout(function () { - self.advance(); - }, 0); - - return this._deferred.promise; -} - - - -/** - * Start another request in this queue if there's an available slot - */ -Zotero.Sync.Storage.Queue.prototype.advance = function () { - this._running = true; - this._finished = false; - - if (this._stopping) { - Zotero.debug(this.Type + " queue for library " + this.libraryID - + "is being stopped in Zotero.Sync.Storage.Queue.advance()", 2); - return; - } - - if (!this.queuedRequests) { - Zotero.debug("No remaining requests in " + this.type - + " queue for library " + this.libraryID + " (" - + this.activeRequests + " active, " - + this.finishedRequests + " finished)"); - return; - } - - if (this.activeRequests >= this.maxConcurrentRequests) { - Zotero.debug(this.Type + " queue for library " + this.libraryID - + " is busy (" + this.activeRequests + "/" - + this.maxConcurrentRequests + ")"); - return; - } - - - - // Start the first unprocessed request - - // Try the high-priority queue first - var self = this; - var request, name; - while (name = this._highPriority.shift()) { - request = this._requests[name]; - if (request.isRunning() || request.isFinished()) { - continue; - } - - let requestName = name; - - Zotero.Promise.try(function () { - var promise = request.start(); - self.advance(); - return promise; - }) - .then(function (result) { - if (result.localChanges) { - self._localChanges = true; - } - if (result.remoteChanges) { - self._remoteChanges = true; - } - if (result.conflict) { - self.addConflict( - requestName, - result.conflict.local, - result.conflict.remote - ); - } - }) - .catch(function (e) { - self.error(e); - }); - - return; - } - - // And then others - for each(var request in this._requests) { - if (request.isRunning() || request.isFinished()) { - continue; - } - - let requestName = request.name; - - // This isn't in a Zotero.Promise.try() because the request needs to get marked - // as running immediately so that it doesn't get run again by a - // subsequent advance() call. - try { - var promise = request.start(); - self.advance(); - } - catch (e) { - self.error(e); - } - - promise.then(function (result) { - if (result.localChanges) { - self._localChanges = true; - } - if (result.remoteChanges) { - self._remoteChanges = true; - } - if (result.conflict) { - self.addConflict( - requestName, - result.conflict.local, - result.conflict.remote - ); - } - }) - .catch(function (e) { - self.error(e); - }); - - return; - } -} - - -Zotero.Sync.Storage.Queue.prototype.updateProgress = function () { - Zotero.Sync.Storage.QueueManager.updateProgress(); -} - - -Zotero.Sync.Storage.Queue.prototype.addConflict = function (requestName, localData, remoteData) { - Zotero.debug('==========='); - Zotero.debug(localData); - Zotero.debug(remoteData); - - this._conflicts.push({ - name: requestName, - localData: localData, - remoteData: remoteData - }); -} - - -Zotero.Sync.Storage.Queue.prototype.error = function (e) { - if (!this._error) { - if (this.isRunning()) { - this._error = e; - } - else { - Zotero.debug("Queue " + this.name + " was no longer running -- not assigning error", 2); - } - } - Zotero.debug(e, 1); - this.stop(); -} - - -/** - * Stops all requests in this queue - */ -Zotero.Sync.Storage.Queue.prototype.stop = function () { - if (!this._running) { - Zotero.debug(this.Type + " queue for library " + this.libraryID - + " is not running"); - return; - } - if (this._stopping) { - Zotero.debug("Already stopping " + this.type + " queue for library " - + this.libraryID); - return; - } - - Zotero.debug("Stopping " + this.type + " queue for library " + this.libraryID); - - // If no requests, finish manually - /*if (this.activeRequests == 0) { - this._finishedRequests = this._finishedRequests; - return; - }*/ - - this._stopping = true; - for each(var request in this._requests) { - if (!request.isFinished()) { - request.stop(true); - } - } - - Zotero.debug("Queue is stopped"); -} - - -Zotero.Sync.Storage.Queue.prototype.reset = function () { - this._finished = false; - this._finishedReqs = 0; - this.totalRequests = 0; -} diff --git a/chrome/content/zotero/xpcom/storage/queueManager.js b/chrome/content/zotero/xpcom/storage/queueManager.js deleted file mode 100644 index e338458d6..000000000 --- a/chrome/content/zotero/xpcom/storage/queueManager.js +++ /dev/null @@ -1,370 +0,0 @@ -/* - ***** BEGIN LICENSE BLOCK ***** - - Copyright © 2009 Center for History and New Media - George Mason University, Fairfax, Virginia, USA - http://zotero.org - - This file is part of Zotero. - - Zotero is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - Zotero is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with Zotero. If not, see . - - ***** END LICENSE BLOCK ***** -*/ - - -Zotero.Sync.Storage.QueueManager = new function () { - var _queues = {}; - var _currentQueues = []; - - this.start = Zotero.Promise.coroutine(function* (libraryID) { - if (libraryID) { - var queues = this.getAll(libraryID); - var suffix = " for library " + libraryID; - } - else { - var queues = this.getAll(); - var suffix = ""; - } - - Zotero.debug("Starting file sync queues" + suffix); - - var promises = []; - for each(var queue in queues) { - if (!queue.unfinishedRequests) { - continue; - } - Zotero.debug("Starting queue " + queue.name); - promises.push(queue.start()); - } - - if (!promises.length) { - Zotero.debug("No files to sync" + suffix); - } - - var results = yield Zotero.Promise.allSettled(promises); - Zotero.debug("All storage queues are finished" + suffix); - - for (let i = 0; i < results.length; i++) { - let result = results[i]; - // Check for conflicts to resolve - if (result.state == "fulfilled") { - result = result.value; - if (result.conflicts.length) { - Zotero.debug("Reconciling conflicts for library " + result.libraryID); - Zotero.debug(result.conflicts); - var data = yield _reconcileConflicts(result.conflicts); - if (data) { - _processMergeData(data); - } - } - } - } - - return promises; - }); - - this.stop = function (libraryID) { - if (libraryID) { - var queues = this.getAll(libraryID); - } - else { - var queues = this.getAll(); - } - for (var queue in queues) { - queue.stop(); - } - }; - - - /** - * Retrieving a queue, creating a new one if necessary - * - * @param {String} queueName - */ - this.get = function (queueName, libraryID, noInit) { - if (typeof libraryID == 'undefined') { - throw new Error("libraryID not specified"); - } - - var hash = queueName + "/" + libraryID; - - // Initialize the queue if it doesn't exist yet - if (!_queues[hash]) { - if (noInit) { - return false; - } - var queue = new Zotero.Sync.Storage.Queue(queueName, libraryID); - switch (queueName) { - case 'download': - queue.maxConcurrentRequests = - Zotero.Prefs.get('sync.storage.maxDownloads') - break; - - case 'upload': - queue.maxConcurrentRequests = - Zotero.Prefs.get('sync.storage.maxUploads') - break; - - default: - throw ("Invalid queue '" + queueName + "' in Zotero.Sync.Storage.QueueManager.get()"); - } - _queues[hash] = queue; - } - - return _queues[hash]; - }; - - - this.getAll = function (libraryID) { - if (typeof libraryID == 'string') { - throw new Error("libraryID must be a number or undefined"); - } - - var queues = []; - for each(var queue in _queues) { - if (typeof libraryID == 'undefined' || queue.libraryID === libraryID) { - queues.push(queue); - } - } - return queues; - }; - - - this.addCurrentQueue = function (queue) { - if (!this.hasCurrentQueue(queue)) { - _currentQueues.push(queue.name); - } - } - - - this.hasCurrentQueue = function (queue) { - return _currentQueues.indexOf(queue.name) != -1; - } - - - /** - * Stop all queues - * - * @param {Boolean} [skipStorageFinish=false] Don't call Zotero.Sync.Storage.finish() - * when done (used when we stopped because of - * an error) - */ - this.cancel = function (skipStorageFinish) { - Zotero.debug("Stopping all storage queues"); - for each(var queue in _queues) { - if (queue.isRunning() && !queue.isStopping()) { - queue.stop(); - } - } - } - - - this.finish = function () { - Zotero.debug("All storage queues are finished"); - _currentQueues = []; - } - - - /** - * Calculate the current progress values and trigger a display update - * - * Also detects when all queues have finished and ends sync progress - */ - this.updateProgress = function () { - var activeRequests = 0; - var allFinished = true; - for each(var queue in _queues) { - // Finished or never started - if (!queue.isRunning() && !queue.isStopping()) { - continue; - } - allFinished = false; - activeRequests += queue.activeRequests; - } - if (activeRequests == 0) { - _updateProgressMeters(0); - if (allFinished) { - this.finish(); - } - return; - } - - var status = {}; - for each(var queue in _queues) { - if (!this.hasCurrentQueue(queue)) { - continue; - } - - if (!status[queue.libraryID]) { - status[queue.libraryID] = {}; - } - if (!status[queue.libraryID][queue.type]) { - status[queue.libraryID][queue.type] = {}; - } - status[queue.libraryID][queue.type].statusString = _getQueueStatus(queue); - status[queue.libraryID][queue.type].percentage = queue.percentage; - status[queue.libraryID][queue.type].totalRequests = queue.totalRequests; - status[queue.libraryID][queue.type].finished = queue.finished; - } - - _updateProgressMeters(activeRequests, status); - } - - - /** - * Get a status string for a queue - * - * @param {Zotero.Sync.Storage.Queue} queue - * @return {String} - */ - function _getQueueStatus(queue) { - var remaining = queue.remaining; - var unfinishedRequests = queue.unfinishedRequests; - - if (!unfinishedRequests) { - return Zotero.getString('sync.storage.none'); - } - - if (remaining > 1000) { - var bytesRemaining = Zotero.getString( - 'sync.storage.mbRemaining', - Zotero.Utilities.numberFormat(remaining / 1000 / 1000, 1) - ); - } - else { - var bytesRemaining = Zotero.getString( - 'sync.storage.kbRemaining', - Zotero.Utilities.numberFormat(remaining / 1000, 0) - ); - } - var totalRequests = queue.totalRequests; - var filesRemaining = Zotero.getString( - 'sync.storage.filesRemaining', - [totalRequests - unfinishedRequests, totalRequests] - ); - return bytesRemaining + ' (' + filesRemaining + ')'; - } - - /** - * Cycle through windows, updating progress meters with new values - */ - function _updateProgressMeters(activeRequests, status) { - // Get overall percentage across queues - var sum = 0, num = 0, percentage, total; - for each(var libraryStatus in status) { - for each(var queueStatus in libraryStatus) { - percentage = queueStatus.percentage; - total = queueStatus.totalRequests; - sum += total * percentage; - num += total; - } - } - var percentage = Math.round(sum / num); - - var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] - .getService(Components.interfaces.nsIWindowMediator); - var enumerator = wm.getEnumerator("navigator:browser"); - while (enumerator.hasMoreElements()) { - var win = enumerator.getNext(); - if (!win.ZoteroPane) continue; - var doc = win.ZoteroPane.document; - - var box = doc.getElementById("zotero-tb-sync-progress-box"); - var meter = doc.getElementById("zotero-tb-sync-progress"); - - if (activeRequests == 0) { - box.hidden = true; - continue; - } - - meter.setAttribute("value", percentage); - box.hidden = false; - - var percentageLabel = doc.getElementById('zotero-tb-sync-progress-tooltip-progress'); - percentageLabel.lastChild.setAttribute('value', percentage + "%"); - - var statusBox = doc.getElementById('zotero-tb-sync-progress-status'); - statusBox.data = status; - } - } - - - var _reconcileConflicts = Zotero.Promise.coroutine(function* (conflicts) { - var objectPairs = []; - for each(var conflict in conflicts) { - var item = Zotero.Sync.Storage.getItemFromRequestName(conflict.name); - var item1 = yield item.clone(false, false, true); - item1.setField('dateModified', - Zotero.Date.dateToSQL(new Date(conflict.localData.modTime), true)); - var item2 = yield item.clone(false, false, true); - item2.setField('dateModified', - Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime), true)); - objectPairs.push([item1, item2]); - } - - var io = { - dataIn: { - type: 'storagefile', - captions: [ - Zotero.getString('sync.storage.localFile'), - Zotero.getString('sync.storage.remoteFile'), - Zotero.getString('sync.storage.savedFile') - ], - objects: objectPairs - } - }; - - var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] - .getService(Components.interfaces.nsIWindowMediator); - var lastWin = wm.getMostRecentWindow("navigator:browser"); - lastWin.openDialog('chrome://zotero/content/merge.xul', '', 'chrome,modal,centerscreen', io); - - if (!io.dataOut) { - return false; - } - - // Since we're only putting cloned items into the merge window, - // we have to manually set the ids - for (var i=0; i. + + ***** END LICENSE BLOCK ***** +*/ + + +if (!Zotero.Sync.Storage) { + Zotero.Sync.Storage = {}; +} + +/** + * An Engine manages file sync processes for a given library + * + * @param {Object} options + * @param {Zotero.Sync.APIClient} options.apiClient + * @param {Integer} options.libraryID + * @param {Function} [onError] - Function to run on error + * @param {Boolean} [stopOnError] + */ +Zotero.Sync.Storage.Engine = function (options) { + if (options.apiClient == undefined) { + throw new Error("options.apiClient not set"); + } + if (options.libraryID == undefined) { + throw new Error("options.libraryID not set"); + } + + this.apiClient = options.apiClient; + this.background = options.background; + this.firstInSession = options.firstInSession; + this.lastFullFileCheck = options.lastFullFileCheck; + this.libraryID = options.libraryID; + this.library = Zotero.Libraries.get(options.libraryID); + + this.local = Zotero.Sync.Storage.Local; + this.utils = Zotero.Sync.Storage.Utilities; + this.mode = this.local.getModeForLibrary(this.libraryID); + var modeClass = this.utils.getClassForMode(this.mode); + this.controller = new modeClass(options); + this.setStatus = options.setStatus || function () {}; + this.onError = options.onError || function (e) {}; + this.stopOnError = options.stopOnError || false; + + this.queues = []; + ['download', 'upload'].forEach(function (type) { + this.queues[type] = new ConcurrentCaller({ + id: `${this.libraryID}/${type}`, + numConcurrent: Zotero.Prefs.get( + 'sync.storage.max' + Zotero.Utilities.capitalize(type) + 's' + ), + onError: this.onError, + stopOnError: this.stopOnError, + logger: Zotero.debug + }); + }.bind(this)) + + this.maxCheckAge = 10800; // maximum age in seconds for upload modification check (3 hours) +} + +Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function* () { + var libraryID = this.libraryID; + if (!Zotero.Prefs.get("sync.storage.enabled")) { + Zotero.debug("File sync is not enabled for " + this.library.name); + return false; + } + + Zotero.debug("Starting file sync for " + this.library.name); + + if (!this.controller.verified) { + Zotero.debug(`${this.mode} file sync is not active`); + + throw new Error("Storage mode verification not implemented"); + + // TODO: Check server + } + if (this.controller.cacheCredentials) { + yield this.controller.cacheCredentials(); + } + + // Get library last-sync time for download-on-sync libraries. + var lastSyncTime = null; + var downloadAll = this.local.downloadOnSync(libraryID); + if (downloadAll) { + lastSyncTime = yield this.controller.getLastSyncTime(libraryID); + } + + // Check for updated files to upload + if (!Zotero.Libraries.isFilesEditable(libraryID)) { + Zotero.debug("No file editing access -- skipping file modification check for " + + this.library.name); + } + // If this is a background sync, it's not the first sync of the session, the library has had + // at least one full check this session, and it's been less than maxCheckAge since the last + // full check of this library, check only files that were previously modified or opened + // recently + else if (this.background + && !this.firstInSession + && this.local.lastFullFileCheck[libraryID] + && (this.local.lastFullFileCheck[libraryID] + + (this.maxCheckAge * 1000)) > new Date().getTime()) { + let itemIDs = this.local.getFilesToCheck(libraryID, this.maxCheckAge); + yield this.local.checkForUpdatedFiles(libraryID, itemIDs); + } + // Otherwise check all files in library + else { + this.local.lastFullFileCheck[libraryID] = new Date().getTime(); + yield this.local.checkForUpdatedFiles(libraryID); + } + + yield this.local.resolveConflicts(libraryID); + + var downloadForced = yield this.local.checkForForcedDownloads(libraryID); + + // If we don't have any forced downloads, we can skip downloads if the last sync time hasn't + // changed or doesn't exist on the server (meaning there are no files) + if (downloadAll && !downloadForced) { + if (lastSyncTime) { + if (this.library.lastStorageSync == lastSyncTime) { + Zotero.debug("Last " + this.mode.toUpperCase() + " sync id hasn't changed for " + + this.library.name + " -- skipping file downloads"); + downloadAll = false; + } + } + else { + Zotero.debug(`No last ${this.mode} sync time for ${this.library.name}` + + " -- skipping file downloads"); + downloadAll = false; + } + } + + // Get files to download + if (downloadAll || downloadForced) { + let itemIDs = yield this.local.getFilesToDownload(libraryID, !downloadAll); + if (itemIDs.length) { + Zotero.debug(itemIDs.length + " file" + (itemIDs.length == 1 ? '' : 's') + " to " + + "download for " + this.library.name); + for (let itemID of itemIDs) { + let item = yield Zotero.Items.getAsync(itemID); + yield this.queueItem(item); + } + } + else { + Zotero.debug("No files to download for " + this.library.name); + } + } + + // Get files to upload + if (Zotero.Libraries.isFilesEditable(libraryID)) { + let itemIDs = yield this.local.getFilesToUpload(libraryID); + if (itemIDs.length) { + Zotero.debug(itemIDs.length + " file" + (itemIDs.length == 1 ? '' : 's') + " to " + + "upload for " + this.library.name); + for (let itemID of itemIDs) { + let item = yield Zotero.Items.getAsync(itemID, { noCache: true }); + yield this.queueItem(item); + } + } + else { + Zotero.debug("No files to upload for " + this.library.name); + } + } + else { + Zotero.debug("No file editing access -- skipping file uploads for " + this.library.name); + } + + var promises = { + download: this.queues.download.runAll(), + upload: this.queues.upload.runAll() + } + + // Process the results + var changes = new Zotero.Sync.Storage.Result; + for (let type of ['download', 'upload']) { + let results = yield promises[type]; + + if (this.stopOnError) { + for (let p of results) { + if (p.isRejected()) { + let e = p.reason(); + Zotero.debug(`File ${type} sync failed for ${this.library.name}`); + throw e; + } + } + } + + Zotero.debug(`File ${type} sync finished for ${this.library.name}`); + + changes.updateFromResults(results.filter(p => p.isFulfilled()).map(p => p.value())); + } + + // If files were uploaded, update the remote last-sync time + if (changes.remoteChanges) { + lastSyncTime = yield this.controller.setLastSyncTime(libraryID); + if (!lastSyncTime) { + throw new Error("Last sync time not set after sync"); + } + } + + // If there's a remote last-sync time from either the check before downloads or when it + // was changed after uploads, store that locally so we know we can skip download checks + // next time + if (lastSyncTime) { + this.library.lastStorageSync = lastSyncTime; + yield this.library.saveTx(); + } + + // If WebDAV sync, purge deleted and orphaned files + if (this.mode == 'webdav') { + try { + yield this.controller.purgeDeletedStorageFiles(libraryID); + yield this.controller.purgeOrphanedStorageFiles(libraryID); + } + catch (e) { + Zotero.logError(e); + } + } + + if (!changes.localChanges) { + Zotero.debug("No local changes made during file sync"); + } + + Zotero.debug("Done with file sync for " + this.library.name); + + return changes; +}) + + +Zotero.Sync.Storage.Engine.prototype.stop = function () { + for (let type in this.queues) { + this.queues[type].stop(); + } +} + +Zotero.Sync.Storage.Engine.prototype.queueItem = Zotero.Promise.coroutine(function* (item) { + switch (yield this.local.getSyncState(item.id)) { + case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD: + case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD: + var type = 'download'; + var onStart = Zotero.Promise.method(function (request) { + return this.controller.downloadFile(request); + }.bind(this)); + break; + + case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD: + case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD: + var type = 'upload'; + var onStart = Zotero.Promise.method(function (request) { + return this.controller.uploadFile(request); + }.bind(this)); + break; + + case false: + Zotero.debug("Sync state for item " + item.id + " not found", 2); + return; + + default: + throw new Error("Invalid sync state " + (yield this.local.getSyncState(item.id))); + } + + var request = new Zotero.Sync.Storage.Request({ + type, + libraryID: this.libraryID, + name: item.libraryKey, + onStart, + onProgress: this.onProgress + }); + if (type == 'upload') { + try { + request.setMaxSize(yield Zotero.Attachments.getTotalFileSize(item)); + } + // If this fails, ignore it, though we might fail later + catch (e) { + // But if the file doesn't exist yet, don't try to upload it + // + // This isn't a perfect test, because the file could still be in the process of being + // downloaded (e.g., from the web). It'd be better to download files to a temp + // directory and move them into place. + if (!(yield item.getFilePathAsync())) { + Zotero.debug("File " + item.libraryKey + " not yet available to upload -- skipping"); + return; + } + + Zotero.logError(e); + } + } + this.queues[type].add(request.start.bind(request)); +}) diff --git a/chrome/content/zotero/xpcom/storage/storageLocal.js b/chrome/content/zotero/xpcom/storage/storageLocal.js new file mode 100644 index 000000000..ea8a13c0e --- /dev/null +++ b/chrome/content/zotero/xpcom/storage/storageLocal.js @@ -0,0 +1,1088 @@ +Zotero.Sync.Storage.Local = { + lastFullFileCheck: {}, + uploadCheckFiles: [], + + getClassForLibrary: function (libraryID) { + return Zotero.Sync.Storage.Utilities.getClassForMode(this.getModeForLibrary(libraryID)); + }, + + getModeForLibrary: function (libraryID) { + var libraryType = Zotero.Libraries.getType(libraryID); + switch (libraryType) { + case 'user': + case 'publications': + return Zotero.Prefs.get("sync.storage.protocol") == 'webdav' ? 'webdav' : 'zfs'; + + case 'group': + return 'zfs'; + + default: + throw new Error(`Unexpected library type '${libraryType}'`); + } + }, + + setModeForLibrary: function (libraryID, mode) { + var libraryType = Zotero.Libraries.getType(libraryID); + + if (libraryType != 'user') { + throw new Error(`Cannot set storage mode for ${libraryType} library`); + } + + switch (mode) { + case 'webdav': + case 'zfs': + Zotero.Prefs.set("sync.storage.protocol", mode); + break; + + default: + throw new Error(`Unexpected storage mode '${mode}'`); + } + }, + + /** + * Check or enable download-as-needed mode + * + * @param {Integer} [libraryID] + * @param {Boolean} [enable] - If true, enable download-as-needed mode for the given library + * @return {Boolean|undefined} - If 'enable' isn't set to true, return true if + * download-as-needed mode enabled and false if not + */ + downloadAsNeeded: function (libraryID, enable) { + var pref = this._getDownloadPrefFromLibrary(libraryID); + var val = 'on-demand'; + if (enable) { + Zotero.Prefs.set(pref, val); + return; + } + return Zotero.Prefs.get(pref) == val; + }, + + /** + * Check or enable download-on-sync mode + * + * @param {Integer} [libraryID] + * @param {Boolean} [enable] - If true, enable download-on-demand mode for the given library + * @return {Boolean|undefined} - If 'enable' isn't set to true, return true if + * download-as-needed mode enabled and false if not + */ + downloadOnSync: function (libraryID, enable) { + var pref = this._getDownloadPrefFromLibrary(libraryID); + var val = 'on-demand'; + if (enable) { + Zotero.Prefs.set(pref, val); + return; + } + return Zotero.Prefs.get(pref) == val; + }, + + _getDownloadPrefFromLibrary: function (libraryID) { + if (libraryID == Zotero.Libraries.userLibraryID) { + return 'sync.storage.downloadMode.personal'; + } + // TODO: Library-specific settings + + // Group library + return 'sync.storage.downloadMode.groups'; + }, + + /** + * Get files to check for local modifications for uploading + * + * This includes files previously modified or opened externally via Zotero within maxCheckAge + */ + getFilesToCheck: Zotero.Promise.coroutine(function* (libraryID, maxCheckAge) { + var minTime = new Date().getTime() - (maxCheckAge * 1000); + + // Get files modified and synced since maxCheckAge + var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) " + + "WHERE libraryID=? AND linkMode IN (?,?) AND syncState IN (?) AND " + + "storageModTime>=?"; + var params = [ + libraryID, + Zotero.Attachments.LINK_MODE_IMPORTED_FILE, + Zotero.Attachments.LINK_MODE_IMPORTED_URL, + Zotero.Sync.Storage.SYNC_STATE_IN_SYNC, + minTime + ]; + var itemIDs = yield Zotero.DB.columnQueryAsync(sql, params); + + // Get files opened since maxCheckAge + itemIDs = itemIDs.concat( + this.uploadCheckFiles.filter(x => x.timestamp >= minTime).map(x => x.itemID) + ); + + return Zotero.Utilities.arrayUnique(itemIDs); + }), + + + /** + * Scans local files and marks any that have changed for uploading + * and any that are missing for downloading + * + * @param {Integer} libraryID + * @param {Integer[]} [itemIDs] + * @param {Object} [itemModTimes] Item mod times indexed by item ids; + * items with stored mod times + * that differ from the provided + * time but file mod times + * matching the stored time will + * be marked for download + * @return {Promise} Promise resolving to TRUE if any items changed state, + * FALSE otherwise + */ + checkForUpdatedFiles: Zotero.Promise.coroutine(function* (libraryID, itemIDs, itemModTimes) { + var libraryName = Zotero.Libraries.getName(libraryID); + var msg = "Checking for locally changed attachment files in " + libraryName; + + var memmgr = Components.classes["@mozilla.org/memory-reporter-manager;1"] + .getService(Components.interfaces.nsIMemoryReporterManager); + memmgr.init(); + //Zotero.debug("Memory usage: " + memmgr.resident); + + if (itemIDs) { + if (!itemIDs.length) { + Zotero.debug("No files to check for local changes"); + return false; + } + } + if (itemModTimes) { + if (!Object.keys(itemModTimes).length) { + return false; + } + msg += " in download-marking mode"; + } + + Zotero.debug(msg); + + var changed = false; + + if (!itemIDs) { + itemIDs = Object.keys(itemModTimes ? itemModTimes : {}); + } + + // Can only handle a certain number of bound parameters at a time + var numIDs = itemIDs.length; + var maxIDs = Zotero.DB.MAX_BOUND_PARAMETERS - 10; + var done = 0; + var rows = []; + + do { + let chunk = itemIDs.splice(0, maxIDs); + let sql = "SELECT itemID, linkMode, path, storageModTime, storageHash, syncState " + + "FROM itemAttachments JOIN items USING (itemID) " + + "WHERE linkMode IN (?,?) AND syncState IN (?,?)"; + let params = [ + Zotero.Attachments.LINK_MODE_IMPORTED_FILE, + Zotero.Attachments.LINK_MODE_IMPORTED_URL, + Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD, + Zotero.Sync.Storage.SYNC_STATE_IN_SYNC + ]; + if (libraryID !== false) { + sql += " AND libraryID=?"; + params.push(libraryID); + } + if (chunk.length) { + sql += " AND itemID IN (" + chunk.map(() => '?').join() + ")"; + params = params.concat(chunk); + } + let chunkRows = yield Zotero.DB.queryAsync(sql, params); + if (chunkRows) { + rows = rows.concat(chunkRows); + } + done += chunk.length; + } + while (done < numIDs); + + // If no files, or everything is already marked for download, + // we don't need to do anything + if (!rows.length) { + Zotero.debug("No in-sync or to-upload files found in " + libraryName); + return false; + } + + // Index attachment data by item id + itemIDs = []; + var attachmentData = {}; + for (let row of rows) { + var id = row.itemID; + itemIDs.push(id); + attachmentData[id] = { + linkMode: row.linkMode, + path: row.path, + mtime: row.storageModTime, + hash: row.storageHash, + state: row.syncState + }; + } + rows = null; + + var t = new Date(); + var items = yield Zotero.Items.getAsync(itemIDs, { noCache: true }); + var numItems = items.length; + var updatedStates = {}; + + //Zotero.debug("Memory usage: " + memmgr.resident); + + var changed = false; + for (let i = 0; i < items.length; i++) { + let item = items[i]; + // TODO: Catch error? + let state = yield this._checkForUpdatedFile(item, attachmentData[item.id]); + if (state !== false) { + yield Zotero.Sync.Storage.Local.setSyncState(item.id, state); + changed = true; + } + } + + if (!items.length) { + Zotero.debug("No synced files have changed locally"); + } + + Zotero.debug(`Checked ${numItems} files in ${libraryName} in ` + (new Date() - t) + " ms"); + + return changed; + }), + + + _checkForUpdatedFile: Zotero.Promise.coroutine(function* (item, attachmentData, remoteModTime) { + var lk = item.libraryKey; + Zotero.debug("Checking attachment file for item " + lk, 4); + + var path = item.getFilePath(); + if (!path) { + Zotero.debug("Marking pathless attachment " + lk + " as in-sync"); + return Zotero.Sync.Storage.SYNC_STATE_IN_SYNC; + } + var fileName = OS.Path.basename(path); + var file; + + try { + file = yield OS.File.open(path); + let info = yield file.stat(); + //Zotero.debug("Memory usage: " + memmgr.resident); + + let fmtime = info.lastModificationDate.getTime(); + //Zotero.debug("File modification time for item " + lk + " is " + fmtime); + + if (fmtime < 0) { + Zotero.debug("File mod time " + fmtime + " is less than 0 -- interpreting as 0", 2); + fmtime = 0; + } + + // If file is already marked for upload, skip check. Even if the file was changed + // both locally and remotely, conflicts are checked at upload time, so we don't need + // to worry about it here. + if ((yield this.getSyncState(item.id)) == Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD) { + Zotero.debug("File is already marked for upload"); + return false; + } + + //Zotero.debug("Stored mtime is " + attachmentData.mtime); + //Zotero.debug("File mtime is " + fmtime); + + //BAIL AFTER DOWNLOAD MARKING MODE, OR CHECK LOCAL? + let mtime = attachmentData ? attachmentData.mtime : false; + + // Download-marking mode + if (remoteModTime) { + Zotero.debug(`Remote mod time for item ${lk} is ${remoteModTime}`); + + // Ignore attachments whose stored mod times haven't changed + mtime = mtime !== false ? mtime : (yield this.getSyncedModificationTime(item.id)); + if (mtime == remoteModTime) { + Zotero.debug(`Synced mod time (${mtime}) hasn't changed for item ${lk}`); + return false; + } + + Zotero.debug(`Marking attachment ${lk} for download (stored mtime: ${mtime})`); + // DEBUG: Always set here, or allow further steps? + return Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD; + } + + var same = !this.checkFileModTime(item, fmtime, mtime); + if (same) { + Zotero.debug("File has not changed"); + return false; + } + + // If file hash matches stored hash, only the mod time changed, so skip + let fileHash = yield Zotero.Utilities.Internal.md5Async(file); + + var hash = attachmentData ? attachmentData.hash : (yield this.getSyncedHash(item.id)); + if (hash && hash == fileHash) { + // We have to close the file before modifying it from the main + // thread (at least on Windows, where assigning lastModifiedTime + // throws an NS_ERROR_FILE_IS_LOCKED otherwise) + yield file.close(); + + Zotero.debug("Mod time didn't match (" + fmtime + " != " + mtime + ") " + + "but hash did for " + fileName + " for item " + lk + + " -- updating file mod time"); + try { + yield OS.File.setDates(path, null, mtime); + } + catch (e) { + Zotero.File.checkPathAccessError(e, path, 'update'); + } + return false; + } + + // Mark file for upload + Zotero.debug("Marking attachment " + lk + " as changed " + + "(" + mtime + " != " + fmtime + ")"); + return Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD; + } + catch (e) { + if (e instanceof OS.File.Error && + (e.becauseNoSuchFile + // This can happen if a path is too long on Windows, + // e.g. a file is being accessed on a VM through a share + // (and probably in other cases). + || (e.winLastError && e.winLastError == 3) + // Handle long filenames on OS X/Linux + || (e.unixErrno && e.unixErrno == 63))) { + Zotero.debug("Marking attachment " + lk + " as missing"); + return Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD; + } + + if (e instanceof OS.File.Error) { + if (e.becauseClosed) { + Zotero.debug("File was closed", 2); + } + Zotero.debug(e); + Zotero.debug(e.toString()); + throw new Error(`Error for operation '${e.operation}' for ${path}`); + } + + throw e; + } + finally { + if (file) { + //Zotero.debug("Closing file for item " + lk); + file.close(); + } + } + }), + + /** + * + * @param {Zotero.Item} item + * @param {Integer} fmtime - File modification time in milliseconds + * @param {Integer} mtime - Remote modification time in milliseconds + * @return {Boolean} - True if file modification time differs from remote mod time, + * false otherwise + */ + checkFileModTime(item, fmtime, mtime) { + var libraryKey = item.libraryKey; + + if (fmtime == mtime) { + Zotero.debug(`Mod time for ${libraryKey} matches remote file -- skipping`); + } + // Compare floored timestamps for filesystems that don't support millisecond + // precision (e.g., HFS+) + else if (Math.floor(mtime / 1000) * 1000 == fmtime + || Math.floor(fmtime / 1000) * 1000 == mtime) { + Zotero.debug(`File mod times for ${libraryKey} are within one-second precision ` + + "(" + fmtime + " ≅ " + mtime + ") -- skipping"); + } + // Allow timestamp to be exactly one hour off to get around time zone issues + // -- there may be a proper way to fix this + else if (Math.abs(fmtime - mtime) == 3600000 + // And check with one-second precision as well + || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000 + || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) { + Zotero.debug(`File mod time (${fmtime}) for {$libraryKey} is exactly one hour off ` + + `remote file (${mtime}) -- assuming time zone issue and skipping`); + } + else { + return true; + } + + return false; + }, + + checkForForcedDownloads: Zotero.Promise.coroutine(function* (libraryID) { + // Forced downloads happen even in on-demand mode + var sql = "SELECT COUNT(*) FROM items JOIN itemAttachments USING (itemID) " + + "WHERE libraryID=? AND syncState=?"; + return !!(yield Zotero.DB.valueQueryAsync( + sql, [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD] + )); + }), + + + /** + * Get files marked as ready to download + * + * @param {Integer} libraryID + * @return {Promise} - Promise for an array of attachment itemIDs + */ + getFilesToDownload: function (libraryID, forcedOnly) { + var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) " + + "WHERE libraryID=? AND syncState IN (?"; + var params = [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD]; + if (!forcedOnly) { + sql += ",?"; + params.push(Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD); + } + sql += ") " + // Skip attachments with empty path, which can't be saved, and files with .zotero* + // paths, which have somehow ended up in some users' libraries + + "AND path!='' AND path NOT LIKE 'storage:.zotero%'"; + return Zotero.DB.columnQueryAsync(sql, params); + }, + + + /** + * Get files marked as ready to upload + * + * @param {Integer} libraryID + * @return {Promise} - Promise for an array of attachment itemIDs + */ + getFilesToUpload: function (libraryID) { + var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) " + + "WHERE libraryID=? AND syncState IN (?,?) AND linkMode IN (?,?)"; + var params = [ + libraryID, + Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD, + Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD, + Zotero.Attachments.LINK_MODE_IMPORTED_FILE, + Zotero.Attachments.LINK_MODE_IMPORTED_URL + ]; + return Zotero.DB.columnQueryAsync(sql, params); + }, + + + /** + * @param {Integer} libraryID + * @return {Promise} - Promise for an array of item keys + */ + getDeletedFiles: function (libraryID) { + var sql = "SELECT key FROM storageDeleteLog WHERE libraryID=?"; + return Zotero.DB.columnQueryAsync(sql, libraryID); + }, + + + /** + * @param {Integer} itemID + */ + getSyncState: function (itemID) { + var sql = "SELECT syncState FROM itemAttachments WHERE itemID=?"; + return Zotero.DB.valueQueryAsync(sql, itemID); + }, + + + /** + * @param {Integer} itemID + * @param {Integer} syncState Constant from Zotero.Sync.Storage + */ + setSyncState: Zotero.Promise.method(function (itemID, syncState) { + switch (syncState) { + case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD: + case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD: + case Zotero.Sync.Storage.SYNC_STATE_IN_SYNC: + case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD: + case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD: + case Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT: + break; + + default: + throw new Error("Invalid sync state " + syncState); + } + + var sql = "UPDATE itemAttachments SET syncState=? WHERE itemID=?"; + return Zotero.DB.valueQueryAsync(sql, [syncState, itemID]); + }), + + + /** + * @param {Integer} itemID + * @return {Integer|NULL} Mod time as timestamp in ms, + * or NULL if never synced + */ + getSyncedModificationTime: Zotero.Promise.coroutine(function* (itemID) { + var sql = "SELECT storageModTime FROM itemAttachments WHERE itemID=?"; + var mtime = yield Zotero.DB.valueQueryAsync(sql, itemID); + if (mtime === false) { + throw new Error("Item " + itemID + " not found") + } + return mtime; + }), + + + /** + * @param {Integer} itemID + * @param {Integer} mtime - File modification time as timestamp in ms + * @param {Boolean} [updateItem=FALSE] - Update clientDateModified field of attachment item + */ + setSyncedModificationTime: Zotero.Promise.coroutine(function* (itemID, mtime, updateItem) { + if (mtime < 0) { + Components.utils.reportError("Invalid file mod time " + mtime + + " in Zotero.Storage.setSyncedModificationTime()"); + mtime = 0; + } + + Zotero.DB.requireTransaction(); + + var sql = "UPDATE itemAttachments SET storageModTime=? WHERE itemID=?"; + yield Zotero.DB.queryAsync(sql, [mtime, itemID]); + + if (updateItem) { + // Update item date modified so the new mod time will be synced + let sql = "UPDATE items SET clientDateModified=? WHERE itemID=?"; + yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]); + } + }), + + + /** + * @param {Integer} itemID + * @return {Promise} - File hash, null if never synced, if false if + * file doesn't exist + */ + getSyncedHash: Zotero.Promise.coroutine(function* (itemID) { + var sql = "SELECT storageHash FROM itemAttachments WHERE itemID=?"; + var hash = yield Zotero.DB.valueQueryAsync(sql, itemID); + if (hash === false) { + throw new Error("Item " + itemID + " not found"); + } + return hash; + }), + + + /** + * @param {Integer} itemID + * @param {String} hash File hash + * @param {Boolean} [updateItem=FALSE] Update dateModified field of + * attachment item + */ + setSyncedHash: Zotero.Promise.coroutine(function* (itemID, hash, updateItem) { + if (hash !== null && hash.length != 32) { + throw new Error("Invalid file hash '" + hash + "'"); + } + + Zotero.DB.requireTransaction(); + + var sql = "UPDATE itemAttachments SET storageHash=? WHERE itemID=?"; + yield Zotero.DB.queryAsync(sql, [hash, itemID]); + + if (updateItem) { + // Update item date modified so the new mod time will be synced + var sql = "UPDATE items SET clientDateModified=? WHERE itemID=?"; + yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]); + } + }), + + + /** + * Extract a downloaded file and update the database metadata + * + * @param {Zotero.Item} data.item + * @param {Integer} data.mtime + * @param {String} data.md5 + * @param {Boolean} data.compressed + * @return {Promise} + */ + processDownload: Zotero.Promise.coroutine(function* (data) { + if (!data) { + throw new Error("'data' not set"); + } + if (!data.item) { + throw new Error("'data.item' not set"); + } + if (!data.mtime) { + throw new Error("'data.mtime' not set"); + } + if (data.mtime != parseInt(data.mtime)) { + throw new Error("Invalid mod time '" + data.mtime + "'"); + } + if (!data.compressed && !data.md5) { + throw new Error("'data.md5' is required if 'data.compressed'"); + } + + var item = data.item; + var mtime = parseInt(data.mtime); + var md5 = data.md5; + + // TODO: Test file hash + + if (data.compressed) { + var newPath = yield this._processZipDownload(item); + } + else { + var newPath = yield this._processSingleFileDownload(item); + } + + // If newPath is set, the file was renamed, so set item filename to that + // and mark for updated + var path = yield item.getFilePathAsync(); + if (newPath && path != newPath) { + // If library isn't editable but filename was changed, update + // database without updating the item's mod time, which would result + // in a library access error + if (!Zotero.Items.isEditable(item)) { + Zotero.debug("File renamed without library access -- " + + "updating itemAttachments path", 3); + yield item.relinkAttachmentFile(newPath, true); + } + else { + yield item.relinkAttachmentFile(newPath); + } + + path = newPath; + } + + if (!path) { + // This can happen if an HTML snapshot filename was changed and synced + // elsewhere but the renamed file wasn't synced, so the ZIP doesn't + // contain a file with the known name + Components.utils.reportError("File '" + item.attachmentFilename + + "' not found after processing download " + item.libraryKey); + return new Zotero.Sync.Storage.Result({ + localChanges: false + }); + } + + try { + // If hash not provided (e.g., WebDAV), calculate it now + if (!md5) { + md5 = yield item.attachmentHash; + } + } + catch (e) { + Zotero.File.checkFileAccessError(e, path, 'update'); + } + + // Set the file mtime to the time from the server + yield OS.File.setDates(path, null, new Date(parseInt(mtime))); + + yield Zotero.DB.executeTransaction(function* () { + yield this.setSyncedHash(item.id, md5); + yield this.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); + yield this.setSyncedModificationTime(item.id, mtime); + }.bind(this)); + + return new Zotero.Sync.Storage.Result({ + localChanges: true + }); + }), + + + _processSingleFileDownload: Zotero.Promise.coroutine(function* (item) { + var tempFilePath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.tmp'); + + if (!(yield OS.File.exists(tempFilePath))) { + Zotero.debug(tempFilePath, 1); + throw new Error("Downloaded file not found"); + } + + var parentDirPath = Zotero.Attachments.getStorageDirectory(item).path; + if (!(yield OS.File.exists(parentDirPath))) { + yield Zotero.Attachments.createDirectoryForItem(item); + } + + yield this._deleteExistingAttachmentFiles(item); + + var path = item.getFilePath(); + if (!path) { + throw new Error("Empty path for item " + item.key); + } + // Don't save Windows aliases + if (path.endsWith('.lnk')) { + return false; + } + + var fileName = OS.Path.basename(path); + var renamed = false; + + // Make sure the new filename is valid, in case an invalid character made it over + // (e.g., from before we checked for them) + var filteredName = Zotero.File.getValidFileName(fileName); + if (filteredName != fileName) { + Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'"); + fileName = filteredName; + path = OS.Path.dirname(path, fileName); + renamed = true; + } + + Zotero.debug("Moving download file " + OS.Path.basename(tempFilePath) + + " into attachment directory as '" + fileName + "'"); + try { + var finalFileName = Zotero.File.createShortened( + path, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644 + ); + } + catch (e) { + Zotero.File.checkFileAccessError(e, path, 'create'); + } + + if (finalFileName != fileName) { + Zotero.debug("Changed filename '" + fileName + "' to '" + finalFileName + "'"); + + fileName = finalFileName; + path = OS.Path.dirname(path, fileName); + + // Abort if Windows path limitation would cause filenames to be overly truncated + if (Zotero.isWin && fileName.length < 40) { + try { + yield OS.File.remove(path); + } + catch (e) {} + // TODO: localize + var msg = "Due to a Windows path length limitation, your Zotero data directory " + + "is too deep in the filesystem for syncing to work reliably. " + + "Please relocate your Zotero data to a higher directory."; + Zotero.debug(msg, 1); + throw new Error(msg); + } + + renamed = true; + } + + try { + yield OS.File.move(tempFilePath, path); + } + catch (e) { + try { + yield OS.File.remove(tempFilePath); + } + catch (e) {} + + Zotero.File.checkFileAccessError(e, path, 'create'); + } + + // processDownload() needs to know that we're renaming the file + return renamed ? path : null; + }), + + + _processZipDownload: Zotero.Promise.coroutine(function* (item) { + var zipFile = Zotero.getTempDirectory(); + zipFile.append(item.key + '.tmp'); + + if (!zipFile.exists()) { + Zotero.debug(zipFile.path); + throw new Error(`Downloaded ZIP file not found for item ${item.libraryKey}`); + } + + var zipReader = Components.classes["@mozilla.org/libjar/zip-reader;1"]. + createInstance(Components.interfaces.nsIZipReader); + try { + zipReader.open(zipFile); + zipReader.test(null); + + Zotero.debug("ZIP file is OK"); + } + catch (e) { + Zotero.debug(zipFile.leafName + " is not a valid ZIP file", 2); + zipReader.close(); + + try { + zipFile.remove(false); + } + catch (e) { + Zotero.File.checkFileAccessError(e, zipFile, 'delete'); + } + + // TODO: Remove prop file to trigger reuploading, in case it was an upload error? + + return false; + } + + var parentDir = Zotero.Attachments.getStorageDirectory(item); + if (!parentDir.exists()) { + yield Zotero.Attachments.createDirectoryForItem(item); + } + + try { + yield this._deleteExistingAttachmentFiles(item); + } + catch (e) { + zipReader.close(); + throw (e); + } + + var returnFile = null; + var count = 0; + + var itemFileName = item.attachmentFilename; + + var entries = zipReader.findEntries(null); + while (entries.hasMore()) { + count++; + var entryName = entries.getNext(); + var b64re = /%ZB64$/; + if (entryName.match(b64re)) { + var fileName = Zotero.Utilities.Internal.Base64.decode( + entryName.replace(b64re, '') + ); + } + else { + var fileName = entryName; + } + + if (fileName.startsWith('.zotero')) { + Zotero.debug("Skipping " + fileName); + continue; + } + + Zotero.debug("Extracting " + fileName); + + var primaryFile = false; + var filtered = false; + var renamed = false; + + // Make sure the new filename is valid, in case an invalid character + // somehow make it into the ZIP (e.g., from before we checked for them) + // + // Do this before trying to use the relative descriptor, since otherwise + // it might fail silently and select the parent directory + var filteredName = Zotero.File.getValidFileName(fileName); + if (filteredName != fileName) { + Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'"); + fileName = filteredName; + filtered = true; + } + + // Name in ZIP is a relative descriptor, so file has to be reconstructed + // using setRelativeDescriptor() + var destFile = parentDir.clone(); + destFile.QueryInterface(Components.interfaces.nsILocalFile); + destFile.setRelativeDescriptor(parentDir, fileName); + + fileName = destFile.leafName; + + // If only one file in zip and it doesn't match the known filename, + // take our chances and use that name + if (count == 1 && !entries.hasMore() && itemFileName) { + // May not be necessary, but let's be safe + itemFileName = Zotero.File.getValidFileName(itemFileName); + if (itemFileName != fileName) { + Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'", 2); + Components.utils.reportError("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'"); + fileName = itemFileName; + destFile.leafName = fileName; + renamed = true; + } + } + + var primaryFile = itemFileName == fileName; + if (primaryFile && filtered) { + renamed = true; + } + + if (destFile.exists()) { + var msg = "ZIP entry '" + fileName + "' " + "already exists"; + Zotero.debug(msg, 2); + Components.utils.reportError(msg + " in " + funcName); + Zotero.debug(destFile.path); + continue; + } + + try { + Zotero.File.createShortened(destFile, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644); + } + catch (e) { + Zotero.debug(e, 1); + Components.utils.reportError(e); + + zipReader.close(); + + Zotero.File.checkFileAccessError(e, destFile, 'create'); + } + + if (destFile.leafName != fileName) { + Zotero.debug("Changed filename '" + fileName + "' to '" + destFile.leafName + "'"); + + // Abort if Windows path limitation would cause filenames to be overly truncated + if (Zotero.isWin && destFile.leafName.length < 40) { + try { + destFile.remove(false); + } + catch (e) {} + zipReader.close(); + // TODO: localize + var msg = "Due to a Windows path length limitation, your Zotero data directory " + + "is too deep in the filesystem for syncing to work reliably. " + + "Please relocate your Zotero data to a higher directory."; + Zotero.debug(msg, 1); + throw new Error(msg); + } + + if (primaryFile) { + renamed = true; + } + } + + try { + zipReader.extract(entryName, destFile); + } + catch (e) { + try { + destFile.remove(false); + } + catch (e) {} + + // For advertising junk files, ignore a bug on Windows where + // destFile.create() works but zipReader.extract() doesn't + // when the path length is close to 255. + if (destFile.leafName.match(/[a-zA-Z0-9+=]{130,}/)) { + var msg = "Ignoring error extracting '" + destFile.path + "'"; + Zotero.debug(msg, 2); + Zotero.debug(e, 2); + Components.utils.reportError(msg + " in " + funcName); + continue; + } + + zipReader.close(); + + Zotero.File.checkFileAccessError(e, destFile, 'create'); + } + + destFile.permissions = 0644; + + // If we're renaming the main file, processDownload() needs to know + if (renamed) { + returnFile = destFile.path; + } + } + zipReader.close(); + zipFile.remove(false); + + return returnFile; + }), + + + _deleteExistingAttachmentFiles: Zotero.Promise.coroutine(function* (item) { + var parentDir = Zotero.Attachments.getStorageDirectory(item).path; + return this._deleteExistingFilesInDirectory(parentDir); + }), + + + _deleteExistingFilesInDirectory: Zotero.Promise.coroutine(function* (dir) { + var dirsToDelete = []; + var iterator = new OS.File.DirectoryIterator(dir); + try { + yield iterator.forEach(function (entry) { + return Zotero.Promise.coroutine(function* () { + if (entry.isDir) { + dirsToDelete.push(entry.path); + } + else { + try { + yield OS.File.remove(entry.path); + } + catch (e) { + Zotero.File.checkFileAccessError(e, entry.path, 'delete'); + } + } + })(); + }); + } + finally { + iterator.close(); + } + for (let path of dirsToDelete) { + yield this._deleteExistingFilesInDirectory(path); + } + }), + + + /** + * @return {Promise} - A promise for an array of conflict objects + */ + getConflicts: Zotero.Promise.coroutine(function* (libraryID) { + var sql = "SELECT itemID, version FROM items JOIN itemAttachments USING (itemID) " + + "WHERE libraryID=? AND syncState=?"; + var rows = yield Zotero.DB.queryAsync( + sql, + [ + { int: libraryID }, + Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT + ] + ); + var keyVersionPairs = rows.map(function (row) { + var { libraryID, key } = Zotero.Items.getLibraryAndKeyFromID(row.itemID); + return [key, row.version]; + }); + var cacheObjects = yield Zotero.Sync.Data.Local.getCacheObjects( + 'item', libraryID, keyVersionPairs + ); + if (!cacheObjects.length) return []; + + var cacheObjectsByKey = {}; + cacheObjects.forEach(obj => cacheObjectsByKey[obj.key] = obj); + + var items = []; + var localItems = yield Zotero.Items.getAsync(rows.map(row => row.itemID)); + for (let localItem of localItems) { + // Use the mtime for the dateModified field, since that's all that's shown in the + // CR window at the moment + let localItemJSON = yield localItem.toJSON(); + localItemJSON.dateModified = Zotero.Date.dateToISO( + new Date(yield localItem.attachmentModificationTime) + ); + + let remoteItemJSON = cacheObjectsByKey[localItem.key]; + if (!remoteItemJSON) { + Zotero.logError("Cached object not found for item " + localItem.libraryKey); + continue; + } + remoteItemJSON = remoteItemJSON.data; + remoteItemJSON.dateModified = Zotero.Date.dateToISO(new Date(remoteItemJSON.mtime)); + items.push({ + left: localItemJSON, + right: remoteItemJSON, + changes: [], + conflicts: [] + }) + } + return items; + }), + + + resolveConflicts: Zotero.Promise.coroutine(function* (libraryID) { + var conflicts = yield this.getConflicts(libraryID); + if (!conflicts.length) return false; + + Zotero.debug("Reconciling conflicts for " + Zotero.Libraries.get(libraryID).name); + + var io = { + dataIn: { + type: 'file', + captions: [ + Zotero.getString('sync.storage.localFile'), + Zotero.getString('sync.storage.remoteFile'), + Zotero.getString('sync.storage.savedFile') + ], + conflicts + } + }; + + var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] + .getService(Components.interfaces.nsIWindowMediator); + var lastWin = wm.getMostRecentWindow("navigator:browser"); + lastWin.openDialog('chrome://zotero/content/merge.xul', '', 'chrome,modal,centerscreen', io); + + if (!io.dataOut) { + return false; + } + yield Zotero.DB.executeTransaction(function* () { + for (let i = 0; i < conflicts.length; i++) { + let conflict = conflicts[i]; + let mtime = io.dataOut[i].dateModified; + // Local + if (mtime == conflict.left.dateModified) { + syncState = Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD; + } + // Remote + else { + syncState = Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD; + } + let itemID = Zotero.Items.getIDFromLibraryAndKey(libraryID, conflict.left.key); + yield Zotero.Sync.Storage.Local.setSyncState(itemID, syncState); + } + }); + return true; + }) +} diff --git a/chrome/content/zotero/xpcom/storage/request.js b/chrome/content/zotero/xpcom/storage/storageRequest.js similarity index 60% rename from chrome/content/zotero/xpcom/storage/request.js rename to chrome/content/zotero/xpcom/storage/storageRequest.js index bf0d5baa0..ca6b496e3 100644 --- a/chrome/content/zotero/xpcom/storage/request.js +++ b/chrome/content/zotero/xpcom/storage/storageRequest.js @@ -27,15 +27,25 @@ /** * Transfer request for storage sync * - * @param {String} name Identifier for request (e.g., "[libraryID]/[key]") - * @param {Function} onStart Callback to run when request starts + * @param {Object} options + * @param {String} options.type + * @param {Integer} options.libraryID + * @param {String} options.name - Identifier for request (e.g., "[libraryID]/[key]") + * @param {Function|Function[]} [options.onStart] + * @param {Function|Function[]} [options.onProgress] + * @param {Function|Function[]} [options.onStop] */ -Zotero.Sync.Storage.Request = function (name, callbacks) { - Zotero.debug("Initializing request '" + name + "'"); +Zotero.Sync.Storage.Request = function (options) { + if (!options.type) throw new Error("type must be provided"); + if (!options.libraryID) throw new Error("libraryID must be provided"); + if (!options.name) throw new Error("name must be provided"); + ['type', 'libraryID', 'name'].forEach(x => this[x] = options[x]); - this.callbacks = ['onStart', 'onProgress']; + Zotero.debug(`Initializing ${this.type} request ${this.name}`); - this.name = name; + this.callbacks = ['onStart', 'onProgress', 'onStop']; + + this.Type = Zotero.Utilities.capitalize(this.type); this.channel = null; this.queue = null; this.progress = 0; @@ -48,17 +58,10 @@ Zotero.Sync.Storage.Request = function (name, callbacks) { this._remaining = null; this._maxSize = null; this._finished = false; - this._forceFinish = false; - this._changesMade = false; - for (var func in callbacks) { - if (this.callbacks.indexOf(func) !== -1) { - // Stuff all single functions into arrays - this['_' + func] = typeof callbacks[func] === 'function' ? [callbacks[func]] : callbacks[func]; - } - else { - throw new Error("Invalid handler '" + func + "'"); - } + for (let name of this.callbacks) { + if (!options[name]) continue; + this['_' + name] = Array.isArray(options[name]) ? options[name] : [options[name]]; } } @@ -99,11 +102,6 @@ Zotero.Sync.Storage.Request.prototype.importCallbacks = function (request) { } -Zotero.Sync.Storage.Request.prototype.__defineGetter__('promise', function () { - return this._deferred.promise; -}); - - Zotero.Sync.Storage.Request.prototype.__defineGetter__('percentage', function () { if (this._finished) { return 100; @@ -142,7 +140,7 @@ Zotero.Sync.Storage.Request.prototype.__defineGetter__('remaining', function () } if (!this.progressMax) { - if (this.queue.type == 'upload' && this._maxSize) { + if (this.type == 'upload' && this._maxSize) { return Math.round(Zotero.Sync.Storage.compressionTracker.ratio * this._maxSize); } @@ -175,72 +173,47 @@ Zotero.Sync.Storage.Request.prototype.setChannel = function (channel) { } -Zotero.Sync.Storage.Request.prototype.start = function () { - if (!this.queue) { - throw ("Request " + this.name + " must be added to a queue before starting"); - } - - Zotero.debug("Starting " + this.queue.name + " request " + this.name); +Zotero.Sync.Storage.Request.prototype.start = Zotero.Promise.coroutine(function* () { + Zotero.debug("Starting " + this.type + " request " + this.name); if (this._running) { - throw new Error("Request " + this.name + " already running"); + throw new Error(this.type + " request " + this.name + " already running"); + } + + if (!this._onStart) { + throw new Error("onStart not provided -- nothing to do!"); } this._running = true; - this.queue.activeRequests++; - if (this.queue.type == 'download') { - Zotero.Sync.Storage.setItemDownloadPercentage(this.name, 0); - } - - var self = this; - - // this._onStart is an array of promises returning changesMade. + // this._onStart is an array of promises for objects of result flags, which are combined + // into a single object here // // The main sync logic is triggered here. - - Zotero.Promise.all([f(this) for each(f in this._onStart)]) - .then(function (results) { - return { - localChanges: results.some(function (val) val && val.localChanges == true), - remoteChanges: results.some(function (val) val && val.remoteChanges == true), - conflict: results.reduce(function (prev, cur) { - return prev.conflict ? prev : cur; - }).conflict - }; - }) - .then(function (results) { - Zotero.debug(results); + try { + var results = yield Zotero.Promise.all(this._onStart.map(f => f(this))); - if (results.localChanges) { - Zotero.debug("Changes were made by " + self.queue.name - + " request " + self.name); - } - else { - Zotero.debug("No changes were made by " + self.queue.name - + " request " + self.name); - } + var result = new Zotero.Sync.Storage.Result; + result.updateFromResults(results); - // This promise updates localChanges/remoteChanges on the queue - self._deferred.resolve(results); - }) - .catch(function (e) { - if (self._stopping) { - Zotero.debug("Skipping error for stopping request " + self.name); - return; + Zotero.debug(this.Type + " request " + this.name + " finished"); + Zotero.debug(result + ""); + + return result; + } + catch (e) { + Zotero.logError(this.Type + " request " + this.name + " failed"); + throw e; + } + finally { + this._finished = true; + this._running = false; + + if (this._onStop) { + this._onStop.forEach(x => x()); } - Zotero.debug(self.queue.Type + " request " + self.name + " failed"); - self._deferred.reject(e); - }) - // Finish the request (and in turn the queue, if this is the last request) - .finally(function () { - if (!self._finished) { - self._finish(); - } - }); - - return this._deferred.promise; -} + } +}); Zotero.Sync.Storage.Request.prototype.isRunning = function () { @@ -263,7 +236,7 @@ Zotero.Sync.Storage.Request.prototype.isFinished = function () { * @param {Integer} progressMax Max progress value for this request * (usually total bytes) */ -Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress, progressMax) { +Zotero.Sync.Storage.Request.prototype.onProgress = function (progress, progressMax) { //Zotero.debug(progress + "/" + progressMax + " for request " + this.name); if (!this._running) { @@ -273,10 +246,6 @@ Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress, return; } - if (!this.channel) { - this.channel = channel; - } - // Workaround for invalid progress values (possibly related to // https://bugzilla.mozilla.org/show_bug.cgi?id=451991 and fixed in 3.1) if (progress < this.progress) { @@ -292,9 +261,8 @@ Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress, this.progress = progress; this.progressMax = progressMax; - this.queue.updateProgress(); - if (this.queue.type == 'download') { + if (this.type == 'download') { Zotero.Sync.Storage.setItemDownloadPercentage(this.name, this.percentage); } @@ -310,59 +278,15 @@ Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress, * Stop the request's underlying network request, if there is one */ Zotero.Sync.Storage.Request.prototype.stop = function (force) { - if (force) { - this._forceFinish = true; - } - if (this.channel && this.channel.isPending()) { this._stopping = true; try { - Zotero.debug("Stopping request '" + this.name + "'"); + Zotero.debug(`Stopping ${this.type} request '${this.name} '`); this.channel.cancel(0x804b0002); // NS_BINDING_ABORTED } catch (e) { - Zotero.debug(e); + Zotero.debug(e, 1); } } - else { - this._finish(); - } -} - - -/** - * Mark request as finished and notify queue that it's done - */ -Zotero.Sync.Storage.Request.prototype._finish = function () { - // If an error occurred, we wait to finish the request, since doing - // so might end the queue before the error flag has been set on the queue. - // When the queue's error handler stops the queue, it stops the request - // with stop(true) to force the finish to occur, allowing the queue's - // promise to be rejected with the error. - if (!this._forceFinish && this._deferred.promise.isRejected()) { - return; - } - - Zotero.debug("Finishing " + this.queue.name + " request '" + this.name + "'"); - this._finished = true; - var active = this._running; - this._running = false; - - Zotero.Sync.Storage.setItemDownloadPercentage(this.name, false); - - if (active) { - this.queue.activeRequests--; - } - // TEMP: mechanism for failures? - try { - this.queue.finishedRequests++; - this.queue.updateProgress(); - } - catch (e) { - Zotero.debug(e, 1); - Components.utils.reportError(e); - this._deferred.reject(e); - throw e; - } } diff --git a/chrome/content/zotero/xpcom/storage/storageResult.js b/chrome/content/zotero/xpcom/storage/storageResult.js new file mode 100644 index 000000000..eaa1f38c1 --- /dev/null +++ b/chrome/content/zotero/xpcom/storage/storageResult.js @@ -0,0 +1,47 @@ +"use strict"; + +/** + * @property {Boolean} localChanges - Changes were made locally. For logging purposes only. + * @property {Boolean} remoteChanges - Changes were made on the server. This causes the + * last-sync time to be updated on the server (WebDAV) or retrieved (ZFS) and stored locally + * to skip additional file syncs until further server changes are made. + * @property {Boolean} syncRequired - A data sync is required to upload local changes + * @propretty {Boolean} fileSyncRequired - Another file sync is required to handle files left in + * conflict + */ +Zotero.Sync.Storage.Result = function (options = {}) { + this._props = ['localChanges', 'remoteChanges', 'syncRequired', 'fileSyncRequired']; + for (let prop of this._props) { + this[prop] = options[prop] || false; + } +} + +/** + * Update the properties on this object from multiple Result objects + * + * @param {Zotero.Sync.Storage.Result[]} results + */ +Zotero.Sync.Storage.Result.prototype.updateFromResults = function (results) { + for (let prop of this._props) { + if (!this[prop]) { + for (let result of results) { + if (!(result instanceof Zotero.Sync.Storage.Result)) { + Zotero.debug(result, 1); + throw new Error("'result' is not a storage result"); + } + if (result[prop]) { + this[prop] = true; + } + } + } + } +} + + +Zotero.Sync.Storage.Result.prototype.toString = function () { + var obj = {}; + for (let prop of this._props) { + obj[prop] = this[prop] || false; + } + return JSON.stringify(obj, null, " "); +} diff --git a/chrome/content/zotero/xpcom/storage/storageUtilities.js b/chrome/content/zotero/xpcom/storage/storageUtilities.js new file mode 100644 index 000000000..7df99f1a6 --- /dev/null +++ b/chrome/content/zotero/xpcom/storage/storageUtilities.js @@ -0,0 +1,67 @@ +Zotero.Sync.Storage.Utilities = { + getClassForMode: function (mode) { + switch (mode) { + case 'zfs': + return Zotero.Sync.Storage.ZFS_Module; + + case 'webdav': + return Zotero.Sync.Storage.WebDAV_Module; + + default: + throw new Error("Invalid storage mode '" + mode + "'"); + } + }, + + getItemFromRequest: function (request) { + var [libraryID, key] = request.name.split('/'); + return Zotero.Items.getByLibraryAndKey(libraryID, key); + }, + + + /** + * Create zip file of attachment directory in the temp directory + * + * @param {Zotero.Sync.Storage.Request} request + * @return {Promise} - True if the zip file was created, false otherwise + */ + createUploadFile: Zotero.Promise.coroutine(function* (request) { + var item = this.getItemFromRequest(request); + Zotero.debug("Creating ZIP file for item " + item.libraryKey); + + switch (item.attachmentLinkMode) { + case Zotero.Attachments.LINK_MODE_LINKED_FILE: + case Zotero.Attachments.LINK_MODE_LINKED_URL: + throw new Error("Upload file must be an imported snapshot or file"); + } + + var zipFile = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.zip'); + + return Zotero.File.zipDirectory( + Zotero.Attachments.getStorageDirectory(item).path, + zipFile, + { + onStopRequest: function (req, context, status) { + var zipFileName = OS.Path.basename(zipFile); + + var originalSize = 0; + for (let entry of context.entries) { + let zipEntry = context.zipWriter.getEntry(entry.name); + if (!zipEntry) { + Zotero.logError("ZIP entry '" + entry.name + "' not found for " + + "request '" + request.name + "'") + continue; + } + originalSize += zipEntry.realSize; + } + + Zotero.debug("Zip of " + zipFileName + " finished with status " + status + + " (original " + Math.round(originalSize / 1024) + "KB, " + + "compressed " + Math.round(context.zipWriter.file.fileSize / 1024) + "KB, " + + Math.round( + ((originalSize - context.zipWriter.file.fileSize) / originalSize) * 100 + ) + "% reduction)"); + } + } + ); + }) +} diff --git a/chrome/content/zotero/xpcom/storage/streamListener.js b/chrome/content/zotero/xpcom/storage/streamListener.js index 8a8e28295..136a8f895 100644 --- a/chrome/content/zotero/xpcom/storage/streamListener.js +++ b/chrome/content/zotero/xpcom/storage/streamListener.js @@ -30,10 +30,9 @@ * Possible properties of data object: * - onStart: f(request) * - onProgress: f(request, progress, progressMax) - * - onStop: f(request, status, response, data) - * - onCancel: f(request, status, data) + * - onStop: f(request, status, response) + * - onCancel: f(request, status) * - streams: array of streams to close on completion - * - Other values to pass to onStop() */ Zotero.Sync.Storage.StreamListener = function (data) { this._data = data; @@ -110,17 +109,15 @@ Zotero.Sync.Storage.StreamListener.prototype = { }, onStateChange: function (wp, request, stateFlags, status) { - Zotero.debug("onStateChange"); - Zotero.debug(stateFlags); - Zotero.debug(status); + Zotero.debug("onStateChange with " + stateFlags); - if ((stateFlags & Components.interfaces.nsIWebProgressListener.STATE_START) - && (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_NETWORK)) { - this._onStart(request); - } - else if ((stateFlags & Components.interfaces.nsIWebProgressListener.STATE_STOP) - && (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_NETWORK)) { - this._onStop(request, status); + if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_REQUEST) { + if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_START) { + this._onStart(request); + } + else if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_STOP) { + this._onStop(request, status); + } } }, @@ -148,18 +145,38 @@ Zotero.Sync.Storage.StreamListener.prototype = { }, // nsIChannelEventSink - onChannelRedirect: function (oldChannel, newChannel, flags) { + // + // If this._data.onChannelRedirect exists, it should return a promise resolving to true to + // follow the redirect or false to cancel it + onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) { Zotero.debug('onChannelRedirect'); + if (this._data && this._data.onChannelRedirect) { + let result = yield this._data.onChannelRedirect(oldChannel, newChannel, flags); + if (!result) { + oldChannel.cancel(Components.results.NS_BINDING_ABORTED); + newChannel.cancel(Components.results.NS_BINDING_ABORTED); + return false; + } + } + // if redirecting, store the new channel this._channel = newChannel; - }, + }), asyncOnChannelRedirect: function (oldChan, newChan, flags, redirectCallback) { Zotero.debug('asyncOnRedirect'); - this.onChannelRedirect(oldChan, newChan, flags); - redirectCallback.onRedirectVerifyCallback(0); + this.onChannelRedirect(oldChan, newChan, flags) + .then(function (result) { + redirectCallback.onRedirectVerifyCallback( + result ? Components.results.NS_SUCCEEDED : Components.results.NS_FAILED + ); + }) + .catch(function (e) { + Zotero.logError(e); + redirectCallback.onRedirectVerifyCallback(Components.results.NS_FAILED); + }); }, // nsIHttpEventSink @@ -177,8 +194,7 @@ Zotero.Sync.Storage.StreamListener.prototype = { _onStart: function (request) { Zotero.debug('Starting request'); if (this._data && this._data.onStart) { - var data = this._getPassData(); - this._data.onStart(request, data); + this._data.onStart(request); } }, @@ -189,7 +205,6 @@ Zotero.Sync.Storage.StreamListener.prototype = { }, _onStop: function (request, status) { - Zotero.debug('Request ended with status ' + status); var cancelled = status == 0x804b0002; // NS_BINDING_ABORTED if (!cancelled && status == 0 && request instanceof Components.interfaces.nsIHttpChannel) { @@ -201,9 +216,11 @@ Zotero.Sync.Storage.StreamListener.prototype = { Zotero.debug("Request responseStatus not available", 1); status = 0; } + Zotero.debug('Request ended with status code ' + status); request.QueryInterface(Components.interfaces.nsIRequest); } else { + Zotero.debug('Request ended with status ' + status); status = 0; } @@ -213,38 +230,20 @@ Zotero.Sync.Storage.StreamListener.prototype = { } } - var data = this._getPassData(); - if (cancelled) { if (this._data.onCancel) { - this._data.onCancel(request, status, data); + this._data.onCancel(request, status); } } else { if (this._data.onStop) { - this._data.onStop(request, status, this._response, data); + this._data.onStop(request, status, this._response); } } this._channel = null; }, - _getPassData: function () { - // Make copy of data without callbacks to pass along - var passData = {}; - for (var i in this._data) { - switch (i) { - case "onStart": - case "onProgress": - case "onStop": - case "onCancel": - continue; - } - passData[i] = this._data[i]; - } - return passData; - }, - // nsIInterfaceRequestor getInterface: function (iid) { try { diff --git a/chrome/content/zotero/xpcom/storage/webdav.js b/chrome/content/zotero/xpcom/storage/webdav.js index 9ce0e9e6d..2b6c8c4a3 100644 --- a/chrome/content/zotero/xpcom/storage/webdav.js +++ b/chrome/content/zotero/xpcom/storage/webdav.js @@ -24,740 +24,105 @@ */ -Zotero.Sync.Storage.WebDAV = (function () { - var _initialized = false; - var _parentURI; - var _rootURI; - var _cachedCredentials = false; +Zotero.Sync.Storage.WebDAV_Module = {}; +Zotero.Sync.Storage.WebDAV_Module.prototype = { + name: "WebDAV", + get verified() { + return Zotero.Prefs.get("sync.storage.verified"); + }, - var _loginManagerHost = 'chrome://zotero'; - var _loginManagerURL = 'Zotero Storage Server'; + _initialized: false, + _parentURI: null, + _rootURI: null, + _cachedCredentials: false, - var _lastSyncIDLength = 30; + _loginManagerHost: 'chrome://zotero', + _loginManagerURL: 'Zotero Storage Server', - // - // Private methods - // - /** - * Get mod time of file on storage server - * - * @param {Zotero.Item} item - * @param {Function} callback Callback f(item, mdate) - */ - function getStorageModificationTime(item, request) { - var uri = getItemPropertyURI(item); + _lastSyncIDLength: 30, + + + get defaultError() { + return Zotero.getString('sync.storage.error.webdav.default'); + }, + + get defaultErrorRestart() { + return Zotero.getString('sync.storage.error.webdav.defaultRestart', Zotero.appName); + }, + + get _username() { + return Zotero.Prefs.get('sync.storage.username'); + }, + + get _password() { + var username = this._username; - return Zotero.HTTP.promise("GET", uri, - { - debug: true, - successCodes: [200, 300, 404], - requestObserver: function (xmlhttp) { - request.setChannel(xmlhttp.channel); - } - }) - .then(function (req) { - checkResponse(req); - - // mod_speling can return 300s for 404s with base name matches - if (req.status == 404 || req.status == 300) { - return false; - } - - // No modification time set - if (!req.responseText) { - return false; - } - - var seconds = false; - var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"] - .createInstance(Components.interfaces.nsIDOMParser); - try { - var xml = parser.parseFromString(req.responseText, "text/xml"); - var mtime = xml.getElementsByTagName('mtime')[0].textContent; - } - catch (e) { - Zotero.debug(e); - var mtime = false; - } - - // TEMP - if (!mtime) { - mtime = req.responseText; - seconds = true; - } - - var invalid = false; - - // Unix timestamps need to be converted to ms-based timestamps - if (seconds) { - if (mtime.match(/^[0-9]{1,10}$/)) { - Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds"); - mtime = mtime * 1000; - } - else { - invalid = true; - } - } - else if (!mtime.match(/^[0-9]{1,13}$/)) { - invalid = true; - } - - // Delete invalid .prop files - if (invalid) { - var msg = "Invalid mod date '" + Zotero.Utilities.ellipsize(mtime, 20) - + "' for item " + Zotero.Items.getLibraryKeyHash(item); - Zotero.debug(msg, 1); - Components.utils.reportError(msg); - return deleteStorageFiles([item.key + ".prop"]) - .then(function (results) { - throw new Error(Zotero.Sync.Storage.WebDAV.defaultError); - }); - } - - return new Date(parseInt(mtime)); - }) - .catch(function (e) { - if (e instanceof Zotero.HTTP.UnexpectedStatusException) { - throw new Error("HTTP " + e.status + " error from WebDAV " - + "server for GET request"); - } - throw e; - }); - } - - - /** - * Set mod time of file on storage server - * - * @param {Zotero.Item} item - */ - function setStorageModificationTime(item) { - var uri = getItemPropertyURI(item); - - var mtime = item.attachmentModificationTime; - var hash = item.attachmentHash; - - var prop = '' - + '' + mtime + '' - + '' + hash + '' - + ''; - - return Zotero.HTTP.promise("PUT", uri, - { body: prop, debug: true, successCodes: [200, 201, 204] }) - .then(function (req) { - return { mtime: mtime, hash: hash }; - }) - .catch(function (e) { - throw new Error("HTTP " + e.xmlhttp.status - + " from WebDAV server for HTTP PUT"); - }); - }; - - - - /** - * Upload the generated ZIP file to the server - * - * @param {Object} Object with 'request' property - * @return {void} - */ - function processUploadFile(data) { - /* - updateSizeMultiplier( - (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100 - ); - */ - var request = data.request; - var item = Zotero.Sync.Storage.getItemFromRequestName(request.name); - - return getStorageModificationTime(item, request) - .then(function (mdate) { - if (!request.isRunning()) { - Zotero.debug("Upload request '" + request.name - + "' is no longer running after getting mod time"); - return false; - } - - // Check for conflict - if (Zotero.Sync.Storage.getSyncState(item.id) - != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) { - if (mdate) { - // Remote prop time - var mtime = mdate.getTime(); - - // Local file time - var fmtime = item.attachmentModificationTime; - - var same = false; - if (fmtime == mtime) { - same = true; - Zotero.debug("File mod time matches remote file -- skipping upload"); - } - // Allow floored timestamps for filesystems that don't support - // millisecond precision (e.g., HFS+) - else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) { - same = true; - Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") " - + "-- skipping upload"); - } - // Allow timestamp to be exactly one hour off to get around - // time zone issues -- there may be a proper way to fix this - else if (Math.abs(fmtime - mtime) == 3600000 - // And check with one-second precision as well - || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000 - || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) { - same = true; - Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") " - + "-- assuming time zone issue and skipping upload"); - } - - if (same) { - Zotero.DB.beginTransaction(); - var syncState = Zotero.Sync.Storage.getSyncState(item.id); - Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true); - Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); - Zotero.DB.commitTransaction(); - return true; - } - - var smtime = Zotero.Sync.Storage.getSyncedModificationTime(item.id); - if (smtime != mtime) { - Zotero.debug("Conflict -- last synced file mod time " - + "does not match time on storage server" - + " (" + smtime + " != " + mtime + ")"); - return { - localChanges: false, - remoteChanges: false, - conflict: { - local: { modTime: fmtime }, - remote: { modTime: mtime } - } - }; - } - } - else { - Zotero.debug("Remote file not found for item " + item.id); - } - } - - var file = Zotero.getTempDirectory(); - file.append(item.key + '.zip'); - - var fis = Components.classes["@mozilla.org/network/file-input-stream;1"] - .createInstance(Components.interfaces.nsIFileInputStream); - fis.init(file, 0x01, 0, 0); - - var bis = Components.classes["@mozilla.org/network/buffered-input-stream;1"] - .createInstance(Components.interfaces.nsIBufferedInputStream) - bis.init(fis, 64 * 1024); - - var uri = getItemURI(item); - - var ios = Components.classes["@mozilla.org/network/io-service;1"]. - getService(Components.interfaces.nsIIOService); - var channel = ios.newChannelFromURI(uri); - channel.QueryInterface(Components.interfaces.nsIUploadChannel); - channel.setUploadStream(bis, 'application/octet-stream', -1); - channel.QueryInterface(Components.interfaces.nsIHttpChannel); - channel.requestMethod = 'PUT'; - channel.allowPipelining = false; - - channel.setRequestHeader('Keep-Alive', '', false); - channel.setRequestHeader('Connection', '', false); - - var deferred = Zotero.Promise.defer(); - - var listener = new Zotero.Sync.Storage.StreamListener( - { - onProgress: function (a, b, c) { - request.onProgress(a, b, c); - }, - onStop: function (httpRequest, status, response, data) { - data.request.setChannel(false); - - deferred.resolve( - Zotero.Promise.try(function () { - return onUploadComplete(httpRequest, status, response, data); - }) - ); - }, - onCancel: function (httpRequest, status, data) { - onUploadCancel(httpRequest, status, data); - deferred.resolve(false); - }, - request: request, - item: item, - streams: [fis, bis] - } - ); - channel.notificationCallbacks = listener; - - var dispURI = uri.clone(); - if (dispURI.password) { - dispURI.password = '********'; - } - Zotero.debug("HTTP PUT of " + file.leafName + " to " + dispURI.spec); - - channel.asyncOpen(listener, null); - - return deferred.promise; - }); - } - - - function onUploadComplete(httpRequest, status, response, data) { - var request = data.request; - var item = data.item; - var url = httpRequest.name; - - Zotero.debug("Upload of attachment " + item.key - + " finished with status code " + status); - - switch (status) { - case 200: - case 201: - case 204: - break; - - case 403: - case 500: - Zotero.debug(response); - throw (Zotero.getString('sync.storage.error.fileUploadFailed') + - " " + Zotero.getString('sync.storage.error.checkFileSyncSettings')); - - case 507: - Zotero.debug(response); - throw Zotero.getString('sync.storage.error.webdav.insufficientSpace'); - - default: - Zotero.debug(response); - throw (Zotero.getString('sync.storage.error.fileUploadFailed') + - " " + Zotero.getString('sync.storage.error.checkFileSyncSettings') - + "\n\n" + "HTTP " + status); + if (!username) { + Zotero.debug('Username not set before getting Zotero.Sync.Storage.WebDAV.password'); + return ''; } - return setStorageModificationTime(item) - .then(function (props) { - if (!request.isRunning()) { - Zotero.debug("Upload request '" + request.name - + "' is no longer running after getting mod time"); - return false; - } - - Zotero.DB.beginTransaction(); - - Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); - Zotero.Sync.Storage.setSyncedModificationTime(item.id, props.mtime, true); - Zotero.Sync.Storage.setSyncedHash(item.id, props.hash); - - Zotero.DB.commitTransaction(); - - try { - var file = Zotero.getTempDirectory(); - file.append(item.key + '.zip'); - file.remove(false); - } - catch (e) { - Components.utils.reportError(e); - } - - return { - localChanges: true, - remoteChanges: true - }; - }); - } - - - function onUploadCancel(httpRequest, status, data) { - var request = data.request; - var item = data.item; + Zotero.debug('Getting WebDAV password'); + var loginManager = Components.classes["@mozilla.org/login-manager;1"] + .getService(Components.interfaces.nsILoginManager); + var logins = loginManager.findLogins({}, this._loginManagerHost, this._loginManagerURL, null); - Zotero.debug("Upload of attachment " + item.key + " cancelled with status code " + status); - - try { - var file = Zotero.getTempDirectory(); - file.append(item.key + '.zip'); - file.remove(false); - } - catch (e) { - Components.utils.reportError(e); - } - } - - - /** - * Create a Zotero directory on the storage server - */ - function createServerDirectory(callback) { - var uri = Zotero.Sync.Storage.WebDAV.rootURI; - Zotero.HTTP.WebDAV.doMkCol(uri, function (req) { - Zotero.debug(req.responseText); - Zotero.debug(req.status); - - switch (req.status) { - case 201: - return [uri, Zotero.Sync.Storage.SUCCESS]; - - case 401: - return [uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]; - - case 403: - return [uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]; - - case 405: - return [uri, Zotero.Sync.Storage.ERROR_NOT_ALLOWED]; - - case 500: - return [uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR]; - - default: - return [uri, Zotero.Sync.Storage.ERROR_UNKNOWN]; + // Find user from returned array of nsILoginInfo objects + for (var i = 0; i < logins.length; i++) { + if (logins[i].username == username) { + return logins[i].password; } - }); - } - - - /** - * Get the storage URI for an item - * - * @inner - * @param {Zotero.Item} - * @return {nsIURI} URI of file on storage server - */ - function getItemURI(item) { - var uri = Zotero.Sync.Storage.WebDAV.rootURI; - uri.spec = uri.spec + item.key + '.zip'; - return uri; - } - - - /** - * Get the storage property file URI for an item - * - * @inner - * @param {Zotero.Item} - * @return {nsIURI} URI of property file on storage server - */ - function getItemPropertyURI(item) { - var uri = Zotero.Sync.Storage.WebDAV.rootURI; - uri.spec = uri.spec + item.key + '.prop'; - return uri; - } - - - /** - * Get the storage property file URI corresponding to a given item storage URI - * - * @param {nsIURI} Item storage URI - * @return {nsIURI|FALSE} Property file URI, or FALSE if not an item storage URI - */ - function getPropertyURIFromItemURI(uri) { - if (!uri.spec.match(/\.zip$/)) { - return false; - } - var propURI = uri.clone(); - propURI.QueryInterface(Components.interfaces.nsIURL); - propURI.fileName = uri.fileName.replace(/\.zip$/, '.prop'); - propURI.QueryInterface(Components.interfaces.nsIURI); - return propURI; - } - - - /** - * @inner - * @param {String[]} files Remote filenames to delete (e.g., ZIPs) - * @param {Function} callback Passed object containing three arrays: - * 'deleted', 'missing', and 'error', - * each containing filenames - */ - function deleteStorageFiles(files) { - var results = { - deleted: [], - missing: [], - error: [] - }; - - if (files.length == 0) { - return Zotero.Promise.resolve(results); } - let deleteURI = _rootURI.clone(); - // This should never happen, but let's be safe - if (!deleteURI.spec.match(/\/$/)) { - return Zotero.Promise.reject("Root URI does not end in slash in " - + "Zotero.Sync.Storage.WebDAV.deleteStorageFiles()"); - } - - var funcs = []; - for (let i=0; i secErrLimit) && (sslErr < 0 || sslErr > sslErrLimit) ) { + set _password(password) { + var username = this._username; + if (!username) { + Zotero.debug('Username not set before setting Zotero.Sync.Server.Mode.WebDAV.password'); return; } - var secInfo = channel.securityInfo; - if (secInfo instanceof Ci.nsITransportSecurityInfo) { - secInfo.QueryInterface(Ci.nsITransportSecurityInfo); - if ((secInfo.securityState & Ci.nsIWebProgressListener.STATE_IS_INSECURE) == Ci.nsIWebProgressListener.STATE_IS_INSECURE) { - var host = 'host'; - try { - host = channel.URI.host; - } - catch (e) { - Zotero.debug(e); - } - - var msg = Zotero.getString('sync.storage.error.webdav.sslCertificateError', host); - // In Standalone, provide cert_override.txt instructions and a - // button to open the Zotero profile directory - if (Zotero.isStandalone) { - msg += "\n\n" + Zotero.getString('sync.storage.error.webdav.seeCertOverrideDocumentation'); - var buttonText = Zotero.getString('general.openDocumentation'); - var func = function () { - var zp = Zotero.getActiveZoteroPane(); - zp.loadURI("https://www.zotero.org/support/kb/cert_override", { shiftKey: true }); - }; - } - // In Firefox display a button to load the WebDAV URL - else { - msg += "\n\n" + Zotero.getString('sync.storage.error.webdav.loadURLForMoreInfo'); - var buttonText = Zotero.getString('sync.storage.error.webdav.loadURL'); - var func = function () { - var zp = Zotero.getActiveZoteroPane(); - zp.loadURI(channel.URI.spec, { shiftKey: true }); - }; - } - - var e = new Zotero.Error( - msg, - 0, - { - dialogText: msg, - dialogButtonText: buttonText, - dialogButtonCallback: func - } - ); - throw e; - } - else if ((secInfo.securityState & Ci.nsIWebProgressListener.STATE_IS_BROKEN) == Ci.nsIWebProgressListener.STATE_IS_BROKEN) { - var msg = Zotero.getString('sync.storage.error.webdav.sslConnectionError', host) + - Zotero.getString('sync.storage.error.webdav.loadURLForMoreInfo'); - var e = new Zotero.Error( - msg, - 0, - { - dialogText: msg, - dialogButtonText: Zotero.getString('sync.storage.error.webdav.loadURL'), - dialogButtonCallback: function () { - var zp = Zotero.getActiveZoteroPane(); - zp.loadURI(channel.URI.spec, { shiftKey: true }); - } - } - ); - throw e; - } - } - } - - - // - // Public methods (called via Zotero.Sync.Storage.WebDAV) - // - var obj = new Zotero.Sync.Storage.Mode; - obj.name = "WebDAV"; - - Object.defineProperty(obj, "defaultError", { - get: function () Zotero.getString('sync.storage.error.webdav.default') - }); - - Object.defineProperty(obj, "defaultErrorRestart", { - get: function () Zotero.getString('sync.storage.error.webdav.defaultRestart', Zotero.appName) - }); - - Object.defineProperty(obj, "includeUserFiles", { - get: function () { - return Zotero.Prefs.get("sync.storage.enabled") && Zotero.Prefs.get("sync.storage.protocol") == 'webdav'; - } - }); - obj.includeGroupItems = false; - - Object.defineProperty(obj, "_verified", { - get: function () Zotero.Prefs.get("sync.storage.verified") - }); - - Object.defineProperty(obj, "_username", { - get: function () Zotero.Prefs.get('sync.storage.username') - }); - - Object.defineProperty(obj, "_password", { - get: function () { - var username = this._username; - - if (!username) { - Zotero.debug('Username not set before getting Zotero.Sync.Storage.WebDAV.password'); - return ''; - } - - Zotero.debug('Getting WebDAV password'); - var loginManager = Components.classes["@mozilla.org/login-manager;1"] - .getService(Components.interfaces.nsILoginManager); - var logins = loginManager.findLogins({}, _loginManagerHost, _loginManagerURL, null); - - // Find user from returned array of nsILoginInfo objects - for (var i = 0; i < logins.length; i++) { - if (logins[i].username == username) { - return logins[i].password; - } - } - - return ''; - }, + _cachedCredentials = false; - set: function (password) { - var username = this._username; - if (!username) { - Zotero.debug('Username not set before setting Zotero.Sync.Server.Mode.WebDAV.password'); - return; - } - - _cachedCredentials = false; - - var loginManager = Components.classes["@mozilla.org/login-manager;1"] - .getService(Components.interfaces.nsILoginManager); - var logins = loginManager.findLogins({}, _loginManagerHost, _loginManagerURL, null); - - for (var i = 0; i < logins.length; i++) { - Zotero.debug('Clearing WebDAV passwords'); - loginManager.removeLogin(logins[i]); - break; - } - - if (password) { - Zotero.debug(_loginManagerURL); - var nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1", - Components.interfaces.nsILoginInfo, "init"); - var loginInfo = new nsLoginInfo(_loginManagerHost, _loginManagerURL, - null, username, password, "", ""); - loginManager.addLogin(loginInfo); - } + var loginManager = Components.classes["@mozilla.org/login-manager;1"] + .getService(Components.interfaces.nsILoginManager); + var logins = loginManager.findLogins({}, this._loginManagerHost, this._loginManagerURL, null); + + for (var i = 0; i < logins.length; i++) { + Zotero.debug('Clearing WebDAV passwords'); + loginManager.removeLogin(logins[i]); + break; } - }); - - Object.defineProperty(obj, "rootURI", { - get: function () { - if (!_rootURI) { - this._init(); - } - return _rootURI.clone(); + + if (password) { + Zotero.debug(this._loginManagerURL); + var nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1", + Components.interfaces.nsILoginInfo, "init"); + var loginInfo = new nsLoginInfo(this._loginManagerHost, this._loginManagerURL, + null, username, password, "", ""); + loginManager.addLogin(loginInfo); } - }); + }, - Object.defineProperty(obj, "parentURI", { - get: function () { - if (!_parentURI) { - this._init(); - } - return _parentURI.clone(); + get rootURI() { + if (!this._rootURI) { + this._init(); } - }); + return this._rootURI.clone(); + }, - obj._init = function () { - _rootURI = false; - _parentURI = false; + get parentURI() { + if (!this._parentURI) { + this._init(); + } + return this._parentURI.clone(); + }, + + init: function () { + this._rootURI = false; + this._parentURI = false; var scheme = Zotero.Prefs.get('sync.storage.scheme'); switch (scheme) { @@ -816,26 +181,52 @@ Zotero.Sync.Storage.WebDAV = (function () { if (!uri.spec.match(/\/$/)) { uri.spec += "/"; } - _parentURI = uri; + this._parentURI = uri; var uri = uri.clone(); uri.spec += "zotero/"; - _rootURI = uri; - }; + this._rootURI = uri; + }, + + cacheCredentials: Zotero.Promise.coroutine(function* () { + if (this._cachedCredentials) { + Zotero.debug("WebDAV credentials are already cached"); + return; + } - obj.clearCachedCredentials = function() { - _rootURI = _parentURI = undefined; - _cachedCredentials = false; - }; + try { + var req = Zotero.HTTP.request("OPTIONS", this.rootURI); + checkResponse(req); + + Zotero.debug("Credentials are cached"); + this._cachedCredentials = true; + } + catch (e) { + if (e instanceof Zotero.HTTP.UnexpectedStatusException) { + let msg = "HTTP " + e.status + " error from WebDAV server " + + "for OPTIONS request"; + Zotero.debug(msg, 1); + Components.utils.reportError(msg); + throw new Error(Zotero.Sync.Storage.WebDAV.defaultErrorRestart); + } + throw e; + } + }), + + + clearCachedCredentials: function() { + this._rootURI = this._parentURI = undefined; + this._cachedCredentials = false; + }, /** * Begin download process for individual file * * @param {Zotero.Sync.Storage.Request} [request] */ - obj._downloadFile = function (request) { - var item = Zotero.Sync.Storage.getItemFromRequestName(request.name); + downloadFile: function (request, requeueCallback) { + var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request); if (!item) { throw new Error("Item '" + request.name + "' not found"); } @@ -868,7 +259,9 @@ Zotero.Sync.Storage.WebDAV = (function () { Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); Zotero.DB.commitTransaction(); return { - localChanges: true + localChanges: true, // ? + remoteChanges: false, + syncRequired: false }; } @@ -930,7 +323,11 @@ Zotero.Sync.Storage.WebDAV = (function () { Zotero.debug("Finished download of " + destFile.path); try { - deferred.resolve(Zotero.Sync.Storage.processDownload(data)); + deferred.resolve( + Zotero.Sync.Storage.processDownload( + data, requeueCallback + ) + ); } catch (e) { deferred.reject(e); @@ -963,10 +360,10 @@ Zotero.Sync.Storage.WebDAV = (function () { return deferred.promise; }); - }; + }, - obj._uploadFile = function (request) { + uploadFile: function (request) { var deferred = Zotero.Promise.defer(); var created = Zotero.Sync.Storage.createUploadFile( request, @@ -986,10 +383,10 @@ Zotero.Sync.Storage.WebDAV = (function () { return Zotero.Promise.resolve(false); } return deferred.promise; - }; + }, - obj._getLastSyncTime = function () { + getLastSyncTime: function () { var lastSyncURI = this.rootURI; lastSyncURI.spec += "lastsync.txt"; @@ -1063,10 +460,10 @@ Zotero.Sync.Storage.WebDAV = (function () { throw (e); } }); - }; + }, - obj._setLastSyncTime = function (libraryID, localLastSyncID) { + setLastSyncTime: function (libraryID, localLastSyncID) { if (libraryID != Zotero.Libraries.userLibraryID) { throw new Error("libraryID must be user library"); } @@ -1102,36 +499,11 @@ Zotero.Sync.Storage.WebDAV = (function () { Components.utils.reportError(msg); throw Zotero.Sync.Storage.WebDAV.defaultError; }); - }; + }, - obj._cacheCredentials = function () { - if (_cachedCredentials) { - Zotero.debug("WebDAV credentials are already cached"); - return; - } - - return Zotero.HTTP.promise("OPTIONS", this.rootURI) - .then(function (req) { - checkResponse(req); - - Zotero.debug("Credentials are cached"); - _cachedCredentials = true; - }) - .catch(function (e) { - if (e instanceof Zotero.HTTP.UnexpectedStatusException) { - var msg = "HTTP " + e.status + " error from WebDAV server " - + "for OPTIONS request"; - Zotero.debug(msg, 1); - Components.utils.reportError(msg); - throw new Error(Zotero.Sync.Storage.WebDAV.defaultErrorRestart); - } - throw e; - }); - }; - - obj._checkServer = function () { + checkServer: function () { var deferred = Zotero.Promise.defer(); try { @@ -1399,7 +771,7 @@ Zotero.Sync.Storage.WebDAV = (function () { }, 0); return deferred.promise; - }; + }, /** @@ -1407,7 +779,7 @@ Zotero.Sync.Storage.WebDAV = (function () { * * @return bool True if the verification succeeded, false otherwise */ - obj._checkServerCallback = function (uri, status, window, skipSuccessMessage) { + checkServerCallback: function (uri, status, window, skipSuccessMessage) { var promptService = Components.classes["@mozilla.org/embedcomp/prompt-service;1"]. createInstance(Components.interfaces.nsIPromptService); @@ -1545,72 +917,58 @@ Zotero.Sync.Storage.WebDAV = (function () { promptService.alert(window, errorTitle, errorMessage); } return false; - }; + }, /** * Remove files on storage server that were deleted locally * - * @param {Function} callback Passed number of files deleted + * @param {Integer} libraryID */ - obj._purgeDeletedStorageFiles = function () { - return Zotero.Promise.try(function () { - if (!this.includeUserFiles) { - return false; - } - - Zotero.debug("Purging deleted storage files"); - var files = Zotero.Sync.Storage.getDeletedFiles(); - if (!files) { - Zotero.debug("No files to delete remotely"); - return false; - } - - // Add .zip extension - var files = files.map(function (file) file + ".zip"); - - return deleteStorageFiles(files) - .then(function (results) { - // Remove deleted and nonexistent files from storage delete log - var toPurge = results.deleted.concat(results.missing); - if (toPurge.length > 0) { - var done = 0; - var maxFiles = 999; - var numFiles = toPurge.length; - - Zotero.DB.beginTransaction(); - - do { - var chunk = toPurge.splice(0, maxFiles); - var sql = "DELETE FROM storageDeleteLog WHERE key IN (" - + chunk.map(function () '?').join() + ")"; - Zotero.DB.query(sql, chunk); - done += chunk.length; + purgeDeletedStorageFiles: Zotero.Promise.coroutine(function* (libraryID) { + Zotero.debug("Purging deleted storage files"); + var files = yield Zotero.Sync.Storage.Local.getDeletedFiles(libraryID); + if (!files.length) { + Zotero.debug("No files to delete remotely"); + return false; + } + + // Add .zip extension + var files = files.map(file => file + ".zip"); + + var results = yield deleteStorageFiles(files) + + // Remove deleted and nonexistent files from storage delete log + var toPurge = results.deleted.concat(results.missing); + if (toPurge.length > 0) { + yield Zotero.DB.executeTransaction(function* () { + yield Zotero.Utilities.Internal.forEachChunkAsync( + toPurge, + Zotero.DB.MAX_BOUND_PARAMETERS, + function (chunk) { + var sql = "DELETE FROM storageDeleteLog WHERE libraryID=? AND key IN (" + + chunk.map(() => '?').join() + ")"; + return Zotero.DB.queryAsync(sql, [libraryID].concat(chunk)); } - while (done < numFiles); - - Zotero.DB.commitTransaction(); - } - - Zotero.debug(results); - - return results.deleted.length; + ); }); - }.bind(this)); - }; + } + + Zotero.debug(results); + + return results.deleted.length; + }), /** * Delete orphaned storage files older than a day before last sync time */ - obj._purgeOrphanedStorageFiles = function () { + purgeOrphanedStorageFiles: function (libraryID) { + // Note: libraryID not currently used + return Zotero.Promise.try(function () { const daysBeforeSyncTime = 1; - if (!this.includeUserFiles) { - return false; - } - // If recently purged, skip var lastpurge = Zotero.Prefs.get('lastWebDAVOrphanPurge'); var days = 10; @@ -1742,7 +1100,602 @@ Zotero.Sync.Storage.WebDAV = (function () { return deferred.promise; }.bind(this)); - }; + }, - return obj; -}()); + + // + // Private methods + // + /** + * Get mod time of file on storage server + * + * @param {Zotero.Item} item + * @param {Function} callback Callback f(item, mdate) + */ + _getStorageModificationTime: function (item, request) { + var uri = getItemPropertyURI(item); + + return Zotero.HTTP.promise("GET", uri, + { + debug: true, + successCodes: [200, 300, 404], + requestObserver: function (xmlhttp) { + request.setChannel(xmlhttp.channel); + } + }) + .then(function (req) { + checkResponse(req); + + // mod_speling can return 300s for 404s with base name matches + if (req.status == 404 || req.status == 300) { + return false; + } + + // No modification time set + if (!req.responseText) { + return false; + } + + var seconds = false; + var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"] + .createInstance(Components.interfaces.nsIDOMParser); + try { + var xml = parser.parseFromString(req.responseText, "text/xml"); + var mtime = xml.getElementsByTagName('mtime')[0].textContent; + } + catch (e) { + Zotero.debug(e); + var mtime = false; + } + + // TEMP + if (!mtime) { + mtime = req.responseText; + seconds = true; + } + + var invalid = false; + + // Unix timestamps need to be converted to ms-based timestamps + if (seconds) { + if (mtime.match(/^[0-9]{1,10}$/)) { + Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds"); + mtime = mtime * 1000; + } + else { + invalid = true; + } + } + else if (!mtime.match(/^[0-9]{1,13}$/)) { + invalid = true; + } + + // Delete invalid .prop files + if (invalid) { + var msg = "Invalid mod date '" + Zotero.Utilities.ellipsize(mtime, 20) + + "' for item " + Zotero.Items.getLibraryKeyHash(item); + Zotero.debug(msg, 1); + Components.utils.reportError(msg); + return deleteStorageFiles([item.key + ".prop"]) + .then(function (results) { + throw new Error(Zotero.Sync.Storage.WebDAV.defaultError); + }); + } + + return new Date(parseInt(mtime)); + }) + .catch(function (e) { + if (e instanceof Zotero.HTTP.UnexpectedStatusException) { + throw new Error("HTTP " + e.status + " error from WebDAV " + + "server for GET request"); + } + throw e; + }); + }, + + + /** + * Set mod time of file on storage server + * + * @param {Zotero.Item} item + */ + _setStorageModificationTime: Zotero.Promise.coroutine(function* (item) { + var uri = getItemPropertyURI(item); + + var mtime = item.attachmentModificationTime; + var hash = yield item.attachmentHash; + + var prop = '' + + '' + mtime + '' + + '' + hash + '' + + ''; + + return Zotero.HTTP.promise("PUT", uri, + { body: prop, debug: true, successCodes: [200, 201, 204] }) + .then(function (req) { + return { mtime: mtime, hash: hash }; + }) + .catch(function (e) { + throw new Error("HTTP " + e.xmlhttp.status + + " from WebDAV server for HTTP PUT"); + }) + }), + + + + /** + * Upload the generated ZIP file to the server + * + * @param {Object} Object with 'request' property + * @return {void} + */ + _processUploadFile: Zotero.Promise.coroutine(function* (data) { + /* + updateSizeMultiplier( + (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100 + ); + */ + var request = data.request; + var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request); + + var mdate = getStorageModificationTime(item, request); + + if (!request.isRunning()) { + Zotero.debug("Upload request '" + request.name + + "' is no longer running after getting mod time"); + return false; + } + + // Check for conflict + if (Zotero.Sync.Storage.getSyncState(item.id) + != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) { + if (mdate) { + // Local file time + var fmtime = yield item.attachmentModificationTime; + // Remote prop time + var mtime = mdate.getTime(); + + var same = !(yield Zotero.Sync.Storage.checkFileModTime(item, fmtime, mtime)); + if (same) { + Zotero.DB.beginTransaction(); + var syncState = Zotero.Sync.Storage.getSyncState(item.id); + Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true); + Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); + Zotero.DB.commitTransaction(); + return true; + } + + let smtime = Zotero.Sync.Storage.getSyncedModificationTime(item.id); + if (smtime != mtime) { + Zotero.debug("Conflict -- last synced file mod time " + + "does not match time on storage server" + + " (" + smtime + " != " + mtime + ")"); + return { + localChanges: false, + remoteChanges: false, + syncRequired: false, + conflict: { + local: { modTime: fmtime }, + remote: { modTime: mtime } + } + }; + } + } + else { + Zotero.debug("Remote file not found for item " + item.id); + } + } + + var file = Zotero.getTempDirectory(); + file.append(item.key + '.zip'); + + var fis = Components.classes["@mozilla.org/network/file-input-stream;1"] + .createInstance(Components.interfaces.nsIFileInputStream); + fis.init(file, 0x01, 0, 0); + + var bis = Components.classes["@mozilla.org/network/buffered-input-stream;1"] + .createInstance(Components.interfaces.nsIBufferedInputStream) + bis.init(fis, 64 * 1024); + + var uri = getItemURI(item); + + var ios = Components.classes["@mozilla.org/network/io-service;1"]. + getService(Components.interfaces.nsIIOService); + var channel = ios.newChannelFromURI(uri); + channel.QueryInterface(Components.interfaces.nsIUploadChannel); + channel.setUploadStream(bis, 'application/octet-stream', -1); + channel.QueryInterface(Components.interfaces.nsIHttpChannel); + channel.requestMethod = 'PUT'; + channel.allowPipelining = false; + + channel.setRequestHeader('Keep-Alive', '', false); + channel.setRequestHeader('Connection', '', false); + + var deferred = Zotero.Promise.defer(); + + var listener = new Zotero.Sync.Storage.StreamListener( + { + onProgress: function (a, b, c) { + request.onProgress(a, b, c); + }, + onStop: function (httpRequest, status, response, data) { + data.request.setChannel(false); + + deferred.resolve( + Zotero.Promise.try(function () { + return onUploadComplete(httpRequest, status, response, data); + }) + ); + }, + onCancel: function (httpRequest, status, data) { + onUploadCancel(httpRequest, status, data); + deferred.resolve(false); + }, + request: request, + item: item, + streams: [fis, bis] + } + ); + channel.notificationCallbacks = listener; + + var dispURI = uri.clone(); + if (dispURI.password) { + dispURI.password = '********'; + } + Zotero.debug("HTTP PUT of " + file.leafName + " to " + dispURI.spec); + + channel.asyncOpen(listener, null); + + return deferred.promise; + }), + + + _onUploadComplete: function (httpRequest, status, response, data) { + var request = data.request; + var item = data.item; + var url = httpRequest.name; + + Zotero.debug("Upload of attachment " + item.key + + " finished with status code " + status); + + switch (status) { + case 200: + case 201: + case 204: + break; + + case 403: + case 500: + Zotero.debug(response); + throw (Zotero.getString('sync.storage.error.fileUploadFailed') + + " " + Zotero.getString('sync.storage.error.checkFileSyncSettings')); + + case 507: + Zotero.debug(response); + throw Zotero.getString('sync.storage.error.webdav.insufficientSpace'); + + default: + Zotero.debug(response); + throw (Zotero.getString('sync.storage.error.fileUploadFailed') + + " " + Zotero.getString('sync.storage.error.checkFileSyncSettings') + + "\n\n" + "HTTP " + status); + } + + return setStorageModificationTime(item) + .then(function (props) { + if (!request.isRunning()) { + Zotero.debug("Upload request '" + request.name + + "' is no longer running after getting mod time"); + return false; + } + + Zotero.DB.beginTransaction(); + + Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); + Zotero.Sync.Storage.setSyncedModificationTime(item.id, props.mtime, true); + Zotero.Sync.Storage.setSyncedHash(item.id, props.hash); + + Zotero.DB.commitTransaction(); + + try { + var file = Zotero.getTempDirectory(); + file.append(item.key + '.zip'); + file.remove(false); + } + catch (e) { + Components.utils.reportError(e); + } + + return { + localChanges: true, + remoteChanges: true, + syncRequired: true + }; + }); + }, + + + _onUploadCancel: function (httpRequest, status, data) { + var request = data.request; + var item = data.item; + + Zotero.debug("Upload of attachment " + item.key + " cancelled with status code " + status); + + try { + var file = Zotero.getTempDirectory(); + file.append(item.key + '.zip'); + file.remove(false); + } + catch (e) { + Components.utils.reportError(e); + } + }, + + + /** + * Create a Zotero directory on the storage server + */ + _createServerDirectory: function (callback) { + var uri = Zotero.Sync.Storage.WebDAV.rootURI; + Zotero.HTTP.WebDAV.doMkCol(uri, function (req) { + Zotero.debug(req.responseText); + Zotero.debug(req.status); + + switch (req.status) { + case 201: + return [uri, Zotero.Sync.Storage.SUCCESS]; + + case 401: + return [uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED]; + + case 403: + return [uri, Zotero.Sync.Storage.ERROR_FORBIDDEN]; + + case 405: + return [uri, Zotero.Sync.Storage.ERROR_NOT_ALLOWED]; + + case 500: + return [uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR]; + + default: + return [uri, Zotero.Sync.Storage.ERROR_UNKNOWN]; + } + }); + }, + + + /** + * Get the storage URI for an item + * + * @inner + * @param {Zotero.Item} + * @return {nsIURI} URI of file on storage server + */ + _getItemURI: function (item) { + var uri = Zotero.Sync.Storage.WebDAV.rootURI; + uri.spec = uri.spec + item.key + '.zip'; + return uri; + }, + + + /** + * Get the storage property file URI for an item + * + * @inner + * @param {Zotero.Item} + * @return {nsIURI} URI of property file on storage server + */ + _getItemPropertyURI: function (item) { + var uri = Zotero.Sync.Storage.WebDAV.rootURI; + uri.spec = uri.spec + item.key + '.prop'; + return uri; + }, + + + /** + * Get the storage property file URI corresponding to a given item storage URI + * + * @param {nsIURI} Item storage URI + * @return {nsIURI|FALSE} Property file URI, or FALSE if not an item storage URI + */ + _getPropertyURIFromItemURI: function (uri) { + if (!uri.spec.match(/\.zip$/)) { + return false; + } + var propURI = uri.clone(); + propURI.QueryInterface(Components.interfaces.nsIURL); + propURI.fileName = uri.fileName.replace(/\.zip$/, '.prop'); + propURI.QueryInterface(Components.interfaces.nsIURI); + return propURI; + }, + + + /** + * @inner + * @param {String[]} files Remote filenames to delete (e.g., ZIPs) + * @param {Function} callback Passed object containing three arrays: + * 'deleted', 'missing', and 'error', + * each containing filenames + */ + _deleteStorageFiles: function (files) { + var results = { + deleted: [], + missing: [], + error: [] + }; + + if (files.length == 0) { + return Zotero.Promise.resolve(results); + } + + let deleteURI = _rootURI.clone(); + // This should never happen, but let's be safe + if (!deleteURI.spec.match(/\/$/)) { + return Zotero.Promise.reject("Root URI does not end in slash in " + + "Zotero.Sync.Storage.WebDAV.deleteStorageFiles()"); + } + + var funcs = []; + for (let i=0; i secErrLimit) && (sslErr < 0 || sslErr > sslErrLimit) ) { + return; + } + + var secInfo = channel.securityInfo; + if (secInfo instanceof Ci.nsITransportSecurityInfo) { + secInfo.QueryInterface(Ci.nsITransportSecurityInfo); + if ((secInfo.securityState & Ci.nsIWebProgressListener.STATE_IS_INSECURE) == Ci.nsIWebProgressListener.STATE_IS_INSECURE) { + var host = 'host'; + try { + host = channel.URI.host; + } + catch (e) { + Zotero.debug(e); + } + + var msg = Zotero.getString('sync.storage.error.webdav.sslCertificateError', host); + // In Standalone, provide cert_override.txt instructions and a + // button to open the Zotero profile directory + if (Zotero.isStandalone) { + msg += "\n\n" + Zotero.getString('sync.storage.error.webdav.seeCertOverrideDocumentation'); + var buttonText = Zotero.getString('general.openDocumentation'); + var func = function () { + var zp = Zotero.getActiveZoteroPane(); + zp.loadURI("https://www.zotero.org/support/kb/cert_override", { shiftKey: true }); + }; + } + // In Firefox display a button to load the WebDAV URL + else { + msg += "\n\n" + Zotero.getString('sync.storage.error.webdav.loadURLForMoreInfo'); + var buttonText = Zotero.getString('sync.storage.error.webdav.loadURL'); + var func = function () { + var zp = Zotero.getActiveZoteroPane(); + zp.loadURI(channel.URI.spec, { shiftKey: true }); + }; + } + + var e = new Zotero.Error( + msg, + 0, + { + dialogText: msg, + dialogButtonText: buttonText, + dialogButtonCallback: func + } + ); + throw e; + } + else if ((secInfo.securityState & Ci.nsIWebProgressListener.STATE_IS_BROKEN) == Ci.nsIWebProgressListener.STATE_IS_BROKEN) { + var msg = Zotero.getString('sync.storage.error.webdav.sslConnectionError', host) + + Zotero.getString('sync.storage.error.webdav.loadURLForMoreInfo'); + var e = new Zotero.Error( + msg, + 0, + { + dialogText: msg, + dialogButtonText: Zotero.getString('sync.storage.error.webdav.loadURL'), + dialogButtonCallback: function () { + var zp = Zotero.getActiveZoteroPane(); + zp.loadURI(channel.URI.spec, { shiftKey: true }); + } + } + ); + throw e; + } + } + } +} diff --git a/chrome/content/zotero/xpcom/storage/zfs.js b/chrome/content/zotero/xpcom/storage/zfs.js index df52f547a..5c6e0ecc9 100644 --- a/chrome/content/zotero/xpcom/storage/zfs.js +++ b/chrome/content/zotero/xpcom/storage/zfs.js @@ -24,605 +24,874 @@ */ -Zotero.Sync.Storage.ZFS = (function () { - var _rootURI; - var _userURI; - var _headers = { - "Zotero-API-Version" : ZOTERO_CONFIG.API_VERSION - }; - var _cachedCredentials = false; - var _s3Backoff = 1; - var _s3ConsecutiveFailures = 0; - var _maxS3Backoff = 60; - var _maxS3ConsecutiveFailures = 5; +Zotero.Sync.Storage.ZFS_Module = function (options) { + this.options = options; + this.apiClient = options.apiClient; + + this._s3Backoff = 1; + this._s3ConsecutiveFailures = 0; + this._maxS3Backoff = 60; + this._maxS3ConsecutiveFailures = 5; +}; +Zotero.Sync.Storage.ZFS_Module.prototype = { + name: "ZFS", + verified: true, /** - * Get file metadata on storage server - * - * @param {Zotero.Item} item - * @param {Function} callback Callback f(item, etag) + * @return {Promise} A promise for the last sync time */ - function getStorageFileInfo(item, request) { - var funcName = "Zotero.Sync.Storage.ZFS.getStorageFileInfo()"; + getLastSyncTime: Zotero.Promise.coroutine(function* (libraryID) { + var params = this._getRequestParams(libraryID, "laststoragesync"); + var uri = this.apiClient.buildRequestURI(params); - return Zotero.HTTP.promise("GET", getItemInfoURI(item), - { - successCodes: [200, 404], - headers: _headers, - requestObserver: function (xmlhttp) { - request.setChannel(xmlhttp.channel); - } - }) - .then(function (req) { - if (req.status == 404) { - return false; - } - - var info = {}; - info.hash = req.getResponseHeader('ETag'); - if (!info.hash) { - var msg = "Hash not found in info response in " + funcName - + " (" + Zotero.Items.getLibraryKeyHash(item) + ")"; - Zotero.debug(msg, 1); - Zotero.debug(req.status); - Zotero.debug(req.responseText); - Components.utils.reportError(msg); - try { - Zotero.debug(req.getAllResponseHeaders()); - } - catch (e) { - Zotero.debug("Response headers unavailable"); - } - var msg = Zotero.getString('sync.storage.error.zfs.restart', Zotero.appName); - throw msg; - } - info.filename = req.getResponseHeader('X-Zotero-Filename'); - var mtime = req.getResponseHeader('X-Zotero-Modification-Time'); - info.mtime = parseInt(mtime); - info.compressed = req.getResponseHeader('X-Zotero-Compressed') == 'Yes'; - Zotero.debug(info); - - return info; - }) - .catch(function (e) { - if (e instanceof Zotero.HTTP.UnexpectedStatusException) { - if (e.xmlhttp.status == 0) { - var msg = "Request cancelled getting storage file info"; - } - else { - var msg = "Unexpected status code " + e.xmlhttp.status - + " getting storage file info for item " + item.libraryKey; - } - Zotero.debug(msg, 1); - Zotero.debug(e.xmlhttp.responseText); - Components.utils.reportError(msg); - throw new Error(Zotero.Sync.Storage.defaultError); - } - - throw e; - }); - } + try { + let req = yield this.apiClient.makeRequest( + "GET", uri, { successCodes: [200, 404], debug: true } + ); + + // Not yet synced + if (req.status == 404) { + Zotero.debug("No last sync time for library " + libraryID); + return null; + } + + let ts = req.responseText; + let date = new Date(ts * 1000); + Zotero.debug("Last successful ZFS sync for library " + libraryID + " was " + date); + return ts; + } + catch (e) { + Zotero.logError(e); + throw e; + } + }), + + + setLastSyncTime: Zotero.Promise.coroutine(function* (libraryID) { + var params = this._getRequestParams(libraryID, "laststoragesync"); + var uri = this.apiClient.buildRequestURI(params); + + try { + var req = yield this.apiClient.makeRequest( + "POST", uri, { successCodes: [200, 404], debug: true } + ); + } + catch (e) { + var msg = "Unexpected status code " + e.xmlhttp.status + " setting last file sync time"; + Zotero.logError(e); + throw new Error(Zotero.Sync.Storage.defaultError); + } + + // Not yet synced + // + // TODO: Don't call this at all if no files uploaded + if (req.status == 404) { + return; + } + + var time = req.responseText; + if (parseInt(time) != time) { + Zotero.logError(`Unexpected response ${time} setting last file sync time`); + throw new Error(Zotero.Sync.Storage.defaultError); + } + return parseInt(time); + }), /** - * Upload the file to the server + * Begin download process for individual file * - * @param {Object} Object with 'request' property - * @return {void} + * @param {Zotero.Sync.Storage.Request} request + * @return {Promise} - True if file download, false if not */ - function processUploadFile(data) { - /* - updateSizeMultiplier( - (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100 - ); - */ + downloadFile: Zotero.Promise.coroutine(function* (request) { + var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request); + if (!item) { + throw new Error("Item '" + request.name + "' not found"); + } - var request = data.request; - var item = Zotero.Sync.Storage.getItemFromRequestName(request.name); - return getStorageFileInfo(item, request) - .then(function (info) { - if (request.isFinished()) { - Zotero.debug("Upload request '" + request.name - + "' is no longer running after getting file info"); - return false; - } - - // Check for conflict - if (Zotero.Sync.Storage.getSyncState(item.id) - != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) { - if (info) { - // Remote mod time - var mtime = info.mtime; - // Local file time - var fmtime = item.attachmentModificationTime; - - var same = false; - var useLocal = false; - if (fmtime == mtime) { - same = true; - Zotero.debug("File mod time matches remote file -- skipping upload"); - } - // Allow floored timestamps for filesystems that don't support - // millisecond precision (e.g., HFS+) - else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) { - same = true; - Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") " - + "-- skipping upload"); - } - // Allow timestamp to be exactly one hour off to get around - // time zone issues -- there may be a proper way to fix this - else if (Math.abs(fmtime - mtime) == 3600000 - // And check with one-second precision as well - || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000 - || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) { - same = true; - Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") " - + "-- assuming time zone issue and skipping upload"); - } - // Ignore maxed-out 32-bit ints, from brief problem after switch to 32-bit servers - else if (mtime == 2147483647) { - Zotero.debug("Remote mod time is invalid -- uploading local file version"); - useLocal = true; - } - - if (same) { - Zotero.debug(Zotero.Sync.Storage.getSyncedModificationTime(item.id)); - - Zotero.DB.beginTransaction(); - var syncState = Zotero.Sync.Storage.getSyncState(item.id); - //Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true); - Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime); - Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); - Zotero.DB.commitTransaction(); - return { - localChanges: true, - remoteChanges: false - }; - } - - var smtime = Zotero.Sync.Storage.getSyncedModificationTime(item.id); - if (!useLocal && smtime != mtime) { - Zotero.debug("Conflict -- last synced file mod time " - + "does not match time on storage server" - + " (" + smtime + " != " + mtime + ")"); - return { - localChanges: false, - remoteChanges: false, - conflict: { - local: { modTime: fmtime }, - remote: { modTime: mtime } - } - }; - } - } - else { - Zotero.debug("Remote file not found for item " + item.libraryID + "/" + item.key); - } - } - - return getFileUploadParameters( - item, - function (item, target, uploadKey, params) { - return postFile(request, item, target, uploadKey, params); - }, - function () { - updateItemFileInfo(item); - return { - localChanges: true, - remoteChanges: false - }; - } - ); + var path = item.getFilePath(); + if (!path) { + Zotero.debug(`Cannot download file for attachment ${item.libraryKey} with no path`); + return new Zotero.Sync.Storage.Result; + } + + var destPath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.tmp'); + + // saveURI() below appears not to create empty files for Content-Length: 0, + // so we create one here just in case, which also lets us check file access + try { + let file = yield OS.File.open(destPath, { + truncate: true }); - } - - - /** - * Get mod time of file on storage server - * - * @param {Zotero.Item} item - * @param {Function} uploadCallback Callback f(request, item, target, params) - * @param {Function} existsCallback Callback f() to call when file already exists - * on server and uploading isn't necessary - */ - function getFileUploadParameters(item, uploadCallback, existsCallback) { - var funcName = "Zotero.Sync.Storage.ZFS.getFileUploadParameters()"; - - var uri = getItemURI(item); - - if (Zotero.Attachments.getNumFiles(item) > 1) { - var file = Zotero.getTempDirectory(); - var filename = item.key + '.zip'; - file.append(filename); - uri.spec = uri.spec; - var zip = true; + file.close(); } - else { - var file = item.getFile(); - var filename = file.leafName; - var zip = false; + catch (e) { + Zotero.File.checkFileAccessError(e, destPath, 'create'); } - var mtime = item.attachmentModificationTime; - var hash = Zotero.Utilities.Internal.md5(file); - - var body = "md5=" + hash + "&filename=" + encodeURIComponent(filename) - + "&filesize=" + file.fileSize + "&mtime=" + mtime; - if (zip) { - body += "&zip=1"; - } - - return Zotero.HTTP.promise("POST", uri, { body: body, headers: _headers, debug: true }) - .then(function (req) { - if (!req.responseXML) { - throw new Error("Invalid response retrieving file upload parameters"); - } - - var rootTag = req.responseXML.documentElement.tagName; - - if (rootTag != 'upload' && rootTag != 'exists') { - throw new Error("Invalid response retrieving file upload parameters"); - } - - // File was already available, so uploading isn't required - if (rootTag == 'exists') { - return existsCallback(); - } - - var url = req.responseXML.getElementsByTagName('url')[0].textContent; - var uploadKey = req.responseXML.getElementsByTagName('key')[0].textContent; - var params = {}, p = ''; - var paramNodes = req.responseXML.getElementsByTagName('params')[0].childNodes; - for (var i = 0; i < paramNodes.length; i++) { - params[paramNodes[i].tagName] = paramNodes[i].textContent; - } - return uploadCallback(item, url, uploadKey, params); - }) - .catch(function (e) { - if (e instanceof Zotero.HTTP.UnexpectedStatusException) { - if (e.status == 413) { - var retry = e.xmlhttp.getResponseHeader('Retry-After'); - if (retry) { - var minutes = Math.round(retry / 60); - var e = new Zotero.Error( - Zotero.getString('sync.storage.error.zfs.tooManyQueuedUploads', minutes), - "ZFS_UPLOAD_QUEUE_LIMIT" - ); - throw e; - } - - var text, buttonText = null, buttonCallback; - - // Group file - if (item.libraryID) { - var group = Zotero.Groups.getByLibraryID(item.libraryID); - text = Zotero.getString('sync.storage.error.zfs.groupQuotaReached1', group.name) + "\n\n" - + Zotero.getString('sync.storage.error.zfs.groupQuotaReached2'); - } - // Personal file - else { - text = Zotero.getString('sync.storage.error.zfs.personalQuotaReached1') + "\n\n" - + Zotero.getString('sync.storage.error.zfs.personalQuotaReached2'); - buttonText = Zotero.getString('sync.storage.openAccountSettings'); - buttonCallback = function () { - var url = "https://www.zotero.org/settings/storage"; - - var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] - .getService(Components.interfaces.nsIWindowMediator); - var win = wm.getMostRecentWindow("navigator:browser"); - win.ZoteroPane.loadURI(url); - } - } - - text += "\n\n" + filename + " (" + Math.round(file.fileSize / 1024) + "KB)"; - - var e = new Zotero.Error( - Zotero.getString('sync.storage.error.zfs.fileWouldExceedQuota', filename), - "ZFS_OVER_QUOTA", - { - dialogText: text, - dialogButtonText: buttonText, - dialogButtonCallback: buttonCallback - } - ); - e.errorType = 'warning'; - Zotero.debug(e, 2); - Components.utils.reportError(e); - throw e; - } - else if (e.status == 403) { - var groupID = Zotero.Groups.getGroupIDFromLibraryID(item.libraryID); - var e = new Zotero.Error( - "File editing denied for group", - "ZFS_FILE_EDITING_DENIED", - { - groupID: groupID - } - ); - throw e; - } - else if (e.status == 404) { - Components.utils.reportError("Unexpected status code 404 in " + funcName - + " (" + Zotero.Items.getLibraryKeyHash(item) + ")"); - if (Zotero.Prefs.get('sync.debugNoAutoResetClient')) { - Components.utils.reportError("Skipping automatic client reset due to debug pref"); - return; - } - if (!Zotero.Sync.Server.canAutoResetClient) { - Components.utils.reportError("Client has already been auto-reset -- manual sync required"); - return; - } - Zotero.Sync.Server.resetClient(); - Zotero.Sync.Server.canAutoResetClient = false; - throw new Error(Zotero.Sync.Storage.defaultError); - } - - var msg = "Unexpected status code " + e.status + " in " + funcName - + " (" + Zotero.Items.getLibraryKeyHash(item) + ")"; - Zotero.debug(msg, 1); - Zotero.debug(e.xmlhttp.getAllResponseHeaders()); - Components.utils.reportError(msg); - throw new Error(Zotero.Sync.Storage.defaultError); - } - - throw e; - }); - } - - - function postFile(request, item, url, uploadKey, params) { - if (request.isFinished()) { - Zotero.debug("Upload request " + request.name + " is no longer running after getting upload parameters"); - return false; - } - - var file = getUploadFile(item); - - // TODO: make sure this doesn't appear in file - var boundary = "---------------------------" + Math.random().toString().substr(2); - - var mis = Components.classes["@mozilla.org/io/multiplex-input-stream;1"] - .createInstance(Components.interfaces.nsIMultiplexInputStream); - - // Add parameters - for (var key in params) { - var storage = Components.classes["@mozilla.org/storagestream;1"] - .createInstance(Components.interfaces.nsIStorageStream); - storage.init(4096, 4294967295, null); // PR_UINT32_MAX - var out = storage.getOutputStream(0); - - var conv = Components.classes["@mozilla.org/intl/converter-output-stream;1"] - .createInstance(Components.interfaces.nsIConverterOutputStream); - conv.init(out, null, 4096, "?"); - - var str = "--" + boundary + '\r\nContent-Disposition: form-data; name="' + key + '"' - + '\r\n\r\n' + params[key] + '\r\n'; - conv.writeString(str); - conv.close(); - - var instr = storage.newInputStream(0); - mis.appendStream(instr); - } - - // Add file - var sis = Components.classes["@mozilla.org/io/string-input-stream;1"] - .createInstance(Components.interfaces.nsIStringInputStream); - var str = "--" + boundary + '\r\nContent-Disposition: form-data; name="file"\r\n\r\n'; - sis.setData(str, -1); - mis.appendStream(sis); - - var fis = Components.classes["@mozilla.org/network/file-input-stream;1"] - .createInstance(Components.interfaces.nsIFileInputStream); - fis.init(file, 0x01, 0, Components.interfaces.nsIFileInputStream.CLOSE_ON_EOF - | Components.interfaces.nsIFileInputStream.REOPEN_ON_REWIND); - - var bis = Components.classes["@mozilla.org/network/buffered-input-stream;1"] - .createInstance(Components.interfaces.nsIBufferedInputStream) - bis.init(fis, 64 * 1024); - mis.appendStream(bis); - - // End request - var sis = Components.classes["@mozilla.org/io/string-input-stream;1"] - .createInstance(Components.interfaces.nsIStringInputStream); - var str = "\r\n--" + boundary + "--"; - sis.setData(str, -1); - mis.appendStream(sis); - - - /* var cstream = Components.classes["@mozilla.org/intl/converter-input-stream;1"]. - createInstance(Components.interfaces.nsIConverterInputStream); - cstream.init(mis, "UTF-8", 0, 0); // you can use another encoding here if you wish - - let (str = {}) { - cstream.readString(-1, str); // read the whole file and put it in str.value - data = str.value; - } - cstream.close(); // this closes fstream - alert(data); - */ - - var ios = Components.classes["@mozilla.org/network/io-service;1"]. - getService(Components.interfaces.nsIIOService); - var uri = ios.newURI(url, null, null); - var channel = ios.newChannelFromURI(uri); - - channel.QueryInterface(Components.interfaces.nsIUploadChannel); - channel.setUploadStream(mis, "multipart/form-data", -1); - channel.QueryInterface(Components.interfaces.nsIHttpChannel); - channel.requestMethod = 'POST'; - channel.allowPipelining = false; - channel.setRequestHeader('Keep-Alive', '', false); - channel.setRequestHeader('Connection', '', false); - channel.setRequestHeader("Content-Type", "multipart/form-data; boundary=" + boundary, false); - //channel.setRequestHeader('Date', date, false); - - request.setChannel(channel); - var deferred = Zotero.Promise.defer(); + var requestData = {item}; var listener = new Zotero.Sync.Storage.StreamListener( { - onProgress: function (a, b, c) { - request.onProgress(a, b, c); + onStart: function (req) { + if (request.isFinished()) { + Zotero.debug("Download request " + request.name + + " stopped before download started -- closing channel"); + req.cancel(Components.results.NS_BINDING_ABORTED); + deferred.resolve(false); + } }, - onStop: function (httpRequest, status, response, data) { - data.request.setChannel(false); + onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) { + // These will be used in processDownload() if the download succeeds + oldChannel.QueryInterface(Components.interfaces.nsIHttpChannel); - // For timeouts and failures from S3, which happen intermittently, - // wait a little and try again - let timeoutMessage = "Your socket connection to the server was not read from or " - + "written to within the timeout period."; - if (status == 0 || (status == 400 && response.indexOf(timeoutMessage) != -1)) { - if (_s3ConsecutiveFailures >= _maxS3ConsecutiveFailures) { - Zotero.debug(_s3ConsecutiveFailures - + " consecutive S3 failures -- aborting", 1); - _s3ConsecutiveFailures = 0; + Zotero.debug("CHANNEL HERE FOR " + item.libraryKey + " WITH " + oldChannel.status); + Zotero.debug(oldChannel.URI.spec); + Zotero.debug(newChannel.URI.spec); + + var header; + try { + header = "Zotero-File-Modification-Time"; + requestData.mtime = oldChannel.getResponseHeader(header); + header = "Zotero-File-MD5"; + requestData.md5 = oldChannel.getResponseHeader(header); + header = "Zotero-File-Compressed"; + requestData.compressed = oldChannel.getResponseHeader(header) == 'Yes'; + } + catch (e) { + deferred.reject(new Error(`${header} header not set in file request for ${item.libraryKey}`)); + return false; + } + + if (!(yield OS.File.exists(path))) { + return true; + } + + var updateHash = false; + var fileModTime = yield item.attachmentModificationTime; + if (requestData.mtime == fileModTime) { + Zotero.debug("File mod time matches remote file -- skipping download of " + + item.libraryKey); + } + // If not compressed, check hash, in case only timestamp changed + else if (!requestData.compressed && (yield item.attachmentHash) == requestData.md5) { + Zotero.debug("File hash matches remote file -- skipping download of " + + item.libraryKey); + updateHash = true; + } + else { + return true; + } + + // Update local metadata and stop request, skipping file download + yield Zotero.DB.executeTransaction(function* () { + if (updateHash) { + yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, requestData.md5); } - else { - let libraryKey = Zotero.Items.getLibraryKeyHash(item); - let msg = "S3 returned " + status - + " (" + libraryKey + ") -- retrying upload" - Components.utils.reportError(msg); - Zotero.debug(msg, 1); - Zotero.debug(response, 1); - if (_s3Backoff < _maxS3Backoff) { - _s3Backoff *= 2; - } - _s3ConsecutiveFailures++; - Zotero.debug("Delaying " + libraryKey + " upload for " - + _s3Backoff + " seconds", 2); - Q.delay(_s3Backoff * 1000) - .then(function () { - deferred.resolve(postFile(request, item, url, uploadKey, params)); - }); + yield Zotero.Sync.Storage.Local.setSyncedModificationTime( + item.id, requestData.mtime + ); + yield Zotero.Sync.Storage.Local.setSyncState( + item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC + ); + }); + return false; + }), + onProgress: function (a, b, c) { + request.onProgress(a, b, c) + }, + onStop: function (req, status, res) { + request.setChannel(false); + + if (status != 200) { + if (status == 404) { + Zotero.debug("Remote file not found for item " + item.libraryKey); + deferred.resolve(new Zotero.Sync.Storage.Result); return; } + + // If S3 connection is interrupted, delay and retry, or bail if too many + // consecutive failures + if (status == 0) { + if (this._s3ConsecutiveFailures < this._maxS3ConsecutiveFailures) { + let libraryKey = item.libraryKey; + let msg = "S3 returned 0 for " + libraryKey + " -- retrying download" + Components.utils.reportError(msg); + Zotero.debug(msg, 1); + if (this._s3Backoff < this._maxS3Backoff) { + this._s3Backoff *= 2; + } + this._s3ConsecutiveFailures++; + Zotero.debug("Delaying " + libraryKey + " download for " + + this._s3Backoff + " seconds", 2); + Zotero.Promise.delay(this._s3Backoff * 1000) + .then(function () { + deferred.resolve(this._downloadFile(request)); + }); + return; + } + + Zotero.debug(this._s3ConsecutiveFailures + + " consecutive S3 failures -- aborting", 1); + this._s3ConsecutiveFailures = 0; + } + + var msg = "Unexpected status code " + status + " for GET " + uri; + Zotero.debug(msg, 1); + Components.utils.reportError(msg); + // Output saved content, in case an error was captured + try { + let sample = Zotero.File.getContents(destPath, null, 4096); + if (sample) { + Zotero.debug(sample, 1); + } + } + catch (e) { + Zotero.debug(e, 1); + } + deferred.reject(new Error(Zotero.Sync.Storage.defaultError)); + return; } - deferred.resolve(onUploadComplete(httpRequest, status, response, data)); - }, - onCancel: function (httpRequest, status, data) { - onUploadCancel(httpRequest, status, data) - deferred.resolve(false); - }, - request: request, - item: item, - uploadKey: uploadKey, - streams: [mis] + // Don't try to process if the request has been cancelled + if (request.isFinished()) { + Zotero.debug("Download request " + request.name + + " is no longer running after file download", 2); + deferred.resolve(false); + return; + } + + Zotero.debug("Finished download of " + destPath); + + try { + deferred.resolve( + Zotero.Sync.Storage.Local.processDownload(requestData) + ); + } + catch (e) { + Zotero.debug("REJECTING"); + deferred.reject(e); + } + }.bind(this), + onCancel: function (req, status) { + Zotero.debug("Request cancelled"); + if (deferred.promise.isPending()) { + deferred.resolve(false); + } + } } ); - channel.notificationCallbacks = listener; - var dispURI = uri.clone(); - if (dispURI.password) { - dispURI.password = '********'; - } - Zotero.debug("HTTP POST of " + file.leafName + " to " + dispURI.spec); + var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`); + var uri = this.apiClient.buildRequestURI(params); + var headers = this.apiClient.getHeaders(); - channel.asyncOpen(listener, null); + Zotero.debug('Saving ' + uri); + const nsIWBP = Components.interfaces.nsIWebBrowserPersist; + var wbp = Components + .classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"] + .createInstance(nsIWBP); + wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE; + wbp.progressListener = listener; + Zotero.Utilities.Internal.saveURI(wbp, uri, destPath, headers); return deferred.promise; - } + }), - function onUploadComplete(httpRequest, status, response, data) { - return Q.try(function () { - var request = data.request; - var item = data.item; - var uploadKey = data.uploadKey; - - Zotero.debug("Upload of attachment " + item.key - + " finished with status code " + status); - - Zotero.debug(response); - - switch (status) { - case 201: - // Decrease backoff delay on successful upload - if (_s3Backoff > 1) { - _s3Backoff /= 2; - } - // And reset consecutive failures - _s3ConsecutiveFailures = 0; - break; - - case 500: - throw new Error("File upload failed. Please try again."); - - default: - var msg = "Unexpected file upload status " + status - + " in Zotero.Sync.Storage.ZFS.onUploadComplete()" - + " (" + Zotero.Items.getLibraryKeyHash(item) + ")"; - Zotero.debug(msg, 1); - Components.utils.reportError(msg); - Components.utils.reportError(response); - throw new Error(Zotero.Sync.Storage.defaultError); + uploadFile: Zotero.Promise.coroutine(function* (request) { + var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request); + if (yield Zotero.Attachments.hasMultipleFiles(item)) { + let created = yield Zotero.Sync.Storage.Utilities.createUploadFile(request); + if (!created) { + return new Zotero.Sync.Storage.Result; } - - var uri = getItemURI(item); - var body = "update=" + uploadKey + "&mtime=" + item.attachmentModificationTime; - - // Register upload on server - return Zotero.HTTP.promise("POST", uri, { body: body, headers: _headers, successCodes: [204] }) - .then(function (req) { - updateItemFileInfo(item); - return { - localChanges: true, - remoteChanges: true - }; - }) - .catch(function (e) { - var msg = "Unexpected file registration status " + e.status - + " (" + Zotero.Items.getLibraryKeyHash(item) + ")"; - Zotero.debug(msg, 1); - Zotero.debug(e.xmlhttp.responseText); - Zotero.debug(e.xmlhttp.getAllResponseHeaders()); - Components.utils.reportError(msg); - Components.utils.reportError(e.xmlhttp.responseText); - throw new Error(Zotero.Sync.Storage.defaultError); - }); - }); - } + return this._processUploadFile(request); + } + return this._processUploadFile(request); + }), - function updateItemFileInfo(item) { - // Mark as changed locally - Zotero.DB.beginTransaction(); - Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); - - // Store file mod time - var mtime = item.attachmentModificationTime; - Zotero.Sync.Storage.setSyncedModificationTime(item.id, mtime, true); - - // Store file hash of individual files - if (Zotero.Attachments.getNumFiles(item) == 1) { - var hash = item.attachmentHash; - Zotero.Sync.Storage.setSyncedHash(item.id, hash); + /** + * Remove all synced files from the server + */ + purgeDeletedStorageFiles: Zotero.Promise.coroutine(function* () { + var sql = "SELECT value FROM settings WHERE setting=? AND key=?"; + var values = yield Zotero.DB.columnQueryAsync(sql, ['storage', 'zfsPurge']); + if (!values) { + return false; } - Zotero.DB.commitTransaction(); + Zotero.debug("Unlinking synced files on ZFS"); + + var uri = this.userURI; + uri.spec += "removestoragefiles?"; + // Unused + for each(var value in values) { + switch (value) { + case 'user': + uri.spec += "user=1&"; + break; + + case 'group': + uri.spec += "group=1&"; + break; + + default: + throw new Error("Invalid zfsPurge value '" + value + "'"); + } + } + uri.spec = uri.spec.substr(0, uri.spec.length - 1); + + yield Zotero.HTTP.request("POST", uri, ""); + + var sql = "DELETE FROM settings WHERE setting=? AND key=?"; + yield Zotero.DB.queryAsync(sql, ['storage', 'zfsPurge']); + }), + + + // + // Private methods + // + _getRequestParams: function (libraryID, target) { + var library = Zotero.Libraries.get(libraryID); + return { + libraryType: library.libraryType, + libraryTypeID: library.libraryTypeID, + target + }; + }, + + + /** + * Get authorization from API for uploading file + * + * @param {Zotero.Item} item + * @return {Object|String} - Object with upload params or 'exists' + */ + _getFileUploadParameters: Zotero.Promise.coroutine(function* (item) { + var funcName = "Zotero.Sync.Storage.ZFS._getFileUploadParameters()"; + + var path = item.getFilePath(); + var filename = OS.Path.basename(path); + var zip = yield Zotero.Attachments.hasMultipleFiles(item); + if (zip) { + var uploadPath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.zip'); + } + else { + var uploadPath = path; + } + + var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`); + var uri = this.apiClient.buildRequestURI(params); + + // TODO: One-step uploads + /*var headers = { + "Content-Type": "application/json" + }; + var storedHash = yield Zotero.Sync.Storage.Local.getSyncedHash(item.id); + //var storedModTime = yield Zotero.Sync.Storage.getSyncedModificationTime(item.id); + if (storedHash) { + headers["If-Match"] = storedHash; + } + else { + headers["If-None-Match"] = "*"; + } + var mtime = yield item.attachmentModificationTime; + var hash = Zotero.Utilities.Internal.md5(file); + var json = { + md5: hash, + mtime, + filename, + size: file.fileSize + }; + var charset = item.attachmentCharset; + var contentType = item.attachmentContentType; + if (charset) { + json.charset = charset; + } + if (contentType) { + json.contentType = contentType; + } + if (zip) { + json.zip = true; + } try { - if (Zotero.Attachments.getNumFiles(item) > 1) { + var req = yield this.apiClient.makeRequest( + "POST", uri, { body: JSON.stringify(json), headers, debug: true } + ); + }*/ + + var headers = { + "Content-Type": "application/x-www-form-urlencoded" + }; + var storedHash = yield Zotero.Sync.Storage.Local.getSyncedHash(item.id); + //var storedModTime = yield Zotero.Sync.Storage.getSyncedModificationTime(item.id); + if (storedHash) { + headers["If-Match"] = storedHash; + } + else { + headers["If-None-Match"] = "*"; + } + + // Build POST body + var mtime = yield item.attachmentModificationTime; + var params = { + md5: yield item.attachmentHash, + mtime, + filename, + filesize: (yield OS.File.stat(uploadPath)).size + }; + var charset = item.attachmentCharset; + var contentType = item.attachmentContentType; + if (charset) { + params.charset = charset; + } + if (contentType) { + params.contentType = contentType; + } + if (zip) { + params.zipMD5 = yield Zotero.Utilities.Internal.md5Async(uploadPath); + params.zipFilename = OS.Path.basename(uploadPath); + } + var body = []; + for (let i in params) { + body.push(i + "=" + encodeURIComponent(params[i])); + } + body = body.join('&'); + + try { + var req = yield this.apiClient.makeRequest( + "POST", + uri, + { + body, + headers, + // This should include all errors in _handleUploadAuthorizationFailure() + successCodes: [200, 201, 204, 403, 404, 412, 413], + debug: true + } + ); + } + catch (e) { + if (e instanceof Zotero.HTTP.UnexpectedStatusException) { + let msg = "Unexpected status code " + e.status + " in " + funcName + + " (" + item.libraryKey + ")"; + Zotero.logError(msg); + Zotero.debug(e.xmlhttp.getAllResponseHeaders()); + throw new Error(Zotero.Sync.Storage.defaultError); + } + throw e; + } + + var result = yield this._handleUploadAuthorizationFailure(req, item); + if (result instanceof Zotero.Sync.Storage.Result) { + return result; + } + + try { + var json = JSON.parse(req.responseText); + } + catch (e) { + Zotero.logError(e); + Zotero.debug(req.responseText, 1); + } + if (!json) { + throw new Error("Invalid response retrieving file upload parameters"); + } + + if (!json.uploadKey && !json.exists) { + throw new Error("Invalid response retrieving file upload parameters"); + } + + if (json.exists) { + let version = req.getResponseHeader('Last-Modified-Version'); + if (!version) { + throw new Error("Last-Modified-Version not provided"); + } + json.version = version; + } + + Zotero.debug('=-=-=--='); + Zotero.debug(json); + + // TEMP + // + // Passed through to _updateItemFileInfo() + json.mtime = mtime; + json.md5 = params.md5; + if (storedHash) { + json.storedHash = storedHash; + } + + return json; + }), + + + /** + * Handle known errors from upload authorization request + * + * These must be included in successCodes in _getFileUploadParameters() + */ + _handleUploadAuthorizationFailure: Zotero.Promise.coroutine(function* (req, item) { + // + // These must be included in successCodes above. + // TODO: 429? + if (req.status == 403) { + let groupID = Zotero.Groups.getGroupIDFromLibraryID(item.libraryID); + let e = new Zotero.Error( + "File editing denied for group", + "ZFS_FILE_EDITING_DENIED", + { + groupID: groupID + } + ); + throw e; + } + else if (req.status == 404) { + Components.utils.reportError("Unexpected status code 404 in upload authorization " + + "request (" + item.libraryKey + ")"); + if (Zotero.Prefs.get('sync.debugNoAutoResetClient')) { + Components.utils.reportError("Skipping automatic client reset due to debug pref"); + } + if (!Zotero.Sync.Server.canAutoResetClient) { + Components.utils.reportError("Client has already been auto-reset -- manual sync required"); + } + + // TODO: Make an API request to fix this + + throw new Error(Zotero.Sync.Storage.defaultError); + } + else if (req.status == 412) { + Zotero.debug("412 BUT WE'RE COOL"); + let version = req.getResponseHeader('Last-Modified-Version'); + if (!version) { + throw new Error("Last-Modified-Version header not provided"); + } + if (version > item.version) { + return new Zotero.Sync.Storage.Result({ + syncRequired: true + }); + } + if (version < item.version) { + throw new Error("Last-Modified-Version is lower than item version " + + `(${version} < ${item.version})`); + } + + // Get updated item metadata + let library = Zotero.Libraries.get(item.libraryID); + let json = yield this.apiClient.downloadObjects( + library.libraryType, + library.libraryTypeID, + 'item', + [item.key] + )[0]; + if (!Array.isArray(json)) { + Zotero.logError(json); + throw new Error(Zotero.Sync.Storage.defaultError); + } + if (json.length > 1) { + throw new Error("More than one result for item lookup"); + } + + yield Zotero.Sync.Data.Local.saveCacheObjects('item', item.libraryID, json); + json = json[0]; + + if (json.data.version > item.version) { + return new Zotero.Sync.Storage.Result({ + syncRequired: true + }); + } + + let fileHash = yield item.attachmentHash; + let fileModTime = yield item.attachmentModificationTime; + + Zotero.debug("MD5"); + Zotero.debug(json.data.md5); + Zotero.debug(fileHash); + + if (json.data.md5 == fileHash) { + yield Zotero.DB.executeTransaction(function* () { + yield Zotero.Sync.Storage.Local.setSyncedModificationTime( + item.id, fileModTime + ); + yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, fileHash); + yield Zotero.Sync.Storage.Local.setSyncState( + item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC + ); + }); + return new Zotero.Sync.Storage.Result; + } + + yield Zotero.Sync.Storage.Local.setSyncState( + item.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT + ); + return new Zotero.Sync.Storage.Result({ + fileSyncRequired: true + }); + } + else if (req.status == 413) { + let retry = req.getResponseHeader('Retry-After'); + if (retry) { + let minutes = Math.round(retry / 60); + throw new Zotero.Error( + Zotero.getString('sync.storage.error.zfs.tooManyQueuedUploads', minutes), + "ZFS_UPLOAD_QUEUE_LIMIT" + ); + } + + let text, buttonText = null, buttonCallback; + + // Group file + if (item.libraryID) { + var group = Zotero.Groups.getByLibraryID(item.libraryID); + text = Zotero.getString('sync.storage.error.zfs.groupQuotaReached1', group.name) + "\n\n" + + Zotero.getString('sync.storage.error.zfs.groupQuotaReached2'); + } + // Personal file + else { + text = Zotero.getString('sync.storage.error.zfs.personalQuotaReached1') + "\n\n" + + Zotero.getString('sync.storage.error.zfs.personalQuotaReached2'); + buttonText = Zotero.getString('sync.storage.openAccountSettings'); + buttonCallback = function () { + var url = "https://www.zotero.org/settings/storage"; + + var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] + .getService(Components.interfaces.nsIWindowMediator); + var win = wm.getMostRecentWindow("navigator:browser"); + win.ZoteroPane.loadURI(url); + } + } + + text += "\n\n" + filename + " (" + Math.round(file.fileSize / 1024) + "KB)"; + + let e = new Zotero.Error( + Zotero.getString('sync.storage.error.zfs.fileWouldExceedQuota', filename), + "ZFS_OVER_QUOTA", + { + dialogText: text, + dialogButtonText: buttonText, + dialogButtonCallback: buttonCallback + } + ); + e.errorType = 'warning'; + Zotero.debug(e, 2); + Components.utils.reportError(e); + throw e; + } + }), + + + /** + * Given parameters from authorization, upload file to S3 + */ + _uploadFile: Zotero.Promise.coroutine(function* (request, item, params) { + if (request.isFinished()) { + Zotero.debug("Upload request " + request.name + " is no longer running after getting " + + "upload parameters"); + return new Zotero.Sync.Storage.Result; + } + + var file = yield this._getUploadFile(item); + + Components.utils.importGlobalProperties(["File"]); + file = File(file); + + var blob = new Blob([params.prefix, file, params.suffix]); + + try { + var req = yield Zotero.HTTP.request( + "POST", + params.url, + { + headers: { + "Content-Type": params.contentType + }, + body: blob, + requestObserver: function (req) { + request.setChannel(req.channel); + req.upload.addEventListener("progress", function (event) { + if (event.lengthComputable) { + request.onProgress(event.loaded, event.total); + } + }); + }, + debug: true, + successCodes: [201] + } + ); + } + catch (e) { + // For timeouts and failures from S3, which happen intermittently, + // wait a little and try again + let timeoutMessage = "Your socket connection to the server was not read from or " + + "written to within the timeout period."; + if (e.status == 0 + || (e.status == 400 && e.xmlhttp.responseText.indexOf(timeoutMessage) != -1)) { + if (this._s3ConsecutiveFailures >= this._maxS3ConsecutiveFailures) { + Zotero.debug(this._s3ConsecutiveFailures + + " consecutive S3 failures -- aborting", 1); + this._s3ConsecutiveFailures = 0; + } + else { + let msg = "S3 returned " + e.status + " (" + item.libraryKey + ") " + + "-- retrying upload" + Zotero.logError(msg); + Zotero.debug(e.xmlhttp.responseText, 1); + if (this._s3Backoff < this._maxS3Backoff) { + this._s3Backoff *= 2; + } + this._s3ConsecutiveFailures++; + Zotero.debug("Delaying " + item.libraryKey + " upload for " + + this._s3Backoff + " seconds", 2); + yield Zotero.Promise.delay(this._s3Backoff * 1000); + return this._uploadFile(request, item, params); + } + } + else if (e.status == 500) { + // TODO: localize + throw new Error("File upload failed. Please try again."); + } + else { + Zotero.logError(`Unexpected file upload status ${e.status} (${item.libraryKey})`); + Zotero.debug(e, 1); + Components.utils.reportError(e.xmlhttp.responseText); + throw new Error(Zotero.Sync.Storage.defaultError); + } + + // TODO: Detect cancel? + //onUploadCancel(httpRequest, status, data) + //deferred.resolve(false); + } + + request.setChannel(false); + return this._onUploadComplete(req, request, item, params); + }), + + + /** + * Post-upload file registration with API + */ + _onUploadComplete: Zotero.Promise.coroutine(function* (req, request, item, params) { + var uploadKey = params.uploadKey; + + Zotero.debug("Upload of attachment " + item.key + " finished with status code " + req.status); + Zotero.debug(req.responseText); + + // Decrease backoff delay on successful upload + if (this._s3Backoff > 1) { + this._s3Backoff /= 2; + } + // And reset consecutive failures + this._s3ConsecutiveFailures = 0; + + var requestParams = this._getRequestParams(item.libraryID, `items/${item.key}/file`); + var uri = this.apiClient.buildRequestURI(requestParams); + var headers = { + "Content-Type": "application/x-www-form-urlencoded" + }; + if (params.storedHash) { + headers["If-Match"] = params.storedHash; + } + else { + headers["If-None-Match"] = "*"; + } + var body = "upload=" + uploadKey; + + // Register upload on server + try { + req = yield this.apiClient.makeRequest( + "POST", + uri, + { + body, + headers, + successCodes: [204], + requestObserver: function (xmlhttp) { + request.setChannel(xmlhttp.channel); + } + } + ); + } + catch (e) { + let msg = `Unexpected file registration status ${e.status} (${item.libraryKey})`; + Zotero.logError(msg); + Zotero.logError(e.xmlhttp.responseText); + Zotero.debug(e.xmlhttp.getAllResponseHeaders()); + throw new Error(Zotero.Sync.Storage.defaultError); + } + + var version = req.getResponseHeader('Last-Modified-Version'); + if (!version) { + throw new Error("Last-Modified-Version not provided"); + } + params.version = version; + + yield this._updateItemFileInfo(item, params); + + return new Zotero.Sync.Storage.Result({ + localChanges: true, + remoteChanges: true + }); + }), + + + /** + * Update the local attachment item with the mtime and hash of the uploaded file and the + * library version returned by the upload request, and save a modified version of the item + * to the sync cache + */ + _updateItemFileInfo: Zotero.Promise.coroutine(function* (item, params) { + // Mark as in-sync + yield Zotero.DB.executeTransaction(function* () { + yield Zotero.Sync.Storage.Local.setSyncState( + item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC + ); + + // Store file mod time and hash + yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, params.mtime); + yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, params.md5); + // Update sync cache with new file metadata and version from server + var json = yield Zotero.Sync.Data.Local.getCacheObject( + 'item', item.libraryID, item.key, item.version + ); + if (json) { + json.version = params.version; + json.data.version = params.version; + json.data.mtime = params.mtime; + json.data.md5 = params.md5; + yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json); + } + // Update item with new version from server + yield Zotero.Items.updateVersion([item.id], params.version); + + // TODO: Can filename, contentType, and charset change the attachment item? + }); + + try { + if (yield Zotero.Attachments.hasMultipleFiles(item)) { var file = Zotero.getTempDirectory(); file.append(item.key + '.zip'); - file.remove(false); + yield OS.File.remove(file.path); } } catch (e) { Components.utils.reportError(e); } - } + }), - function onUploadCancel(httpRequest, status, data) { + _onUploadCancel: Zotero.Promise.coroutine(function* (httpRequest, status, data) { var request = data.request; var item = data.item; Zotero.debug("Upload of attachment " + item.key + " cancelled with status code " + status); try { - if (Zotero.Attachments.getNumFiles(item) > 1) { + if (yield Zotero.Attachments.hasMultipleFiles(item)) { var file = Zotero.getTempDirectory(); file.append(item.key + '.zip'); file.remove(false); @@ -631,40 +900,11 @@ Zotero.Sync.Storage.ZFS = (function () { catch (e) { Components.utils.reportError(e); } - } + }), - /** - * Get the storage URI for an item - * - * @inner - * @param {Zotero.Item} - * @return {nsIURI} URI of file on storage server - */ - function getItemURI(item) { - var uri = Zotero.Sync.Storage.ZFS.rootURI; - // Be sure to mirror parameter changes to getItemInfoURI() below - uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1&version=1'; - return uri; - } - - - /** - * Get the storage info URI for an item - * - * @inner - * @param {Zotero.Item} - * @return {nsIURI} URI of file on storage server with info flag - */ - function getItemInfoURI(item) { - var uri = Zotero.Sync.Storage.ZFS.rootURI; - uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1&version=1&info=1'; - return uri; - } - - - function getUploadFile(item) { - if (Zotero.Attachments.getNumFiles(item) > 1) { + _getUploadFile: Zotero.Promise.coroutine(function* (item) { + if (yield Zotero.Attachments.hasMultipleFiles(item)) { var file = Zotero.getTempDirectory(); var filename = item.key + '.zip'; file.append(filename); @@ -673,500 +913,169 @@ Zotero.Sync.Storage.ZFS = (function () { var file = item.getFile(); } return file; - } + }), - // - // Public methods (called via Zotero.Sync.Storage.ZFS) - // - var obj = new Zotero.Sync.Storage.Mode; - obj.name = "ZFS"; - - Object.defineProperty(obj, "includeUserFiles", { - get: function () { - return Zotero.Prefs.get("sync.storage.enabled") && Zotero.Prefs.get("sync.storage.protocol") == 'zotero'; - } - }); - - Object.defineProperty(obj, "includeGroupFiles", { - get: function () { - return Zotero.Prefs.get("sync.storage.groups.enabled"); - } - }); - - obj._verified = true; - - Object.defineProperty(obj, "rootURI", { - get: function () { - if (!_rootURI) { - this._init(); - } - return _rootURI.clone(); - } - }); - - Object.defineProperty(obj, "userURI", { - get: function () { - if (!_userURI) { - this._init(); - } - return _userURI.clone(); - } - }); - - - obj._init = function () { - _rootURI = false; - _userURI = false; - - var url = ZOTERO_CONFIG.API_URL; - var username = Zotero.Sync.Server.username; - var password = Zotero.Sync.Server.password; - - var ios = Components.classes["@mozilla.org/network/io-service;1"]. - getService(Components.interfaces.nsIIOService); - var uri = ios.newURI(url, null, null); - uri.username = encodeURIComponent(username); - uri.password = encodeURIComponent(password); - _rootURI = uri; - - uri = uri.clone(); - uri.spec += 'users/' + Zotero.Users.getCurrentUserID() + '/'; - _userURI = uri; - }; - - obj.clearCachedCredentials = function() { - _rootURI = _userURI = undefined; - _cachedCredentials = false; - }; - /** - * Begin download process for individual file + * Get attachment item metadata on storage server * - * @param {Zotero.Sync.Storage.Request} [request] + * @param {Zotero.Item} item + * @param {Zotero.Sync.Storage.Request} request + * @return {Promise|false} - Promise for object with 'hash', 'filename', 'mtime', + * 'compressed', or false if item not found */ - obj._downloadFile = function (request) { - var item = Zotero.Sync.Storage.getItemFromRequestName(request.name); - if (!item) { - throw new Error("Item '" + request.name + "' not found"); - } + _getStorageFileInfo: Zotero.Promise.coroutine(function* (item, request) { + var funcName = "Zotero.Sync.Storage.ZFS._getStorageFileInfo()"; - var self = this; + var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`); + var uri = this.apiClient.buildRequestURI(params); - // Retrieve file info from server to store locally afterwards - return getStorageFileInfo(item, request) - .then(function (info) { - if (!request.isRunning()) { - Zotero.debug("Download request '" + request.name - + "' is no longer running after getting remote file info"); - return false; - } - - if (!info) { - Zotero.debug("Remote file not found for item " + item.libraryID + "/" + item.key); - return false; - } - - var syncModTime = info.mtime; - var syncHash = info.hash; - - var file = item.getFile(); - // Skip download if local file exists and matches mod time - if (file && file.exists()) { - if (syncModTime == file.lastModifiedTime) { - Zotero.debug("File mod time matches remote file -- skipping download"); - - Zotero.DB.beginTransaction(); - var syncState = Zotero.Sync.Storage.getSyncState(item.id); - //var updateItem = syncState != 1; - var updateItem = false; - Zotero.Sync.Storage.setSyncedModificationTime(item.id, syncModTime, updateItem); - Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); - Zotero.DB.commitTransaction(); - return { - localChanges: true, - remoteChanges: false - }; - } - // If not compressed, check hash, in case only timestamp changed - else if (!info.compressed && item.attachmentHash == syncHash) { - Zotero.debug("File hash matches remote file -- skipping download"); - - Zotero.DB.beginTransaction(); - var syncState = Zotero.Sync.Storage.getSyncState(item.id); - //var updateItem = syncState != 1; - var updateItem = false; - if (!info.compressed) { - Zotero.Sync.Storage.setSyncedHash(item.id, syncHash, false); - } - Zotero.Sync.Storage.setSyncedModificationTime(item.id, syncModTime, updateItem); - Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); - Zotero.DB.commitTransaction(); - return { - localChanges: true, - remoteChanges: false - }; + try { + let req = yield this.apiClient.makeRequest( + "GET", + uri, + { + successCodes: [200, 404], + requestObserver: function (xmlhttp) { + request.setChannel(xmlhttp.channel); } } - - var destFile = Zotero.getTempDirectory(); - if (info.compressed) { - destFile.append(item.key + '.zip.tmp'); - } - else { - destFile.append(item.key + '.tmp'); - } - - if (destFile.exists()) { - try { - destFile.remove(false); - } - catch (e) { - Zotero.File.checkFileAccessError(e, destFile, 'delete'); - } - } - - // saveURI() below appears not to create empty files for Content-Length: 0, - // so we create one here just in case + ); + if (req.status == 404) { + return new Zotero.Sync.Storage.Result; + } + + let info = {}; + info.hash = req.getResponseHeader('ETag'); + if (!info.hash) { + let msg = `Hash not found in info response in ${funcName} (${item.libraryKey})`; + Zotero.debug(msg, 1); + Zotero.debug(req.status); + Zotero.debug(req.responseText); + Components.utils.reportError(msg); try { - destFile.create(Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644); + Zotero.debug(req.getAllResponseHeaders()); } catch (e) { - Zotero.File.checkFileAccessError(e, destFile, 'create'); + Zotero.debug("Response headers unavailable"); } - - var deferred = Zotero.Promise.defer(); - - var listener = new Zotero.Sync.Storage.StreamListener( - { - onStart: function (request, data) { - if (data.request.isFinished()) { - Zotero.debug("Download request " + data.request.name - + " stopped before download started -- closing channel"); - request.cancel(0x804b0002); // NS_BINDING_ABORTED - deferred.resolve(false); - } - }, - onProgress: function (a, b, c) { - request.onProgress(a, b, c) - }, - onStop: function (request, status, response, data) { - data.request.setChannel(false); - - if (status != 200) { - if (status == 404) { - deferred.resolve(false); - return; - } - - if (status == 0) { - if (_s3ConsecutiveFailures >= _maxS3ConsecutiveFailures) { - Zotero.debug(_s3ConsecutiveFailures - + " consecutive S3 failures -- aborting", 1); - _s3ConsecutiveFailures = 0; - } - else { - let libraryKey = Zotero.Items.getLibraryKeyHash(item); - let msg = "S3 returned " + status - + " (" + libraryKey + ") -- retrying download" - Components.utils.reportError(msg); - Zotero.debug(msg, 1); - if (_s3Backoff < _maxS3Backoff) { - _s3Backoff *= 2; - } - _s3ConsecutiveFailures++; - Zotero.debug("Delaying " + libraryKey + " download for " - + _s3Backoff + " seconds", 2); - Q.delay(_s3Backoff * 1000) - .then(function () { - deferred.resolve(self._downloadFile(data.request)); - }); - return; - } - } - - var msg = "Unexpected status code " + status - + " for request " + data.request.name - + " in Zotero.Sync.Storage.ZFS.downloadFile()"; - Zotero.debug(msg, 1); - Components.utils.reportError(msg); - // Ignore files not found in S3 - try { - Zotero.debug(Zotero.File.getContents(destFile, null, 4096), 1); - } - catch (e) { - Zotero.debug(e, 1); - } - deferred.reject(Zotero.Sync.Storage.defaultError); - return; - } - - // Don't try to process if the request has been cancelled - if (data.request.isFinished()) { - Zotero.debug("Download request " + data.request.name - + " is no longer running after file download", 2); - deferred.resolve(false); - return; - } - - Zotero.debug("Finished download of " + destFile.path); - - try { - deferred.resolve(Zotero.Sync.Storage.processDownload(data)); - } - catch (e) { - deferred.reject(e); - } - }, - onCancel: function (request, status, data) { - Zotero.debug("Request cancelled"); - deferred.resolve(false); - }, - request: request, - item: item, - compressed: info.compressed, - syncModTime: syncModTime, - syncHash: syncHash - } - ); - - var uri = getItemURI(item); - - // Don't display password in console - var disp = uri.clone(); - if (disp.password) { - disp.password = "********"; - } - Zotero.debug('Saving ' + disp.spec + ' with saveURI()'); - const nsIWBP = Components.interfaces.nsIWebBrowserPersist; - var wbp = Components - .classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"] - .createInstance(nsIWBP); - wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE; - wbp.progressListener = listener; - Zotero.Utilities.Internal.saveURI(wbp, uri, destFile); - - return deferred.promise; - }); - }; - - - obj._uploadFile = function (request) { - var item = Zotero.Sync.Storage.getItemFromRequestName(request.name); - if (Zotero.Attachments.getNumFiles(item) > 1) { - var deferred = Zotero.Promise.defer(); - var created = Zotero.Sync.Storage.createUploadFile( - request, - function (data) { - if (!data) { - deferred.resolve(false); - return; - } - deferred.resolve(processUploadFile(data)); - } - ); - if (!created) { - return Zotero.Promise.resolve(false); - } - return deferred.promise; - } - else { - return processUploadFile({ request: request }); - } - }; - - - /** - * @return {Promise} A promise for the last sync time - */ - obj._getLastSyncTime = function (libraryID) { - var lastSyncURI = this._getLastSyncURI(libraryID); - - var self = this; - return Zotero.Promise.try(function () { - // Cache the credentials at the root - return self._cacheCredentials(); - }) - .then(function () { - return Zotero.HTTP.promise("GET", lastSyncURI, - { headers: _headers, successCodes: [200, 404], debug: true }); - }) - .then(function (req) { - // Not yet synced - if (req.status == 404) { - Zotero.debug("No last sync time for library " + libraryID); - return null; + let e = Zotero.getString('sync.storage.error.zfs.restart', Zotero.appName); + throw new Error(e); } + info.filename = req.getResponseHeader('X-Zotero-Filename'); + let mtime = req.getResponseHeader('X-Zotero-Modification-Time'); + info.mtime = parseInt(mtime); + info.compressed = req.getResponseHeader('X-Zotero-Compressed') == 'Yes'; + Zotero.debug(info); - var ts = req.responseText; - var date = new Date(ts * 1000); - Zotero.debug("Last successful ZFS sync for library " - + libraryID + " was " + date); - return ts; - }) - .catch(function (e) { + return info; + } + catch (e) { if (e instanceof Zotero.HTTP.UnexpectedStatusException) { - if (e.status == 401 || e.status == 403) { - Zotero.debug("Clearing ZFS authentication credentials", 2); - _cachedCredentials = false; - } - - return Zotero.Promise.reject(e); - } - // TODO: handle browser offline exception - else { - throw e; - } - }); - }; - - - obj._setLastSyncTime = function (libraryID, localLastSyncTime) { - if (localLastSyncTime) { - var sql = "REPLACE INTO version VALUES (?, ?)"; - Zotero.DB.query( - sql, ['storage_zfs_' + libraryID, { int: localLastSyncTime }] - ); - return; - } - - var lastSyncURI = this._getLastSyncURI(libraryID); - - return Zotero.HTTP.promise("POST", lastSyncURI, { headers: _headers, successCodes: [200, 404], debug: true }) - .then(function (req) { - // Not yet synced - // - // TODO: Don't call this at all if no files uploaded - if (req.status == 404) { - return; - } - - var ts = req.responseText; - - var sql = "REPLACE INTO version VALUES (?, ?)"; - Zotero.DB.query( - sql, ['storage_zfs_' + libraryID, { int: ts }] - ); - }) - .catch(function (e) { - var msg = "Unexpected status code " + e.xmlhttp.status - + " setting last file sync time"; - Zotero.debug(msg, 1); - Components.utils.reportError(msg); - throw new Error(Zotero.Sync.Storage.defaultError); - }); - }; - - - obj._getLastSyncURI = function (libraryID) { - if (libraryID === Zotero.Libraries.userLibraryID) { - var lastSyncURI = this.userURI; - } - else if (libraryID) { - var ios = Components.classes["@mozilla.org/network/io-service;1"]. - getService(Components.interfaces.nsIIOService); - var uri = ios.newURI(Zotero.URI.getLibraryURI(libraryID), null, null); - var path = uri.path; - // We don't want the user URI, but it already has the right domain - // and credentials, so just start with that and replace the path - var lastSyncURI = this.userURI; - lastSyncURI.path = path + "/"; - } - else { - throw new Error("libraryID not specified"); - } - lastSyncURI.spec += "laststoragesync"; - return lastSyncURI; - } - - - obj._cacheCredentials = function () { - if (_cachedCredentials) { - Zotero.debug("ZFS credentials are already cached"); - return Zotero.Promise.resolve(); - } - - var uri = this.rootURI; - // TODO: move to root uri - uri.spec += "?auth=1"; - - return Zotero.HTTP.promise("GET", uri, { headers: _headers }). - then(function (req) { - Zotero.debug("Credentials are cached"); - _cachedCredentials = true; - }) - .catch(function (e) { - if (e instanceof Zotero.HTTP.UnexpectedStatusException) { - if (e.status == 401) { - var msg = "File sync login failed\n\n" - + "Check your username and password in the Sync " - + "pane of the Zotero preferences."; - throw (msg); - } - - var msg = "Unexpected status code " + e.status + " " - + "caching ZFS credentials"; - Zotero.debug(msg, 1); - throw (msg); + if (e.xmlhttp.status == 0) { + var msg = "Request cancelled getting storage file info"; } else { - throw (e); + var msg = "Unexpected status code " + e.xmlhttp.status + + " getting storage file info for item " + item.libraryKey; } - }); - }; + Zotero.debug(msg, 1); + Zotero.debug(e.xmlhttp.responseText); + Components.utils.reportError(msg); + throw new Error(Zotero.Sync.Storage.defaultError); + } + + throw e; + } + }), /** - * Remove all synced files from the server + * Upload the file to the server + * + * @param {Zotero.Sync.Storage.Request} request + * @return {Promise} */ - obj._purgeDeletedStorageFiles = function () { - return Zotero.Promise.try(function () { - // Cache the credentials at the root - return this._cacheCredentials(); - }.bind(this)) - then(function () { - // If we don't have a user id we've never synced and don't need to bother - if (!Zotero.Users.getCurrentUserID()) { - return false; - } - - var sql = "SELECT value FROM settings WHERE setting=? AND key=?"; - var values = Zotero.DB.columnQuery(sql, ['storage', 'zfsPurge']); - if (!values) { - return false; - } - - // TODO: promisify - - Zotero.debug("Unlinking synced files on ZFS"); - - var uri = this.userURI; - uri.spec += "removestoragefiles?"; - // Unused - for each(var value in values) { - switch (value) { - case 'user': - uri.spec += "user=1&"; - break; - - case 'group': - uri.spec += "group=1&"; - break; - - default: - throw "Invalid zfsPurge value '" + value - + "' in ZFS purgeDeletedStorageFiles()"; + _processUploadFile: Zotero.Promise.coroutine(function* (request) { + /* + updateSizeMultiplier( + (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100 + ); + */ + + var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request); + + + /*var info = yield this._getStorageFileInfo(item, request); + + if (request.isFinished()) { + Zotero.debug("Upload request '" + request.name + + "' is no longer running after getting file info"); + return false; + } + + // Check for conflict + if ((yield Zotero.Sync.Storage.Local.getSyncState(item.id)) + != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) { + if (info) { + // Local file time + var fmtime = yield item.attachmentModificationTime; + // Remote mod time + var mtime = info.mtime; + + var useLocal = false; + var same = !(yield Zotero.Sync.Storage.checkFileModTime(item, fmtime, mtime)); + + // Ignore maxed-out 32-bit ints, from brief problem after switch to 32-bit servers + if (!same && mtime == 2147483647) { + Zotero.debug("Remote mod time is invalid -- uploading local file version"); + useLocal = true; + } + + if (same) { + yield Zotero.DB.executeTransaction(function* () { + yield Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime); + yield Zotero.Sync.Storage.setSyncState( + item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC + ); + }); + return { + localChanges: true, + remoteChanges: false + }; + } + + let smtime = yield Zotero.Sync.Storage.getSyncedModificationTime(item.id); + if (!useLocal && smtime != mtime) { + Zotero.debug("Conflict -- last synced file mod time " + + "does not match time on storage server" + + " (" + smtime + " != " + mtime + ")"); + return { + localChanges: false, + remoteChanges: false, + conflict: { + local: { modTime: fmtime }, + remote: { modTime: mtime } + } + }; } } - uri.spec = uri.spec.substr(0, uri.spec.length - 1); - - return Zotero.HTTP.promise("POST", uri, "") - .then(function (req) { - var sql = "DELETE FROM settings WHERE setting=? AND key=?"; - Zotero.DB.query(sql, ['storage', 'zfsPurge']); + else { + Zotero.debug("Remote file not found for item " + item.libraryKey); + } + }*/ + + var result = yield this._getFileUploadParameters(item); + if (result.exists) { + yield this._updateItemFileInfo(item, result); + return new Zotero.Sync.Storage.Result({ + localChanges: true, + remoteChanges: true }); - }.bind(this)); - }; - - return obj; -}()); + } + else if (result instanceof Zotero.Sync.Storage.Result) { + return result; + } + return this._uploadFile(request, item, result); + }) +} diff --git a/chrome/content/zotero/xpcom/sync.js b/chrome/content/zotero/xpcom/sync.js index a8779cf68..4f79121d1 100644 --- a/chrome/content/zotero/xpcom/sync.js +++ b/chrome/content/zotero/xpcom/sync.js @@ -1488,43 +1488,6 @@ Zotero.Sync.Server = new function () { -Zotero.BufferedInputListener = function (callback) { - this._callback = callback; -} - -Zotero.BufferedInputListener.prototype = { - binaryInputStream: null, - size: 0, - data: '', - - onStartRequest: function(request, context) {}, - - onStopRequest: function(request, context, status) { - this.binaryInputStream.close(); - delete this.binaryInputStream; - - this._callback(this.data); - }, - - onDataAvailable: function(request, context, inputStream, offset, count) { - this.size += count; - - this.binaryInputStream = Components.classes["@mozilla.org/binaryinputstream;1"] - .createInstance(Components.interfaces.nsIBinaryInputStream) - this.binaryInputStream.setInputStream(inputStream); - this.data += this.binaryInputStream.readBytes(this.binaryInputStream.available()); - }, - - QueryInterface: function (iid) { - if (iid.equals(Components.interfaces.nsISupports) - || iid.equals(Components.interfaces.nsIStreamListener)) { - return this; - } - throw Components.results.NS_ERROR_NO_INTERFACE; - } -} - - Zotero.Sync.Server.Data = new function() { var _noMergeTypes = ['search']; diff --git a/chrome/content/zotero/xpcom/sync/syncAPIClient.js b/chrome/content/zotero/xpcom/sync/syncAPIClient.js index 3c01b44bb..81d3eaf9a 100644 --- a/chrome/content/zotero/xpcom/sync/syncAPIClient.js +++ b/chrome/content/zotero/xpcom/sync/syncAPIClient.js @@ -28,14 +28,15 @@ if (!Zotero.Sync) { } Zotero.Sync.APIClient = function (options) { - this.baseURL = options.baseURL; - this.apiKey = options.apiKey; - this.concurrentCaller = options.concurrentCaller; + if (!options.baseURL) throw new Error("baseURL not set"); + if (!options.apiVersion) throw new Error("apiVersion not set"); + if (!options.apiKey) throw new Error("apiKey not set"); + if (!options.caller) throw new Error("caller not set"); - if (options.apiVersion == undefined) { - throw new Error("options.apiVersion not set"); - } + this.baseURL = options.baseURL; this.apiVersion = options.apiVersion; + this.apiKey = options.apiKey; + this.caller = options.caller; } Zotero.Sync.APIClient.prototype = { @@ -44,7 +45,7 @@ Zotero.Sync.APIClient.prototype = { getKeyInfo: Zotero.Promise.coroutine(function* () { var uri = this.baseURL + "keys/" + this.apiKey; - var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 404] }); + var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 404] }); if (xmlhttp.status == 404) { return false; } @@ -63,7 +64,7 @@ Zotero.Sync.APIClient.prototype = { if (!userID) throw new Error("User ID not provided"); var uri = this.baseURL + "users/" + userID + "/groups?format=versions"; - var xmlhttp = yield this._makeRequest("GET", uri); + var xmlhttp = yield this.makeRequest("GET", uri); return this._parseJSON(xmlhttp.responseText); }), @@ -76,7 +77,7 @@ Zotero.Sync.APIClient.prototype = { if (!groupID) throw new Error("Group ID not provided"); var uri = this.baseURL + "groups/" + groupID; - var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 404] }); + var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 404] }); if (xmlhttp.status == 404) { return false; } @@ -93,7 +94,7 @@ Zotero.Sync.APIClient.prototype = { if (since) { params.since = since; } - var uri = this._buildRequestURI(params); + var uri = this.buildRequestURI(params); var options = { successCodes: [200, 304] }; @@ -102,7 +103,7 @@ Zotero.Sync.APIClient.prototype = { "If-Modified-Since-Version": since }; } - var xmlhttp = yield this._makeRequest("GET", uri, options); + var xmlhttp = yield this.makeRequest("GET", uri, options); if (xmlhttp.status == 304) { return false; } @@ -128,8 +129,8 @@ Zotero.Sync.APIClient.prototype = { libraryTypeID: libraryTypeID, since: since || 0 }; - var uri = this._buildRequestURI(params); - var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 409] }); + var uri = this.buildRequestURI(params); + var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 409] }); if (xmlhttp.status == 409) { Zotero.debug(`'since' value '${since}' is earlier than the beginning of the delete log`); return false; @@ -154,7 +155,7 @@ Zotero.Sync.APIClient.prototype = { * @param {String} libraryType 'user' or 'group' * @param {Integer} libraryTypeID userID or groupID * @param {String} objectType 'item', 'collection', 'search' - * @param {Object} queryParams Query parameters (see _buildRequestURI()) + * @param {Object} queryParams Query parameters (see buildRequestURI()) * @return {Promise|FALSE} Object with 'libraryVersion' and 'results' */ getVersions: Zotero.Promise.coroutine(function* (libraryType, libraryTypeID, objectType, queryParams, libraryVersion) { @@ -176,7 +177,7 @@ Zotero.Sync.APIClient.prototype = { } // TODO: Use pagination - var uri = this._buildRequestURI(params); + var uri = this.buildRequestURI(params); var options = { successCodes: [200, 304] @@ -186,7 +187,7 @@ Zotero.Sync.APIClient.prototype = { "If-Modified-Since-Version": libraryVersion }; } - var xmlhttp = yield this._makeRequest("GET", uri, options); + var xmlhttp = yield this.makeRequest("GET", uri, options); if (xmlhttp.status == 304) { return false; } @@ -256,10 +257,10 @@ Zotero.Sync.APIClient.prototype = { if (objectType == 'item') { params.includeTrashed = 1; } - var uri = this._buildRequestURI(params); + var uri = this.buildRequestURI(params); return [ - this._makeRequest("GET", uri) + this.makeRequest("GET", uri) .then(function (xmlhttp) { return this._parseJSON(xmlhttp.responseText) }.bind(this)) @@ -294,9 +295,9 @@ Zotero.Sync.APIClient.prototype = { libraryType: libraryType, libraryTypeID: libraryTypeID }; - var uri = this._buildRequestURI(params); + var uri = this.buildRequestURI(params); - var xmlhttp = yield this._makeRequest(method, uri, { + var xmlhttp = yield this.makeRequest(method, uri, { headers: { "If-Unmodified-Since-Version": version }, @@ -319,7 +320,7 @@ Zotero.Sync.APIClient.prototype = { }), - _buildRequestURI: function (params) { + buildRequestURI: function (params) { var uri = this.baseURL; switch (params.libraryType) { @@ -332,6 +333,10 @@ Zotero.Sync.APIClient.prototype = { break; } + if (params.target === undefined) { + throw new Error("'target' not provided"); + } + uri += "/" + params.target; if (params.objectKey) { @@ -382,30 +387,33 @@ Zotero.Sync.APIClient.prototype = { }, - _makeRequest: function (method, uri, options) { - if (!options) { - options = {}; + getHeaders: function (headers = {}) { + headers["Zotero-API-Version"] = this.apiVersion; + if (this.apiKey) { + headers["Zotero-API-Key"] = this.apiKey; } - if (!options.headers) { - options.headers = {}; - } - options.headers["Zotero-API-Version"] = this.apiVersion; + return headers; + }, + + + makeRequest: function (method, uri, options = {}) { + options.headers = this.getHeaders(options.headers); options.dontCache = true; options.foreground = !options.background; options.responseType = options.responseType || 'text'; - if (this.apiKey) { - options.headers.Authorization = "Bearer " + this.apiKey; - } - var self = this; - return this.concurrentCaller.fcall(function () { - return Zotero.HTTP.request(method, uri, options) - .catch(function (e) { - if (e instanceof Zotero.HTTP.UnexpectedStatusException) { - self._checkResponse(e.xmlhttp); - } + return this.caller.start(Zotero.Promise.coroutine(function* () { + try { + var xmlhttp = yield Zotero.HTTP.request(method, uri, options); + this._checkBackoff(xmlhttp); + return xmlhttp; + } + catch (e) { + /*if (e instanceof Zotero.HTTP.UnexpectedStatusException) { + this._checkRetry(e.xmlhttp); + }*/ throw e; - }); - }); + } + }.bind(this))); }, @@ -422,21 +430,6 @@ Zotero.Sync.APIClient.prototype = { }, - _checkResponse: function (xmlhttp) { - this._checkBackoff(xmlhttp); - this._checkAuth(xmlhttp); - }, - - - _checkAuth: function (xmlhttp) { - if (xmlhttp.status == 403) { - var e = new Zotero.Error(Zotero.getString('sync.error.invalidLogin'), "INVALID_SYNC_LOGIN"); - e.fatal = true; - throw e; - } - }, - - _checkBackoff: function (xmlhttp) { var backoff = xmlhttp.getResponseHeader("Backoff"); if (backoff) { @@ -444,7 +437,7 @@ Zotero.Sync.APIClient.prototype = { if (backoff > 3600) { // TODO: Update status? - this.concurrentCaller.pause(backoff * 1000); + this.caller.pause(backoff * 1000); } } } diff --git a/chrome/content/zotero/xpcom/sync/syncEngine.js b/chrome/content/zotero/xpcom/sync/syncEngine.js index 91eafb050..714bc1bfa 100644 --- a/chrome/content/zotero/xpcom/sync/syncEngine.js +++ b/chrome/content/zotero/xpcom/sync/syncEngine.js @@ -76,7 +76,13 @@ Zotero.Sync.Data.Engine = function (options) { onError: this.onError } - this.syncCachePromise = Zotero.Promise.resolve().bind(this); + Components.utils.import("resource://zotero/concurrentCaller.js"); + this.syncCacheProcessor = new ConcurrentCaller({ + id: "Sync Cache Processor", + numConcurrent: 1, + logger: Zotero.debug, + stopOnError: this.stopOnError + }); }; Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_SUCCESS = 1; @@ -167,12 +173,8 @@ Zotero.Sync.Data.Engine.prototype.start = Zotero.Promise.coroutine(function* () } } - // TEMP: make more reliable - while (this.syncCachePromise.isPending()) { - Zotero.debug("Waiting for sync cache to be processed"); - yield this.syncCachePromise; - yield Zotero.Promise.delay(50); - } + Zotero.debug("Waiting for sync cache to be processed"); + yield this.syncCacheProcessor.wait(); yield Zotero.Libraries.updateLastSyncTime(this.libraryID); @@ -286,12 +288,7 @@ Zotero.Sync.Data.Engine.prototype._startDownload = Zotero.Promise.coroutine(func } // Wait for sync process to clear - // TEMP: make more reliable - while (this.syncCachePromise.isPending()) { - Zotero.debug("Waiting for sync cache to be processed"); - yield this.syncCachePromise; - yield Zotero.Promise.delay(50); - } + yield this.syncCacheProcessor.wait(); // // Get deleted objects @@ -671,7 +668,8 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi if (state == 'successful') { // Update local object with saved data if necessary - yield obj.fromJSON(current.data); + yield obj.loadAllData(); + obj.fromJSON(current.data); toSave.push(obj); toCache.push(current); } @@ -701,8 +699,11 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi // Handle failed objects for (let index in json.results.failed) { - let e = json.results.failed[index]; - Zotero.logError(e.message); + let { code, message } = json.results.failed[index]; + e = new Error(message); + e.name = "ZoteroUploadObjectError"; + e.code = code; + Zotero.logError(e); // This shouldn't happen, because the upload request includes a library // version and should prevent an outdated upload before the object version is @@ -711,12 +712,11 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi return this.UPLOAD_RESULT_OBJECT_CONFLICT; } - if (this.stopOnError) { - Zotero.debug("WE FAILED!!!"); - throw new Error(e.message); - } if (this.onError) { - this.onError(e.message); + this.onError(e); + } + if (this.stopOnError) { + throw new Error(e); } batch[index].tries++; // Mark 400 errors as permanently failed @@ -990,7 +990,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function* this._failedCheck(); let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); - let ObjectType = objectType[0].toUpperCase() + objectType.substr(1); + let ObjectType = Zotero.Utilities.capitalize(objectType); // TODO: localize this.setStatus("Updating " + objectTypePlural + " in " + this.libraryName); @@ -1037,8 +1037,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function* let cacheVersions = yield Zotero.Sync.Data.Local.getLatestCacheObjectVersions( this.libraryID, objectType ); - // Queue objects that are out of date or don't exist locally and aren't up-to-date - // in the cache + // Queue objects that are out of date or don't exist locally for (let key in results.versions) { let version = results.versions[key]; let obj = yield objectsClass.getByLibraryAndKeyAsync(this.libraryID, key, { @@ -1060,12 +1059,12 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function* } if (obj) { - Zotero.debug(Zotero.Utilities.capitalize(objectType) + " " + obj.libraryKey + Zotero.debug(ObjectType + " " + obj.libraryKey + " is older than version in sync cache"); } else { - Zotero.debug(Zotero.Utilities.capitalize(objectType) + " " - + this.libraryID + "/" + key + " in sync cache not found locally"); + Zotero.debug(ObjectType + " " + this.libraryID + "/" + key + + " in sync cache not found locally"); } toDownload.push(key); @@ -1127,7 +1126,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function* break; } - yield this.syncCachePromise; + yield this.syncCacheProcessor.wait(); yield Zotero.Libraries.setVersion(this.libraryID, lastLibraryVersion); @@ -1145,20 +1144,19 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function* * @param {String} objectType */ Zotero.Sync.Data.Engine.prototype._processCache = function (objectType) { - var self = this; - this.syncCachePromise = this.syncCachePromise.then(function () { - self._failedCheck(); + this.syncCacheProcessor.start(function () { + this._failedCheck(); return Zotero.Sync.Data.Local.processSyncCacheForObjectType( - self.libraryID, objectType, self.options + this.libraryID, objectType, this.options ) .catch(function (e) { Zotero.logError(e); - if (self.stopOnError) { + if (this.stopOnError) { Zotero.debug("WE FAILED!!!"); - self.failed = e; + this.failed = e; } - }); - }) + }.bind(this)); + }.bind(this)) } diff --git a/chrome/content/zotero/xpcom/sync/syncEventListeners.js b/chrome/content/zotero/xpcom/sync/syncEventListeners.js index 9f3fff136..82f8ecbfb 100644 --- a/chrome/content/zotero/xpcom/sync/syncEventListeners.js +++ b/chrome/content/zotero/xpcom/sync/syncEventListeners.js @@ -39,10 +39,9 @@ Zotero.Sync.EventListeners.ChangeListener = new function () { var syncSQL = "REPLACE INTO syncDeleteLog (syncObjectTypeID, libraryID, key, synced) " + "VALUES (?, ?, ?, 0)"; + var storageSQL = "REPLACE INTO storageDeleteLog VALUES (?, ?, 0)"; - if (type == 'item' && Zotero.Sync.Storage.WebDAV.includeUserFiles) { - var storageSQL = "REPLACE INTO storageDeleteLog VALUES (?, ?, 0)"; - } + var storageForLibrary = {}; return Zotero.DB.executeTransaction(function* () { for (let i = 0; i < ids.length; i++) { @@ -74,18 +73,25 @@ Zotero.Sync.EventListeners.ChangeListener = new function () { key ] ); - if (storageSQL && oldItem.itemType == 'attachment' && - [ - Zotero.Attachments.LINK_MODE_IMPORTED_FILE, - Zotero.Attachments.LINK_MODE_IMPORTED_URL - ].indexOf(oldItem.linkMode) != -1) { - yield Zotero.DB.queryAsync( - storageSQL, - [ - libraryID, - key - ] - ); + + if (type == 'item') { + if (storageForLibrary[libraryID] === undefined) { + storageForLibrary[libraryID] = + Zotero.Sync.Storage.Local.getModeForLibrary(libraryID) == 'webdav'; + } + if (storageForLibrary[libraryID] && oldItem.itemType == 'attachment' && + [ + Zotero.Attachments.LINK_MODE_IMPORTED_FILE, + Zotero.Attachments.LINK_MODE_IMPORTED_URL + ].indexOf(oldItem.linkMode) != -1) { + yield Zotero.DB.queryAsync( + storageSQL, + [ + libraryID, + key + ] + ); + } } } }); @@ -215,3 +221,23 @@ Zotero.Sync.EventListeners.progressListener = { } }; + + +Zotero.Sync.EventListeners.StorageFileOpenListener = { + init: function () { + Zotero.Notifier.registerObserver(this, ['file'], 'storageFileOpen'); + }, + + notify: function (event, type, ids, extraData) { + if (event == 'open' && type == 'file') { + let timestamp = new Date().getTime(); + + for (let i = 0; i < ids.length; i++) { + Zotero.Sync.Storage.Local.uploadCheckFiles.push({ + itemID: ids[i], + timestamp: timestamp + }); + } + } + } +} diff --git a/chrome/content/zotero/xpcom/sync/syncLocal.js b/chrome/content/zotero/xpcom/sync/syncLocal.js index a5ba10f7a..104958e03 100644 --- a/chrome/content/zotero/xpcom/sync/syncLocal.js +++ b/chrome/content/zotero/xpcom/sync/syncLocal.js @@ -28,6 +28,8 @@ if (!Zotero.Sync.Data) { } Zotero.Sync.Data.Local = { + _loginManagerHost: 'https://api.zotero.org', + _loginManagerRealm: 'Zotero Web API', _lastSyncTime: null, _lastClassicSyncTime: null, @@ -39,6 +41,71 @@ Zotero.Sync.Data.Local = { }), + getAPIKey: function () { + var apiKey = Zotero.Prefs.get('devAPIKey'); + if (apiKey) { + return apiKey; + } + var loginManager = Components.classes["@mozilla.org/login-manager;1"] + .getService(Components.interfaces.nsILoginManager); + var logins = loginManager.findLogins( + {}, this._loginManagerHost, null, this._loginManagerRealm + ); + // Get API from returned array of nsILoginInfo objects + if (logins.length) { + return logins[0].password; + } + if (!apiKey) { + let username = Zotero.Prefs.get('sync.server.username'); + if (username) { + let password = Zotero.Sync.Data.Local.getLegacyPassword(username); + if (!password) { + return false; + } + throw new Error("Unimplemented"); + // Get API key from server + + // Store API key + + // Remove old logins and username pref + } + } + return apiKey; + }, + + + setAPIKey: function (apiKey) { + var loginManager = Components.classes["@mozilla.org/login-manager;1"] + .getService(Components.interfaces.nsILoginManager); + var nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1", + Components.interfaces.nsILoginInfo, "init"); + var loginInfo = new nsLoginInfo( + this._loginManagerHost, + null, + this._loginManagerRealm, + 'API Key', + apiKey, + "", + "" + ); + loginManager.addLogin(loginInfo); + }, + + + getLegacyPassword: function (username) { + var loginManager = Components.classes["@mozilla.org/login-manager;1"] + .getService(Components.interfaces.nsILoginManager); + var logins = loginManager.findLogins({}, "chrome://zotero", "Zotero Storage Server", null); + // Find user from returned array of nsILoginInfo objects + for (let login of logins) { + if (login.username == username) { + return login.password; + } + } + return false; + }, + + getLastSyncTime: function () { if (_lastSyncTime === null) { throw new Error("Last sync time not yet loaded"); @@ -86,7 +153,7 @@ Zotero.Sync.Data.Local = { var sql = "SELECT " + objectsClass.idColumn + " FROM " + objectsClass.table + " WHERE libraryID=? AND synced=0"; - // RETRIEVE PARENT DOWN? EVEN POSSIBLE? + // TODO: RETRIEVE PARENT DOWN? EVEN POSSIBLE? // items via parent // collections via getDescendents? @@ -154,6 +221,35 @@ Zotero.Sync.Data.Local = { }), + getCacheObjects: Zotero.Promise.coroutine(function* (objectType, libraryID, keyVersionPairs) { + if (!keyVersionPairs.length) return []; + var sql = "SELECT data FROM syncCache SC JOIN (SELECT " + + keyVersionPairs.map(function (pair) { + Zotero.DataObjectUtilities.checkKey(pair[0]); + return "'" + pair[0] + "' AS key, " + parseInt(pair[1]) + " AS version"; + }).join(" UNION SELECT ") + + ") AS pairs ON (pairs.key=SC.key AND pairs.version=SC.version) " + + "WHERE libraryID=? AND " + + "syncObjectTypeID IN (SELECT syncObjectTypeID FROM syncObjectTypes WHERE name=?)"; + var rows = yield Zotero.DB.columnQueryAsync(sql, [libraryID, objectType]); + return rows.map(row => JSON.parse(row)); + }), + + + saveCacheObject: Zotero.Promise.coroutine(function* (objectType, libraryID, json) { + json = this._checkCacheJSON(json); + + Zotero.debug("Saving to sync cache:"); + Zotero.debug(json); + + var syncObjectTypeID = Zotero.Sync.Data.Utilities.getSyncObjectTypeID(objectType); + var sql = "INSERT OR REPLACE INTO syncCache " + + "(libraryID, key, syncObjectTypeID, version, data) VALUES (?, ?, ?, ?, ?)"; + var params = [libraryID, json.key, syncObjectTypeID, json.version, JSON.stringify(json)]; + return Zotero.DB.queryAsync(sql, params); + }), + + saveCacheObjects: Zotero.Promise.coroutine(function* (objectType, libraryID, jsonArray) { if (!Array.isArray(jsonArray)) { throw new Error("'json' must be an array"); @@ -165,20 +261,7 @@ Zotero.Sync.Data.Local = { return; } - jsonArray = jsonArray.map(o => { - if (o.key === undefined) { - throw new Error("Missing 'key' property in JSON"); - } - if (o.version === undefined) { - throw new Error("Missing 'version' property in JSON"); - } - // If direct data object passed, wrap in fake response object - return o.data === undefined ? { - key: o.key, - version: o.version, - data: o - } : o; - }); + jsonArray = jsonArray.map(json => this._checkCacheJSON(json)); Zotero.debug("Saving to sync cache:"); Zotero.debug(jsonArray); @@ -206,6 +289,22 @@ Zotero.Sync.Data.Local = { }), + _checkCacheJSON: function (json) { + if (json.key === undefined) { + throw new Error("Missing 'key' property in JSON"); + } + if (json.version === undefined) { + throw new Error("Missing 'version' property in JSON"); + } + // If direct data object passed, wrap in fake response object + return json.data === undefined ? { + key: json.key, + version: json.version, + data: json + } : json; + }, + + processSyncCache: Zotero.Promise.coroutine(function* (libraryID, options) { for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(libraryID)) { yield this.processSyncCacheForObjectType(libraryID, objectType, options); @@ -213,8 +312,7 @@ Zotero.Sync.Data.Local = { }), - processSyncCacheForObjectType: Zotero.Promise.coroutine(function* (libraryID, objectType, options) { - options = options || {}; + processSyncCacheForObjectType: Zotero.Promise.coroutine(function* (libraryID, objectType, options = {}) { var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); var ObjectType = Zotero.Utilities.capitalize(objectType); @@ -227,7 +325,6 @@ Zotero.Sync.Data.Local = { var numSkipped = 0; var data = yield this._getUnwrittenData(libraryID, objectType); - if (!data.length) { Zotero.debug("No unwritten " + objectTypePlural + " in sync cache"); return; @@ -260,9 +357,9 @@ Zotero.Sync.Data.Local = { for (let i = 0; i < chunk.length; i++) { let json = chunk[i]; let jsonData = json.data; - let isNewObject; let objectKey = json.key; + Zotero.debug(`Processing ${objectType} ${libraryID}/${objectKey}`); Zotero.debug(json); if (!jsonData) { @@ -302,26 +399,22 @@ Zotero.Sync.Data.Local = { }*/ } + let isNewObject = false; + let skipCache = false; let obj = yield objectsClass.getByLibraryAndKeyAsync( libraryID, objectKey, { noCache: true } ); if (obj) { Zotero.debug("Matching local " + objectType + " exists", 4); - isNewObject = false; - // Local object has not been modified since last sync - if (obj.synced) { - // Overwrite local below - } - else { + // Local object has been modified since last sync + if (!obj.synced) { Zotero.debug("Local " + objectType + " " + obj.libraryKey + " has been modified since last sync", 4); let cachedJSON = yield this.getCacheObject( objectType, obj.libraryID, obj.key, obj.version ); - Zotero.debug("GOT CACHED"); - Zotero.debug(cachedJSON); let jsonDataLocal = yield obj.toJSON(); @@ -333,42 +426,51 @@ Zotero.Sync.Data.Local = { ['dateAdded', 'dateModified'] ); - // If no changes, update local version and keep as unsynced + // If no changes, update local version number and mark as synced if (!result.changes.length && !result.conflicts.length) { - Zotero.debug("No remote changes to apply to local " + objectType - + " " + obj.libraryKey); - yield obj.updateVersion(json.version); + Zotero.debug("No remote changes to apply to local " + + objectType + " " + obj.libraryKey); + obj.version = json.version; + obj.synced = true; + yield obj.save(); + continue; + } + + if (result.conflicts.length) { + if (objectType != 'item') { + throw new Error(`Unexpected conflict on ${objectType} object`); + } + Zotero.debug("Conflict!"); + conflicts.push({ + left: jsonDataLocal, + right: jsonData, + changes: result.changes, + conflicts: result.conflicts + }); continue; } // If no conflicts, apply remote changes automatically - if (!result.conflicts.length) { - Zotero.DataObjectUtilities.applyChanges( - jsonData, result.changes - ); - let saved = yield this._saveObjectFromJSON(obj, jsonData, options); - if (saved) numSaved++; - continue; - } - - if (objectType != 'item') { - throw new Error(`Unexpected conflict on ${objectType} object`); - } - - conflicts.push({ - left: jsonDataLocal, - right: jsonData, - changes: result.changes, - conflicts: result.conflicts - }); - continue; + Zotero.debug(`Applying remote changes to ${objectType} ` + + obj.libraryKey); + Zotero.debug(result.changes); + Zotero.DataObjectUtilities.applyChanges( + jsonDataLocal, result.changes + ); + // Transfer properties that aren't in the changeset + ['version', 'dateAdded', 'dateModified'].forEach(x => { + if (jsonDataLocal[x] !== jsonData[x]) { + Zotero.debug(`Applying remote '${x}' value`); + } + jsonDataLocal[x] = jsonData[x]; + }) + jsonData = jsonDataLocal; } - - let saved = yield this._saveObjectFromJSON(obj, jsonData, options); - if (saved) numSaved++; } // Object doesn't exist locally else { + Zotero.debug(ObjectType + " doesn't exist locally"); + isNewObject = true; // Check if object has been deleted locally @@ -376,6 +478,8 @@ Zotero.Sync.Data.Local = { objectType, libraryID, objectKey ); if (dateDeleted) { + Zotero.debug(ObjectType + " was deleted locally"); + switch (objectType) { case 'item': conflicts.push({ @@ -410,24 +514,30 @@ Zotero.Sync.Data.Local = { obj.key = objectKey; yield obj.loadPrimaryData(); - let saved = yield this._saveObjectFromJSON(obj, jsonData, options, { - // Don't cache new items immediately, which skips reloading after save - skipCache: true - }); - if (saved) numSaved++; + // Don't cache new items immediately, which skips reloading after save + skipCache = true; + } + + let saved = yield this._saveObjectFromJSON( + obj, jsonData, options, { skipCache } + ); + // Mark updated attachments for download + if (saved && objectType == 'item' && obj.isImportedAttachment()) { + yield this._checkAttachmentForDownload( + obj, jsonData.mtime, isNewObject + ); + } + + if (saved) { + numSaved++; } } }.bind(this)); }.bind(this) ); - // Keep retrying if we skipped any, as long as we're still making progress - if (numSkipped && numSaved != 0) { - Zotero.debug("More " + objectTypePlural + " in cache -- continuing"); - yield this.processSyncCacheForObjectType(libraryID, objectType, options); - } - if (conflicts.length) { + // Sort conflicts by local Date Modified/Deleted conflicts.sort(function (a, b) { var d1 = a.left.dateDeleted || a.left.dateModified; var d2 = b.left.dateDeleted || b.left.dateModified; @@ -442,6 +552,7 @@ Zotero.Sync.Data.Local = { var mergeData = this.resolveConflicts(conflicts); if (mergeData) { + Zotero.debug("Processing resolved conflicts"); let mergeOptions = {}; Object.assign(mergeOptions, options); // Tell _saveObjectFromJSON not to save with 'synced' set to true @@ -484,11 +595,55 @@ Zotero.Sync.Data.Local = { } } + // Keep retrying if we skipped any, as long as we're still making progress + if (numSkipped && numSaved != 0) { + Zotero.debug("More " + objectTypePlural + " in cache -- continuing"); + return this.processSyncCacheForObjectType(libraryID, objectType, options); + } + data = yield this._getUnwrittenData(libraryID, objectType); - Zotero.debug("Skipping " + data.length + " " - + (data.length == 1 ? objectType : objectTypePlural) - + " in sync cache"); - return data; + if (data.length) { + Zotero.debug(`Skipping ${data.length} ` + + (data.length == 1 ? objectType : objectTypePlural) + + " in sync cache"); + } + }), + + + _checkAttachmentForDownload: Zotero.Promise.coroutine(function* (item, mtime, isNewObject) { + var markToDownload = false; + if (!isNewObject) { + // Convert previously used Unix timestamps to ms-based timestamps + if (mtime < 10000000000) { + Zotero.debug("Converting Unix timestamp '" + mtime + "' to ms"); + mtime = mtime * 1000; + } + var fmtime = null; + try { + fmtime = yield item.attachmentModificationTime; + } + catch (e) { + // This will probably fail later too, but ignore it for now + Zotero.logError(e); + } + if (fmtime) { + let state = Zotero.Sync.Storage.Local.checkFileModTime(item, fmtime, mtime); + if (state !== false) { + markToDownload = true; + } + } + else { + markToDownload = true; + } + } + else { + markToDownload = true; + } + if (markToDownload) { + yield Zotero.Sync.Storage.Local.setSyncState( + item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD + ); + } }), @@ -501,6 +656,8 @@ Zotero.Sync.Data.Local = { resolveConflicts: function (conflicts) { + Zotero.debug("Showing conflict resolution window"); + var io = { dataIn: { captions: [ @@ -511,9 +668,7 @@ Zotero.Sync.Data.Local = { conflicts } }; - var url = 'chrome://zotero/content/merge.xul'; - var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] .getService(Components.interfaces.nsIWindowMediator); var lastWin = wm.getMostRecentWindow("navigator:browser"); @@ -553,7 +708,8 @@ Zotero.Sync.Data.Local = { _saveObjectFromJSON: Zotero.Promise.coroutine(function* (obj, json, options) { try { - yield obj.fromJSON(json); + yield obj.loadAllData(); + obj.fromJSON(json); if (!options.saveAsChanged) { obj.version = json.version; obj.synced = true; @@ -611,6 +767,11 @@ Zotero.Sync.Data.Local = { var changeset1 = Zotero.DataObjectUtilities.diff(originalJSON, currentJSON, ignoreFields); var changeset2 = Zotero.DataObjectUtilities.diff(originalJSON, newJSON, ignoreFields); + Zotero.debug("CHANGESET1"); + Zotero.debug(changeset1); + Zotero.debug("CHANGESET2"); + Zotero.debug(changeset2); + var conflicts = []; for (let i = 0; i < changeset1.length; i++) { @@ -725,27 +886,43 @@ Zotero.Sync.Data.Local = { var conflicts = []; for (let i = 0; i < changeset.length; i++) { - let c = changeset[i]; + let c2 = changeset[i]; // Member changes are additive only, so ignore removals - if (c.op.endsWith('-remove')) { + if (c2.op.endsWith('-remove')) { continue; } // Record member changes - if (c.op.startsWith('member-') || c.op.startsWith('property-member-')) { - changes.push(c); + if (c2.op.startsWith('member-') || c2.op.startsWith('property-member-')) { + changes.push(c2); continue; } // Automatically apply remote changes for non-items, even if in conflict if (objectType != 'item') { - changes.push(c); + changes.push(c2); continue; } // Field changes are conflicts - conflicts.push(c); + // + // Since we don't know what changed, use only 'add' and 'delete' + if (c2.op == 'modify') { + c2.op = 'add'; + } + let val = currentJSON[c2.field]; + let c1 = { + field: c2.field, + op: val !== undefined ? 'add' : 'delete' + }; + if (val !== undefined) { + c1.value = val; + } + if (c2.op == 'modify') { + c2.op = 'add'; + } + conflicts.push([c1, c2]); } return { changes, conflicts }; diff --git a/chrome/content/zotero/xpcom/sync/syncRunner.js b/chrome/content/zotero/xpcom/sync/syncRunner.js index 430639a22..e440b07f1 100644 --- a/chrome/content/zotero/xpcom/sync/syncRunner.js +++ b/chrome/content/zotero/xpcom/sync/syncRunner.js @@ -29,34 +29,62 @@ if (!Zotero.Sync) { Zotero.Sync = {}; } -Zotero.Sync.Runner_Module = function () { +// Initialized as Zotero.Sync.Runner in zotero.js +Zotero.Sync.Runner_Module = function (options = {}) { + const stopOnError = true; + Zotero.defineProperty(this, 'background', { get: () => _background }); Zotero.defineProperty(this, 'lastSyncStatus', { get: () => _lastSyncStatus }); - const stopOnError = true; + this.baseURL = options.baseURL || ZOTERO_CONFIG.API_URL; + this.apiVersion = options.apiVersion || ZOTERO_CONFIG.API_VERSION; + this.apiKey = options.apiKey || Zotero.Sync.Data.Local.getAPIKey(); + + Components.utils.import("resource://zotero/concurrentCaller.js"); + this.caller = new ConcurrentCaller(4); + this.caller.setLogger(msg => Zotero.debug(msg)); + this.caller.stopOnError = stopOnError; + this.caller.onError = function (e) { + this.addError(e); + if (e.fatal) { + this.caller.stop(); + throw e; + } + }.bind(this); var _autoSyncTimer; var _background; var _firstInSession = true; var _syncInProgress = false; + var _syncEngines = []; + var _storageEngines = []; + var _lastSyncStatus; var _currentSyncStatusLabel; var _currentLastSyncLabel; var _errors = []; + this.getAPIClient = function () { + return new Zotero.Sync.APIClient({ + baseURL: this.baseURL, + apiVersion: this.apiVersion, + apiKey: this.apiKey, + caller: this.caller + }); + } + + /** * Begin a sync session * - * @param {Object} [options] - * @param {String} [apiKey] - * @param {Boolean} [background=false] - Whether this is a background request, which prevents - * some alerts from being shown - * @param {String} [baseURL] - * @param {Integer[]} [libraries] - IDs of libraries to sync - * @param {Function} [onError] - Function to pass errors to instead of handling internally - * (used for testing) + * @param {Object} [options] + * @param {Boolean} [options.background=false] Whether this is a background request, which + * prevents some alerts from being shown + * @param {Integer[]} [options.libraries] IDs of libraries to sync + * @param {Function} [options.onError] Function to pass errors to instead of + * handling internally (used for testing) */ this.sync = Zotero.Promise.coroutine(function* (options = {}) { // Clear message list @@ -84,14 +112,13 @@ Zotero.Sync.Runner_Module = function () { // Purge deleted objects so they don't cause sync errors (e.g., long tags) yield Zotero.purgeDataObjects(true); - options.apiKey = options.apiKey || Zotero.Prefs.get('devAPIKey'); - if (!options.apiKey) { - let msg = "API key not provided"; + if (!this.apiKey) { + let msg = "API key not set"; let e = new Zotero.Error(msg, 0, { dialogButtonText: null }) this.updateIcons(e); + _syncInProgress = false; return false; } - options.baseURL = options.baseURL || ZOTERO_CONFIG.API_URL; if (_firstInSession) { options.firstInSession = true; _firstInSession = false; @@ -102,66 +129,45 @@ Zotero.Sync.Runner_Module = function () { this.updateIcons('animate'); try { - Components.utils.import("resource://zotero/concurrent-caller.js"); - var caller = new ConcurrentCaller(4); // TEMP: one for now - caller.setLogger(msg => Zotero.debug(msg)); - caller.stopOnError = stopOnError; - caller.onError = function (e) { - this.addError(e); - if (e.fatal) { - caller.stop(); - throw e; - } - }.bind(this); + let client = this.getAPIClient(); - // TODO: Use a single client for all operations? - var client = new Zotero.Sync.APIClient({ - baseURL: options.baseURL, - apiVersion: ZOTERO_CONFIG.API_VERSION, - apiKey: options.apiKey, - concurrentCaller: caller, - background: options.background - }); - - var keyInfo = yield this.checkAccess(client, options); + let keyInfo = yield this.checkAccess(client, options); if (!keyInfo) { - this.stop(); + this.end(); Zotero.debug("Syncing cancelled"); return false; } - var libraries = yield this.checkLibraries(client, options, keyInfo, libraries); - - for (let libraryID of libraries) { - try { - let engine = new Zotero.Sync.Data.Engine({ - libraryID: libraryID, - apiClient: client, - setStatus: this.setSyncStatus.bind(this), - stopOnError: stopOnError, - onError: this.addError.bind(this) - }); - yield engine.start(); - } - catch (e) { - Zotero.debug("Sync failed for library " + libraryID); - Zotero.debug(e, 1); - Components.utils.reportError(e); - this.checkError(e); + let engineOptions = { + apiClient: client, + caller: this.caller, + setStatus: this.setSyncStatus.bind(this), + stopOnError, + onError: function (e) { if (options.onError) { options.onError(e); } else { - this.addError(e); + this.addError.bind(this); } - if (stopOnError || e.fatal) { - caller.stop(); - break; - } - } - } + }.bind(this), + background: _background, + firstInSession: _firstInSession + }; - yield Zotero.Sync.Data.Local.updateLastSyncTime(); + let nextLibraries = yield this.checkLibraries( + client, options, keyInfo, options.libraries + ); + // Sync data, files, and then any data that needs to be uploaded + let attempt = 1; + while (nextLibraries.length) { + if (attempt > 3) { + throw new Error("Too many sync attempts -- stopping"); + } + nextLibraries = yield _doDataSync(nextLibraries, engineOptions); + nextLibraries = yield _doFileSync(nextLibraries, engineOptions); + attempt++; + } } catch (e) { if (options.onError) { @@ -171,62 +177,19 @@ Zotero.Sync.Runner_Module = function () { this.addError(e); } } - - this.stop(); + finally { + this.end(); + } Zotero.debug("Done syncing"); + /*if (results.changesMade) { + Zotero.debug("Changes made during file sync " + + "-- performing additional data sync"); + this.sync(options); + }*/ + return; - - var storageSync = function () { - Zotero.Sync.Runner.setSyncStatus(Zotero.getString('sync.status.syncingFiles')); - - Zotero.Sync.Storage.sync(options) - .then(function (results) { - Zotero.debug("File sync is finished"); - - if (results.errors.length) { - Zotero.debug(results.errors, 1); - for each(var e in results.errors) { - Components.utils.reportError(e); - } - Zotero.Sync.Runner.setErrors(results.errors); - return; - } - - if (results.changesMade) { - Zotero.debug("Changes made during file sync " - + "-- performing additional data sync"); - Zotero.Sync.Server.sync(finalCallbacks); - } - else { - Zotero.Sync.Runner.stop(); - } - }) - .catch(function (e) { - Zotero.debug("File sync failed", 1); - Zotero.Sync.Runner.error(e); - }) - .done(); - }; - - Zotero.Sync.Server.sync({ - // Sync 1 success - onSuccess: storageSync, - - // Sync 1 skip - onSkip: storageSync, - - // Sync 1 stop - onStop: function () { - Zotero.Sync.Runner.stop(); - }, - - // Sync 1 error - onError: function (e) { - Zotero.Sync.Runner.error(e); - } - }); }); @@ -242,8 +205,9 @@ Zotero.Sync.Runner_Module = function () { } // Sanity check - if (!json.userID) throw new Error("userID not found in response"); - if (!json.username) throw new Error("username not found in response"); + if (!json.userID) throw new Error("userID not found in key response"); + if (!json.username) throw new Error("username not found in key response"); + if (!json.access) throw new Error("'access' not found in key response"); // Make sure user hasn't changed, and prompt to update database if so if (!(yield this.checkUser(json.userID, json.username))) { @@ -446,8 +410,6 @@ Zotero.Sync.Runner_Module = function () { * * @param {Integer} userID New userID * @param {Integer} libraryID New libraryID - * @param {Integer} noServerData The server account is empty — this is - * the account after a server clear * @return {Boolean} - True to continue, false to cancel */ this.checkUser = Zotero.Promise.coroutine(function* (userID, username) { @@ -544,7 +506,154 @@ Zotero.Sync.Runner_Module = function () { }); + var _doDataSync = Zotero.Promise.coroutine(function* (libraries, options, skipUpdateLastSyncTime) { + var successfulLibraries = []; + for (let libraryID of libraries) { + try { + let opts = {}; + Object.assign(opts, options); + opts.libraryID = libraryID; + + let engine = new Zotero.Sync.Data.Engine(opts); + yield engine.start(); + successfulLibraries.push(libraryID); + } + catch (e) { + Zotero.debug("Sync failed for library " + libraryID); + Zotero.logError(e); + this.checkError(e); + if (options.onError) { + options.onError(e); + } + else { + this.addError(e); + } + if (stopOnError || e.fatal) { + Zotero.debug("Stopping on error", 1); + options.caller.stop(); + break; + } + } + } + // Update last-sync time if any libraries synced + // TEMP: Do we want to show updated time if some libraries haven't synced? + if (!libraries.length || successfulLibraries.length) { + yield Zotero.Sync.Data.Local.updateLastSyncTime(); + } + return successfulLibraries; + }.bind(this)); + + + var _doFileSync = Zotero.Promise.coroutine(function* (libraries, options) { + Zotero.debug("Starting file syncing"); + this.setSyncStatus(Zotero.getString('sync.status.syncingFiles')); + let librariesToSync = []; + for (let libraryID of libraries) { + try { + let opts = {}; + Object.assign(opts, options); + opts.libraryID = libraryID; + + let tries = 3; + while (true) { + if (tries == 0) { + throw new Error("Too many file sync attempts for library " + libraryID); + } + tries--; + let engine = new Zotero.Sync.Storage.Engine(opts); + let results = yield engine.start(); + if (results.syncRequired) { + librariesToSync.push(libraryID); + } + else if (results.fileSyncRequired) { + Zotero.debug("Another file sync required -- restarting"); + continue; + } + break; + } + } + catch (e) { + Zotero.debug("File sync failed for library " + libraryID); + Zotero.debug(e, 1); + Components.utils.reportError(e); + this.checkError(e); + if (options.onError) { + options.onError(e); + } + else { + this.addError(e); + } + if (stopOnError || e.fatal) { + options.caller.stop(); + break; + } + } + } + Zotero.debug("Done with file syncing"); + return librariesToSync; + }.bind(this)); + + + /** + * Download a single file on demand (not within a sync process) + */ + this.downloadFile = Zotero.Promise.coroutine(function* (item, requestCallbacks) { + if (Zotero.HTTP.browserIsOffline()) { + Zotero.debug("Browser is offline", 2); + return false; + } + + // TEMP + var options = {}; + + var itemID = item.id; + var modeClass = Zotero.Sync.Storage.Local.getClassForLibrary(item.libraryID); + var controller = new modeClass({ + apiClient: this.getAPIClient() + }); + + // TODO: verify WebDAV on-demand? + if (!controller.verified) { + Zotero.debug("File syncing is not active for item's library -- skipping download"); + return false; + } + + if (!item.isImportedAttachment()) { + throw new Error("Not an imported attachment"); + } + + if (yield item.getFilePathAsync()) { + Zotero.debug("File already exists -- replacing"); + } + + // TODO: start sync icon? + // TODO: create queue for cancelling + + if (!requestCallbacks) { + requestCallbacks = {}; + } + var onStart = function (request) { + return controller.downloadFile(request); + }; + var request = new Zotero.Sync.Storage.Request({ + type: 'download', + libraryID: item.libraryID, + name: item.libraryKey, + onStart: requestCallbacks.onStart + ? [onStart, requestCallbacks.onStart] + : [onStart] + }); + return request.start(); + }); + + this.stop = function () { + _syncEngines.forEach(engine => engine.stop()); + _storageEngines.forEach(engine => engine.stop()); + } + + + this.end = function () { this.updateIcons(_errors); _errors = []; _syncInProgress = false; @@ -669,7 +778,6 @@ Zotero.Sync.Runner_Module = function () { if (libraryID) { e.libraryID = libraryID; } - Zotero.logError(e); _errors.push(this.parseError(e)); } @@ -1027,7 +1135,8 @@ Zotero.Sync.Runner_Module = function () { var lastSyncTime = Zotero.Sync.Data.Local.getLastSyncTime(); if (!lastSyncTime) { try { - lastSyncTime = Zotero.Sync.Data.Local.getLastClassicSyncTime() + lastSyncTime = Zotero.Sync.Data.Local.getLastClassicSyncTime(); + Zotero.debug(lastSyncTime); } catch (e) { Zotero.debug(e, 2); @@ -1052,5 +1161,3 @@ Zotero.Sync.Runner_Module = function () { _currentLastSyncLabel.hidden = false; } } - -Zotero.Sync.Runner = new Zotero.Sync.Runner_Module; diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index 1a6f09e6b..7cbb0ee92 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -607,6 +607,7 @@ Components.utils.import("resource://gre/modules/osfile.jsm"); yield Zotero.Sync.Data.Local.init(); yield Zotero.Sync.Data.Utilities.init(); Zotero.Sync.EventListeners.init(); + Zotero.Sync.Runner = new Zotero.Sync.Runner_Module; Zotero.MIMETypeHandler.init(); yield Zotero.Proxies.init(); @@ -2706,6 +2707,9 @@ Zotero.Browser = new function() { if(!win) { var win = Services.ww.activeWindow; } + if (!win) { + throw new Error("Parent window not available for hidden browser"); + } } // Create a hidden browser diff --git a/chrome/content/zotero/zoteroPane.js b/chrome/content/zotero/zoteroPane.js index 7840b2444..0126a10cb 100644 --- a/chrome/content/zotero/zoteroPane.js +++ b/chrome/content/zotero/zoteroPane.js @@ -1325,6 +1325,7 @@ var ZoteroPane = new function() else if (item.isAttachment()) { var attachmentBox = document.getElementById('zotero-attachment-box'); attachmentBox.mode = this.collectionsView.editable ? 'edit' : 'view'; + yield item.loadNote(); attachmentBox.item = item; document.getElementById('zotero-item-pane-content').selectedIndex = 3; @@ -3692,38 +3693,41 @@ var ZoteroPane = new function() } } else { - if (!item.isImportedAttachment() || !Zotero.Sync.Storage.downloadAsNeeded(item.libraryID)) { + if (!item.isImportedAttachment() + || !Zotero.Sync.Storage.Local.downloadAsNeeded(item.libraryID)) { this.showAttachmentNotFoundDialog(itemID, noLocateOnMissing); return; } let downloadedItem = item; - yield Zotero.Sync.Storage.downloadFile( - downloadedItem, - { - onProgress: function (progress, progressMax) {} - } - ) - .then(function () { - if (!downloadedItem.getFile()) { - ZoteroPane_Local.showAttachmentNotFoundDialog(downloadedItem.id, noLocateOnMissing); - return; - } - - // check if unchanged? - // maybe not necessary, since we'll get an error if there's an error - - - Zotero.Notifier.trigger('redraw', 'item', []); - Zotero.debug('downloaded'); - Zotero.debug(downloadedItem.id); - return ZoteroPane_Local.viewAttachment(downloadedItem.id, event, false, forceExternalViewer); - }) - .catch(function (e) { + try { + yield Zotero.Sync.Runner.downloadFile( + downloadedItem, + { + onProgress: function (progress, progressMax) {} + } + ); + } + catch (e) { // TODO: show error somewhere else Zotero.debug(e, 1); ZoteroPane_Local.syncAlert(e); - }); + return; + } + + if (!(yield downloadedItem.getFilePathAsync())) { + ZoteroPane_Local.showAttachmentNotFoundDialog(downloadedItem.id, noLocateOnMissing); + return; + } + + // check if unchanged? + // maybe not necessary, since we'll get an error if there's an error + + + Zotero.Notifier.trigger('redraw', 'item', []); + Zotero.debug('downloaded'); + Zotero.debug(downloadedItem.id); + return ZoteroPane_Local.viewAttachment(downloadedItem.id, event, false, forceExternalViewer); } } }); @@ -3962,7 +3966,7 @@ var ZoteroPane = new function() this.syncAlert = function (e) { - e = Zotero.Sync.Runner.parseSyncError(e); + e = Zotero.Sync.Runner.parseError(e); var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"] .getService(Components.interfaces.nsIPromptService); diff --git a/chrome/content/zotero/zoteroPane.xul b/chrome/content/zotero/zoteroPane.xul index 4f6b73121..003a2da04 100644 --- a/chrome/content/zotero/zoteroPane.xul +++ b/chrome/content/zotero/zoteroPane.xul @@ -195,9 +195,10 @@