suzanne.soy/zotero
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ZFS file sync overhaul for API syncing

Browse Source 
This mostly gets ZFS file syncing and file conflict resolution working
with the API sync process. WebDAV will need to be updated separately.

Known issues:

- File sync progress is temporarily gone
- File uploads can result in an unnecessary 412 loop on the next data
  sync
- This causes Firefox to crash on one of my computers during tests,
  which would be easier to debug if it produced a crash log.

Also:

- Adds httpd.js for use in tests when FakeXMLHttpRequest can't be used
  (e.g., saveURI()).
- Adds some additional test data files for attachment tests
This commit is contained in:
Dan Stillman 2015-10-29 03:41:54 -04:00
parent 6d46b06617
commit 73f4d28ab2
44 changed files with 11226 additions and 5239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

371
chrome/content/zotero/bindings/attachmentbox.xml
View File

@ -57,6 +57,7 @@
Zotero.debug("Setting mode to '" + val + "'");
this.editable = false;
this.synchronous = false;
this.displayURL = false;
this.displayFileName = false;
this.clickableLink = false;
@ -93,6 +94,7 @@
break;
case 'merge':
this.synchronous = true;
this.displayURL = true;
this.displayFileName = true;
this.displayAccessed = true;
@ -102,6 +104,7 @@
break;
case 'mergeedit':
this.synchronous = true;
this.editable = true;
this.displayURL = true;
this.displayFileName = true;
@ -112,6 +115,13 @@
this.displayDateModified = true;
break;
case 'filemerge':
this.synchronous = true;
this.displayURL = true;
this.displayFileName = true;
this.displayDateModified = true;
break;
default:
throw ("Invalid mode '" + val + "' in attachmentbox.xml");
}
@ -123,18 +133,16 @@
</property>
<field name="_item"/>
<property name="item"
onget="return this._item;"
onset="this._item = val; this.refresh();">
<property name="item" onget="return this._item;">
<setter><![CDATA[
if (!(val instanceof Zotero.Item)) {
throw new Error("'item' must be a Zotero.Item");
}
this._item = val;
this.refresh();
]]></setter>
</property>
<!-- .ref is an alias for .item -->
<property name="ref"
onget="return this._item;"
onset="this._item = val; this.refresh();">
</property>
<!-- Methods -->
<constructor>
@ -167,125 +175,122 @@
<method name="refresh">
<body><![CDATA[
Zotero.spawn(function* () {
Zotero.debug('Refreshing attachment box');
Zotero.debug('Refreshing attachment box');
var attachmentBox = document.getAnonymousNodes(this)[0];
var title = this._id('title');
var fileNameRow = this._id('fileNameRow');
var urlField = this._id('url');
var accessed = this._id('accessedRow');
var pagesRow = this._id('pagesRow');
var dateModifiedRow = this._id('dateModifiedRow');
var indexStatusRow = this._id('indexStatusRow');
var selectButton = this._id('select-button');
// DEBUG: this is annoying -- we really want to use an abstracted
// version of createValueElement() from itemPane.js
// (ideally in an XBL binding)
// Wrap title to multiple lines if necessary
while (title.hasChildNodes()) {
title.removeChild(title.firstChild);
}
var val = this.item.getField('title');
if (typeof val != 'string') {
val += "";
}
var firstSpace = val.indexOf(" ");
// Crop long uninterrupted text
if ((firstSpace == -1 && val.length > 29 ) || firstSpace > 29) {
title.setAttribute('crop', 'end');
title.setAttribute('value', val);
}
// Create a <description> element, essentially
else {
title.removeAttribute('value');
title.appendChild(document.createTextNode(val));
}
if (this.editable) {
title.className = 'zotero-clicky';
yield Zotero.Promise.all([this.item.loadItemData(), this.item.loadNote()])
.tap(() => Zotero.Promise.check(this.item));
// For the time being, use a silly little popup
title.addEventListener('click', this.editTitle, false);
}
var isImportedURL = this.item.attachmentLinkMode ==
Zotero.Attachments.LINK_MODE_IMPORTED_URL;
// Metadata for URL's
if (this.item.attachmentLinkMode == Zotero.Attachments.LINK_MODE_LINKED_URL
|| isImportedURL) {
var attachmentBox = document.getAnonymousNodes(this)[0];
var title = this._id('title');
var fileNameRow = this._id('fileNameRow');
var urlField = this._id('url');
var accessed = this._id('accessedRow');
var pagesRow = this._id('pagesRow');
var dateModifiedRow = this._id('dateModifiedRow');
var indexStatusRow = this._id('indexStatusRow');
var selectButton = this._id('select-button');
// DEBUG: this is annoying -- we really want to use an abstracted
// version of createValueElement() from itemPane.js
// (ideally in an XBL binding)
// Wrap title to multiple lines if necessary
while (title.hasChildNodes()) {
title.removeChild(title.firstChild);
}
var val = this.item.getField('title');
if (typeof val != 'string') {
val += "";
}
var firstSpace = val.indexOf(" ");
// Crop long uninterrupted text
if ((firstSpace == -1 && val.length > 29 ) || firstSpace > 29) {
title.setAttribute('crop', 'end');
title.setAttribute('value', val);
}
// Create a <description> element, essentially
else {
title.removeAttribute('value');
title.appendChild(document.createTextNode(val));
}
if (this.editable) {
title.className = 'zotero-clicky';
// For the time being, use a silly little popup
title.addEventListener('click', this.editTitle, false);
}
var isImportedURL = this.item.attachmentLinkMode ==
Zotero.Attachments.LINK_MODE_IMPORTED_URL;
// Metadata for URL's
if (this.item.attachmentLinkMode == Zotero.Attachments.LINK_MODE_LINKED_URL
|| isImportedURL) {
// URL
if (this.displayURL) {
var urlSpec = this.item.getField('url');
urlField.setAttribute('value', urlSpec);
urlField.setAttribute('hidden', false);
if (this.clickableLink) {
urlField.onclick = function (event) {
ZoteroPane_Local.loadURI(this.value, event)
};
urlField.className = 'zotero-text-link';
}
else {
urlField.className = '';
}
urlField.hidden = false;
// URL
if (this.displayURL) {
var urlSpec = this.item.getField('url');
urlField.setAttribute('value', urlSpec);
urlField.setAttribute('hidden', false);
if (this.clickableLink) {
urlField.onclick = function (event) {
ZoteroPane_Local.loadURI(this.value, event)
};
urlField.className = 'zotero-text-link';
}
else {
urlField.hidden = true;
}
// Access date
if (this.displayAccessed) {
this._id("accessed-label").value = Zotero.getString('itemFields.accessDate')
+ Zotero.getString('punctuation.colon');
this._id("accessed").value = Zotero.Date.sqlToDate(
this.item.getField('accessDate'), true
).toLocaleString();
accessed.hidden = false;
}
else {
accessed.hidden = true;
urlField.className = '';
}
urlField.hidden = false;
}
// Metadata for files
else {
urlField.hidden = true;
accessed.hidden = true;
}
if (this.item.attachmentLinkMode
!= Zotero.Attachments.LINK_MODE_LINKED_URL
&& this.displayFileName) {
var fileName = this.item.getFilename();
if (fileName) {
this._id("fileName-label").value = Zotero.getString('pane.item.attachments.filename')
+ Zotero.getString('punctuation.colon');
this._id("fileName").value = fileName;
fileNameRow.hidden = false;
}
else {
fileNameRow.hidden = true;
}
// Access date
if (this.displayAccessed) {
this._id("accessed-label").value = Zotero.getString('itemFields.accessDate')
+ Zotero.getString('punctuation.colon');
this._id("accessed").value = Zotero.Date.sqlToDate(
this.item.getField('accessDate'), true
).toLocaleString();
accessed.hidden = false;
}
else {
accessed.hidden = true;
}
}
// Metadata for files
else {
urlField.hidden = true;
accessed.hidden = true;
}
if (this.item.attachmentLinkMode
!= Zotero.Attachments.LINK_MODE_LINKED_URL
&& this.displayFileName) {
var fileName = this.item.attachmentFilename;
if (fileName) {
this._id("fileName-label").value = Zotero.getString('pane.item.attachments.filename')
+ Zotero.getString('punctuation.colon');
this._id("fileName").value = fileName;
fileNameRow.hidden = false;
}
else {
fileNameRow.hidden = true;
}
// Page count
if (this.displayPages) {
var pages = yield Zotero.Fulltext.getPages(this.item.id)
.tap(() => Zotero.Promise.check(this.item));
var pages = pages ? pages.total : null;
}
else {
fileNameRow.hidden = true;
}
// Page count
if (this.displayPages) {
Zotero.Fulltext.getPages(this.item.id)
.tap(() => Zotero.Promise.check(this.item))
.then(function (pages) {
pages = pages ? pages.total : null;
if (pages) {
this._id("pages-label").value = Zotero.getString('itemFields.pages')
+ Zotero.getString('punctuation.colon');
@ -295,77 +300,85 @@
else {
pagesRow.hidden = true;
}
}
else {
pagesRow.hidden = true;
}
if (this.displayDateModified) {
this._id("dateModified-label").value = Zotero.getString('itemFields.dateModified')
+ Zotero.getString('punctuation.colon');
var mtime = yield this.item.attachmentModificationTime
.tap(() => Zotero.Promise.check(this.item));
if (mtime) {
this._id("dateModified").value = new Date(mtime).toLocaleString();
}
// Use the item's mod time as a backup (e.g., when sync
// passes in the mod time for the nonexistent remote file)
else {
this._id("dateModified").value = Zotero.Date.sqlToDate(
this.item.getField('dateModified'), true
).toLocaleString();
}
});
}
else {
pagesRow.hidden = true;
}
if (this.displayDateModified) {
this._id("dateModified-label").value = Zotero.getString('itemFields.dateModified')
+ Zotero.getString('punctuation.colon');
// Conflict resolution uses a modal window, so promises won't work, but
// the sync process passes in the file mod time as dateModified
if (this.synchronous) {
this._id("dateModified").value = Zotero.Date.sqlToDate(
this.item.getField('dateModified'), true
).toLocaleString();
dateModifiedRow.hidden = false;
}
else {
dateModifiedRow.hidden = true;
this.item.attachmentModificationTime
.tap(() => Zotero.Promise.check(this._id))
.then(function (mtime) {
if (!this._id) return;
if (mtime) {
this._id("dateModified").value = new Date(mtime).toLocaleString();
}
dateModifiedRow.hidden = false;
});
}
// Full-text index information
if (this.displayIndexed) {
yield this.updateItemIndexedState()
.tap(() => Zotero.Promise.check(this.item));
}
else {
dateModifiedRow.hidden = true;
}
// Full-text index information
if (this.displayIndexed) {
this.updateItemIndexedState()
.tap(() => Zotero.Promise.check(this.item))
.then(function () {
indexStatusRow.hidden = false;
}
else {
indexStatusRow.hidden = true;
}
// Note editor
var noteEditor = this._id('attachment-note-editor');
if (this.displayNote) {
if (this.displayNoteIfEmpty || this.item.getNote() != '') {
Zotero.debug("setting links on top");
noteEditor.linksOnTop = true;
noteEditor.hidden = false;
// Don't make note editable (at least for now)
if (this.mode == 'merge' || this.mode == 'mergeedit') {
noteEditor.mode = 'merge';
noteEditor.displayButton = false;
}
else {
noteEditor.mode = this.mode;
}
noteEditor.parent = null;
noteEditor.item = this.item;
}
}
else {
noteEditor.hidden = true;
}
});
}
else {
indexStatusRow.hidden = true;
}
// Note editor
var noteEditor = this._id('attachment-note-editor');
if (this.displayNote) {
if (this.displayNoteIfEmpty || this.item.getNote() != '') {
Zotero.debug("setting links on top");
noteEditor.linksOnTop = true;
noteEditor.hidden = false;
// Don't make note editable (at least for now)
if (this.mode == 'merge' || this.mode == 'mergeedit') {
noteEditor.mode = 'merge';
noteEditor.displayButton = false;
}
else {
noteEditor.mode = this.mode;
}
noteEditor.parent = null;
noteEditor.item = this.item;
}
}
else {
noteEditor.hidden = true;
}
if (this.displayButton) {
selectButton.label = this.buttonCaption;
selectButton.hidden = false;
selectButton.setAttribute('oncommand',
'document.getBindingParent(this).clickHandler(this)');
}
else {
selectButton.hidden = true;
}
}, this);
if (this.displayButton) {
selectButton.label = this.buttonCaption;
selectButton.hidden = false;
selectButton.setAttribute('oncommand',
'document.getBindingParent(this).clickHandler(this)');
}
else {
selectButton.hidden = true;
}
]]></body>
</method>

9
chrome/content/zotero/bindings/itembox.xml
View File

@ -71,6 +71,7 @@
switch (val) {
case 'view':
case 'merge':
break;
case 'edit':
@ -99,10 +100,9 @@
<field name="_item"/>
<property name="item" onget="return this._item;">
<setter>
<![CDATA[
<setter><![CDATA[
if (!(val instanceof Zotero.Item)) {
throw ("<zoteroitembox>.item must be a Zotero.Item");
throw new Error("'item' must be a Zotero.Item");
}
// When changing items, reset truncation of creator list
@ -112,8 +112,7 @@
this._item = val;
this.refresh();
]]>
</setter>
]]></setter>
</property>
<!-- .ref is an alias for .item -->

22
chrome/content/zotero/bindings/merge.xml
View File

@ -99,9 +99,11 @@
}
// Check for note or attachment
this.type = this._getTypeFromObject(
this._data.left.deleted ? this._data.right : this._data.left
);
if (!this.type) {
this.type = this._getTypeFromObject(
this._data.left.deleted ? this._data.right : this._data.left
);
}
var showButton = this.type != 'item';
@ -109,7 +111,6 @@
this._rightpane.showButton = showButton;
this._leftpane.data = this._data.left;
this._rightpane.data = this._data.right;
this._mergepane.type = this.type;
this._mergepane.data = this._data.merge;
if (this._data.selected == 'left') {
@ -313,6 +314,7 @@
break;
case 'attachment':
case 'file':
elementName = 'zoteroattachmentbox';
break;
@ -320,13 +322,8 @@
elementName = 'zoteronoteeditor';
break;
case 'file':
elementName = 'zoterostoragefilebox';
break;
default:
throw ("Object type '" + this.type
+ "' not supported in <zoteromergepane>.ref");
throw new Error("Object type '" + this.type + "' not supported");
}
var objbox = document.createElement(elementName);
@ -342,8 +339,7 @@
objbox.setAttribute("anonid", "objectbox");
objbox.setAttribute("flex", "1");
objbox.mode = 'view';
objbox.mode = this.type == 'file' ? 'filemerge' : 'merge';
var button = this._id('choose-button');
if (this.showButton) {
@ -363,7 +359,7 @@
// Create item from JSON for metadata box
var item = new Zotero.Item(val.itemType);
item.fromJSON(val);
objbox.ref = item;
objbox.item = item;
]]>
</setter>
</property>

1
chrome/content/zotero/bindings/noteeditor.xml
View File

@ -64,6 +64,7 @@
switch (val) {
case 'view':
case 'merge':
break;
case 'edit':

1
chrome/content/zotero/bindings/relatedbox.xml
View File

@ -109,6 +109,7 @@
]]>
</destructor>
<!-- TODO: Asyncify -->
<method name="notify">
<parameter name="event"/>
<parameter name="type"/>

32
chrome/content/zotero/merge.js
View File

@ -47,13 +47,20 @@ var Zotero_Merge_Window = new function () {
_wizard.getButton('cancel').setAttribute('label', Zotero.getString('sync.cancel'));
_io = window.arguments[0].wrappedJSObject;
_io = window.arguments[0];
// Not totally clear when this is necessary
if (window.arguments[0].wrappedJSObject) {
_io = window.arguments[0].wrappedJSObject;
}
_conflicts = _io.dataIn.conflicts;
if (!_conflicts.length) {
// TODO: handle no conflicts
return;
}
if (_io.dataIn.type) {
_mergeGroup.type = _io.dataIn.type;
}
_mergeGroup.leftCaption = _io.dataIn.captions[0];
_mergeGroup.rightCaption = _io.dataIn.captions[1];
_mergeGroup.mergeCaption = _io.dataIn.captions[2];
@ -240,7 +247,7 @@ var Zotero_Merge_Window = new function () {
}
// Apply changes from each side and pick most recent version for conflicting fields
var mergeInfo = {
data: {}
data: {}
};
Object.assign(mergeInfo.data, _conflicts[pos].left)
Zotero.DataObjectUtilities.applyChanges(mergeInfo.data, _conflicts[pos].changes);
@ -251,7 +258,9 @@ var Zotero_Merge_Window = new function () {
else {
var side = 1;
}
Zotero.DataObjectUtilities.applyChanges(mergeInfo.data, _conflicts[pos].conflicts.map(x => x[side]));
Zotero.DataObjectUtilities.applyChanges(
mergeInfo.data, _conflicts[pos].conflicts.map(x => x[side])
);
mergeInfo.selected = side ? 'right' : 'left';
return mergeInfo;
}
@ -284,13 +293,22 @@ var Zotero_Merge_Window = new function () {
function _updateResolveAllCheckbox() {
if (_mergeGroup.rightpane.getAttribute("selected") == 'true') {
var label = 'resolveAllRemoteFields';
if (_mergeGroup.type == 'file') {
if (_mergeGroup.rightpane.getAttribute("selected") == 'true') {
var label = 'resolveAllRemote';
}
else {
var label = 'resolveAllLocal';
}
}
else {
var label = 'resolveAllLocalFields';
if (_mergeGroup.rightpane.getAttribute("selected") == 'true') {
var label = 'resolveAllRemoteFields';
}
else {
var label = 'resolveAllLocalFields';
}
}
// TODO: files
_resolveAllCheckbox.label = Zotero.getString('sync.conflict.' + label);
}

28
chrome/content/zotero/xpcom/data/item.js
View File

@ -50,7 +50,6 @@ Zotero.Item = function(itemTypeOrID) {
this._attachmentLinkMode = null;
this._attachmentContentType = null;
this._attachmentPath = null;
this._attachmentSyncState = null;
// loadCreators
this._creators = [];
@ -1453,14 +1452,13 @@ Zotero.Item.prototype._saveData = Zotero.Promise.coroutine(function* (env) {
if (this._changed.attachmentData) {
let sql = "REPLACE INTO itemAttachments (itemID, parentItemID, linkMode, "
+ "contentType, charsetID, path, syncState) VALUES (?,?,?,?,?,?,?)";
+ "contentType, charsetID, path) VALUES (?,?,?,?,?,?)";
let linkMode = this.attachmentLinkMode;
let contentType = this.attachmentContentType;
let charsetID = this.attachmentCharset
? Zotero.CharacterSets.getID(this.attachmentCharset)
: null;
let path = this.attachmentPath;
let syncState = this.attachmentSyncState;
if (linkMode == Zotero.Attachments.LINK_MODE_LINKED_FILE && libraryType != 'user') {
throw new Error("Linked files can only be added to user library");
@ -1472,8 +1470,7 @@ Zotero.Item.prototype._saveData = Zotero.Promise.coroutine(function* (env) {
{ int: linkMode },
contentType ? { string: contentType } : null,
charsetID ? { int: charsetID } : null,
path ? { string: path } : null,
syncState ? { int: syncState } : 0
path ? { string: path } : null
];
yield Zotero.DB.queryAsync(sql, params);
@ -2295,8 +2292,10 @@ Zotero.Item.prototype.renameAttachmentFile = Zotero.Promise.coroutine(function*
yield this.relinkAttachmentFile(destPath);
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.setSyncedHash(this.id, null, false);
yield Zotero.Sync.Storage.setSyncState(this.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD);
yield Zotero.Sync.Storage.Local.setSyncedHash(this.id, null, false);
yield Zotero.Sync.Storage.Local.setSyncState(
this.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
);
}.bind(this));
return true;
@ -2317,11 +2316,10 @@ Zotero.Item.prototype.renameAttachmentFile = Zotero.Promise.coroutine(function*
/**
* @param {string} path File path
* @param {Boolean} [skipItemUpdate] Don't update attachment item mod time,
* so that item doesn't sync. Used when a file
* needs to be renamed to be accessible but the
* user doesn't have access to modify the
* attachment metadata
* @param {Boolean} [skipItemUpdate] Don't update attachment item mod time, so that item doesn't
* sync. Used when a file needs to be renamed to be accessible but the user doesn't have
* access to modify the attachment metadata. This also allows a save when the library is
* read-only.
*/
Zotero.Item.prototype.relinkAttachmentFile = Zotero.Promise.coroutine(function* (path, skipItemUpdate) {
if (path instanceof Components.interfaces.nsIFile) {
@ -2382,7 +2380,8 @@ Zotero.Item.prototype.relinkAttachmentFile = Zotero.Promise.coroutine(function*
yield this.saveTx({
skipDateModifiedUpdate: true,
skipClientDateModifiedUpdate: skipItemUpdate
skipClientDateModifiedUpdate: skipItemUpdate,
skipEditCheck: skipItemUpdate
});
return true;
@ -3606,9 +3605,6 @@ Zotero.Item.prototype.clone = Zotero.Promise.coroutine(function* (libraryID, ski
if (this.attachmentPath) {
newItem.attachmentPath = this.attachmentPath;
}
if (this.attachmentSyncState) {
newItem.attachmentSyncState = this.attachmentSyncState;
}
}
}
}

3
chrome/content/zotero/xpcom/data/items.js
View File

@ -84,8 +84,7 @@ Zotero.Items = function() {
attachmentCharset: "CS.charset AS attachmentCharset",
attachmentLinkMode: "IA.linkMode AS attachmentLinkMode",
attachmentContentType: "IA.contentType AS attachmentContentType",
attachmentPath: "IA.path AS attachmentPath",
attachmentSyncState: "IA.syncState AS attachmentSyncState"
attachmentPath: "IA.path AS attachmentPath"
};
}
}, {lazy: true});

90
chrome/content/zotero/xpcom/file.js
View File

@ -48,7 +48,7 @@ Zotero.File = new function(){
else if (pathOrFile instanceof Ci.nsIFile) {
return pathOrFile;
}
throw new Error('Unexpected value provided to Zotero.File.pathToFile() (' + pathOrFile + ')');
throw new Error("Unexpected value '" + pathOrFile + "'");
}
@ -348,7 +348,7 @@ Zotero.File = new function(){
* @param {String} [charset] - The character set; defaults to UTF-8
* @return {Promise} - A promise that is resolved when the file has been written
*/
this.putContentsAsync = function putContentsAsync(path, data, charset) {
this.putContentsAsync = function (path, data, charset) {
if (path instanceof Ci.nsIFile) {
path = path.path;
}
@ -424,18 +424,17 @@ Zotero.File = new function(){
* iterator when done
*
* The DirectoryInterator is passed as the first parameter to the generator.
* A StopIteration error will be caught automatically.
*
* Zotero.File.iterateDirectory(path, function* (iterator) {
* while (true) {
* var entry = yield iterator.next();
* [...]
* }
* }).done()
* })
*
* @return {Promise}
*/
this.iterateDirectory = function iterateDirectory(path, generator) {
this.iterateDirectory = function (path, generator) {
var iterator = new OS.File.DirectoryIterator(path);
return Zotero.Promise.coroutine(generator)(iterator)
.catch(function (e) {
@ -470,6 +469,8 @@ Zotero.File = new function(){
this.createShortened = function (file, type, mode, maxBytes) {
file = this.pathToFile(file);
if (!maxBytes) {
maxBytes = 255;
}
@ -575,6 +576,8 @@ Zotero.File = new function(){
}
break;
}
return file.leafName;
}
@ -902,29 +905,28 @@ Zotero.File = new function(){
this.checkFileAccessError = function (e, file, operation) {
var str = 'file.accessError.';
if (file) {
var str = Zotero.getString('file.accessError.theFile', file.path);
str += 'theFile'
}
else {
var str = Zotero.getString('file.accessError.aFile');
str += 'aFile'
}
str += 'CannotBe';
switch (operation) {
case 'create':
var opWord = Zotero.getString('file.accessError.created');
break;
case 'update':
var opWord = Zotero.getString('file.accessError.updated');
str += 'Created';
break;
case 'delete':
var opWord = Zotero.getString('file.accessError.deleted');
str += 'Deleted';
break;
default:
var opWord = Zotero.getString('file.accessError.updated');
str += 'Updated';
}
str = Zotero.getString(str, file.path ? file.path : undefined);
Zotero.debug(file.path);
Zotero.debug(e, 1);
@ -962,4 +964,64 @@ Zotero.File = new function(){
throw (e);
}
this.checkPathAccessError = function (e, path, operation) {
var str = 'file.accessError.';
if (path) {
str += 'theFile'
}
else {
str += 'aFile'
}
str += 'CannotBe';
switch (operation) {
case 'create':
str += 'Created';
break;
case 'delete':
str += 'Deleted';
break;
default:
str += 'Updated';
}
str = Zotero.getString(str, path ? path : undefined);
Zotero.debug(path);
Zotero.debug(e, 1);
Components.utils.reportError(e);
// TODO: Check for specific errors?
if (e instanceof OS.File.Error) {
let checkFileWindows = Zotero.getString('file.accessError.message.windows');
let checkFileOther = Zotero.getString('file.accessError.message.other');
var msg = str + "\n\n"
+ (Zotero.isWin ? checkFileWindows : checkFileOther)
+ "\n\n"
+ Zotero.getString('file.accessError.restart');
var e = new Zotero.Error(
msg,
0,
{
dialogButtonText: Zotero.getString('file.accessError.showParentDir'),
dialogButtonCallback: function () {
try {
file.parent.QueryInterface(Components.interfaces.nsILocalFile);
file.parent.reveal();
}
// Unsupported on some platforms
catch (e2) {
Zotero.launchFile(file.parent);
}
}
}
);
}
throw e;
}
}

1661
chrome/content/zotero/xpcom/storage.js
View File

File diff suppressed because it is too large Load Diff

87
chrome/content/zotero/xpcom/storage/mode.js
View File

@ -1,87 +0,0 @@
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2009 Center for History and New Media
George Mason University, Fairfax, Virginia, USA
http://zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
Zotero.Sync.Storage.Mode = function () {};
Zotero.Sync.Storage.Mode.prototype.__defineGetter__('verified', function () {
return this._verified;
});
Zotero.Sync.Storage.Mode.prototype.__defineGetter__('username', function () {
return this._username;
});
Zotero.Sync.Storage.Mode.prototype.__defineGetter__('password', function () {
return this._password;
});
Zotero.Sync.Storage.Mode.prototype.__defineSetter__('password', function (val) {
this._password = val;
});
Zotero.Sync.Storage.Mode.prototype.init = function () {
return this._init();
}
Zotero.Sync.Storage.Mode.prototype.sync = function (observer) {
return Zotero.Sync.Storage.sync(this.name, observer);
}
Zotero.Sync.Storage.Mode.prototype.downloadFile = function (request) {
return this._downloadFile(request);
}
Zotero.Sync.Storage.Mode.prototype.uploadFile = function (request) {
return this._uploadFile(request);
}
Zotero.Sync.Storage.Mode.prototype.getLastSyncTime = function (libraryID) {
return this._getLastSyncTime(libraryID);
}
Zotero.Sync.Storage.Mode.prototype.setLastSyncTime = function (callback, useLastSyncTime) {
return this._setLastSyncTime(callback, useLastSyncTime);
}
Zotero.Sync.Storage.Mode.prototype.checkServer = function (callback) {
return this._checkServer(callback);
}
Zotero.Sync.Storage.Mode.prototype.checkServerCallback = function (uri, status, window, skipSuccessMessage) {
return this._checkServerCallback(uri, status, window, skipSuccessMessage);
}
Zotero.Sync.Storage.Mode.prototype.cacheCredentials = function () {
return this._cacheCredentials();
}
Zotero.Sync.Storage.Mode.prototype.purgeDeletedStorageFiles = function (callback) {
return this._purgeDeletedStorageFiles(callback);
}
Zotero.Sync.Storage.Mode.prototype.purgeOrphanedStorageFiles = function (callback) {
return this._purgeOrphanedStorageFiles(callback);
}

427
chrome/content/zotero/xpcom/storage/queue.js
View File

@ -1,427 +0,0 @@
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2009 Center for History and New Media
George Mason University, Fairfax, Virginia, USA
http://zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
/**
* Queue for storage sync transfer requests
*
* @param {String} type Queue type (e.g., 'download' or 'upload')
*/
Zotero.Sync.Storage.Queue = function (type, libraryID) {
Zotero.debug("Initializing " + type + " queue for library " + libraryID);
// Public properties
this.type = type;
this.libraryID = libraryID;
this.maxConcurrentRequests = 1;
this.activeRequests = 0;
this.totalRequests = 0;
// Private properties
this._requests = {};
this._highPriority = [];
this._running = false;
this._stopping = false;
this._finished = false;
this._error = false;
this._finishedReqs = 0;
this._localChanges = false;
this._remoteChanges = false;
this._conflicts = [];
this._cachedPercentage;
this._cachedPercentageTime;
}
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('name', function () {
return this.type + "/" + this.libraryID;
});
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('Type', function () {
return this.type[0].toUpperCase() + this.type.substr(1);
});
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('running', function () this._running);
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('stopping', function () this._stopping);
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('finished', function () this._finished);
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('unfinishedRequests', function () {
return this.totalRequests - this.finishedRequests;
});
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('finishedRequests', function () {
return this._finishedReqs;
});
Zotero.Sync.Storage.Queue.prototype.__defineSetter__('finishedRequests', function (val) {
Zotero.debug("Finished requests: " + val);
Zotero.debug("Total requests: " + this.totalRequests);
this._finishedReqs = val;
if (val == 0) {
return;
}
// Last request
if (val == this.totalRequests) {
Zotero.debug(this.Type + " queue is done for library " + this.libraryID);
// DEBUG info
Zotero.debug("Active requests: " + this.activeRequests);
if (this.activeRequests) {
throw new Error(this.Type + " queue for library " + this.libraryID
+ " can't be done if there are active requests");
}
this._running = false;
this._stopping = false;
this._finished = true;
this._requests = {};
this._highPriority = [];
var localChanges = this._localChanges;
var remoteChanges = this._remoteChanges;
var conflicts = this._conflicts.concat();
var deferred = this._deferred;
this._localChanges = false;
this._remoteChanges = false;
this._conflicts = [];
this._deferred = null;
if (!this._error) {
Zotero.debug("Resolving promise for queue " + this.name);
Zotero.debug(this._localChanges);
Zotero.debug(this._remoteChanges);
Zotero.debug(this._conflicts);
deferred.resolve({
libraryID: this.libraryID,
type: this.type,
localChanges: localChanges,
remoteChanges: remoteChanges,
conflicts: conflicts
});
}
else {
Zotero.debug("Rejecting promise for queue " + this.name);
var e = this._error;
this._error = false;
e.libraryID = this.libraryID;
e.type = this.type;
deferred.reject(e);
}
return;
}
if (this._stopping) {
return;
}
this.advance();
});
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('queuedRequests', function () {
return this.unfinishedRequests - this.activeRequests;
});
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('remaining', function () {
var remaining = 0;
for each(var request in this._requests) {
remaining += request.remaining;
}
return remaining;
});
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('percentage', function () {
if (this.totalRequests == 0) {
return 0;
}
if (this._finished) {
return 100;
}
// Cache percentage for a second
if (this._cachedPercentage && (new Date() - this._cachedPercentageTime) < 1000) {
return this._cachedPercentage;
}
var completedRequests = 0;
for each(var request in this._requests) {
completedRequests += request.percentage / 100;
}
this._cachedPercentage = Math.round((completedRequests / this.totalRequests) * 100);
this._cachedPercentageTime = new Date();
return this._cachedPercentage;
});
Zotero.Sync.Storage.Queue.prototype.isRunning = function () {
return this._running;
}
Zotero.Sync.Storage.Queue.prototype.isStopping = function () {
return this._stopping;
}
/**
* Add a request to this queue
*
* @param {Zotero.Sync.Storage.Request} request
* @param {Boolean} highPriority Add or move request to high priority queue
*/
Zotero.Sync.Storage.Queue.prototype.addRequest = function (request, highPriority) {
if (this._finished) {
this.reset();
}
request.queue = this;
var name = request.name;
Zotero.debug("Queuing " + this.type + " request '" + name + "' for library " + this.libraryID);
if (this._requests[name]) {
if (highPriority) {
Zotero.debug("Moving " + name + " to high-priority queue");
this._requests[name].importCallbacks(request);
this._highPriority.push(name);
return;
}
Zotero.debug("Request '" + name + "' already exists");
return;
}
this._requests[name] = request;
this.totalRequests++;
if (highPriority) {
this._highPriority.push(name);
}
}
Zotero.Sync.Storage.Queue.prototype.start = function () {
if (!this._deferred || this._deferred.promise.isFulfilled()) {
Zotero.debug("Creating deferred for queue " + this.name);
this._deferred = Zotero.Promise.defer();
}
// The queue manager needs to know what queues were running in the
// current session
Zotero.Sync.Storage.QueueManager.addCurrentQueue(this);
var self = this;
setTimeout(function () {
self.advance();
}, 0);
return this._deferred.promise;
}
/**
* Start another request in this queue if there's an available slot
*/
Zotero.Sync.Storage.Queue.prototype.advance = function () {
this._running = true;
this._finished = false;
if (this._stopping) {
Zotero.debug(this.Type + " queue for library " + this.libraryID
+ "is being stopped in Zotero.Sync.Storage.Queue.advance()", 2);
return;
}
if (!this.queuedRequests) {
Zotero.debug("No remaining requests in " + this.type
+ " queue for library " + this.libraryID + " ("
+ this.activeRequests + " active, "
+ this.finishedRequests + " finished)");
return;
}
if (this.activeRequests >= this.maxConcurrentRequests) {
Zotero.debug(this.Type + " queue for library " + this.libraryID
+ " is busy (" + this.activeRequests + "/"
+ this.maxConcurrentRequests + ")");
return;
}
// Start the first unprocessed request
// Try the high-priority queue first
var self = this;
var request, name;
while (name = this._highPriority.shift()) {
request = this._requests[name];
if (request.isRunning() || request.isFinished()) {
continue;
}
let requestName = name;
Zotero.Promise.try(function () {
var promise = request.start();
self.advance();
return promise;
})
.then(function (result) {
if (result.localChanges) {
self._localChanges = true;
}
if (result.remoteChanges) {
self._remoteChanges = true;
}
if (result.conflict) {
self.addConflict(
requestName,
result.conflict.local,
result.conflict.remote
);
}
})
.catch(function (e) {
self.error(e);
});
return;
}
// And then others
for each(var request in this._requests) {
if (request.isRunning() || request.isFinished()) {
continue;
}
let requestName = request.name;
// This isn't in a Zotero.Promise.try() because the request needs to get marked
// as running immediately so that it doesn't get run again by a
// subsequent advance() call.
try {
var promise = request.start();
self.advance();
}
catch (e) {
self.error(e);
}
promise.then(function (result) {
if (result.localChanges) {
self._localChanges = true;
}
if (result.remoteChanges) {
self._remoteChanges = true;
}
if (result.conflict) {
self.addConflict(
requestName,
result.conflict.local,
result.conflict.remote
);
}
})
.catch(function (e) {
self.error(e);
});
return;
}
}
Zotero.Sync.Storage.Queue.prototype.updateProgress = function () {
Zotero.Sync.Storage.QueueManager.updateProgress();
}
Zotero.Sync.Storage.Queue.prototype.addConflict = function (requestName, localData, remoteData) {
Zotero.debug('===========');
Zotero.debug(localData);
Zotero.debug(remoteData);
this._conflicts.push({
name: requestName,
localData: localData,
remoteData: remoteData
});
}
Zotero.Sync.Storage.Queue.prototype.error = function (e) {
if (!this._error) {
if (this.isRunning()) {
this._error = e;
}
else {
Zotero.debug("Queue " + this.name + " was no longer running -- not assigning error", 2);
}
}
Zotero.debug(e, 1);
this.stop();
}
/**
* Stops all requests in this queue
*/
Zotero.Sync.Storage.Queue.prototype.stop = function () {
if (!this._running) {
Zotero.debug(this.Type + " queue for library " + this.libraryID
+ " is not running");
return;
}
if (this._stopping) {
Zotero.debug("Already stopping " + this.type + " queue for library "
+ this.libraryID);
return;
}
Zotero.debug("Stopping " + this.type + " queue for library " + this.libraryID);
// If no requests, finish manually
/*if (this.activeRequests == 0) {
this._finishedRequests = this._finishedRequests;
return;
}*/
this._stopping = true;
for each(var request in this._requests) {
if (!request.isFinished()) {
request.stop(true);
}
}
Zotero.debug("Queue is stopped");
}
Zotero.Sync.Storage.Queue.prototype.reset = function () {
this._finished = false;
this._finishedReqs = 0;
this.totalRequests = 0;
}

370
chrome/content/zotero/xpcom/storage/queueManager.js
View File

@ -1,370 +0,0 @@
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2009 Center for History and New Media
George Mason University, Fairfax, Virginia, USA
http://zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
Zotero.Sync.Storage.QueueManager = new function () {
var _queues = {};
var _currentQueues = [];
this.start = Zotero.Promise.coroutine(function* (libraryID) {
if (libraryID) {
var queues = this.getAll(libraryID);
var suffix = " for library " + libraryID;
}
else {
var queues = this.getAll();
var suffix = "";
}
Zotero.debug("Starting file sync queues" + suffix);
var promises = [];
for each(var queue in queues) {
if (!queue.unfinishedRequests) {
continue;
}
Zotero.debug("Starting queue " + queue.name);
promises.push(queue.start());
}
if (!promises.length) {
Zotero.debug("No files to sync" + suffix);
}
var results = yield Zotero.Promise.allSettled(promises);
Zotero.debug("All storage queues are finished" + suffix);
for (let i = 0; i < results.length; i++) {
let result = results[i];
// Check for conflicts to resolve
if (result.state == "fulfilled") {
result = result.value;
if (result.conflicts.length) {
Zotero.debug("Reconciling conflicts for library " + result.libraryID);
Zotero.debug(result.conflicts);
var data = yield _reconcileConflicts(result.conflicts);
if (data) {
_processMergeData(data);
}
}
}
}
return promises;
});
this.stop = function (libraryID) {
if (libraryID) {
var queues = this.getAll(libraryID);
}
else {
var queues = this.getAll();
}
for (var queue in queues) {
queue.stop();
}
};
/**
* Retrieving a queue, creating a new one if necessary
*
* @param {String} queueName
*/
this.get = function (queueName, libraryID, noInit) {
if (typeof libraryID == 'undefined') {
throw new Error("libraryID not specified");
}
var hash = queueName + "/" + libraryID;
// Initialize the queue if it doesn't exist yet
if (!_queues[hash]) {
if (noInit) {
return false;
}
var queue = new Zotero.Sync.Storage.Queue(queueName, libraryID);
switch (queueName) {
case 'download':
queue.maxConcurrentRequests =
Zotero.Prefs.get('sync.storage.maxDownloads')
break;
case 'upload':
queue.maxConcurrentRequests =
Zotero.Prefs.get('sync.storage.maxUploads')
break;
default:
throw ("Invalid queue '" + queueName + "' in Zotero.Sync.Storage.QueueManager.get()");
}
_queues[hash] = queue;
}
return _queues[hash];
};
this.getAll = function (libraryID) {
if (typeof libraryID == 'string') {
throw new Error("libraryID must be a number or undefined");
}
var queues = [];
for each(var queue in _queues) {
if (typeof libraryID == 'undefined' || queue.libraryID === libraryID) {
queues.push(queue);
}
}
return queues;
};
this.addCurrentQueue = function (queue) {
if (!this.hasCurrentQueue(queue)) {
_currentQueues.push(queue.name);
}
}
this.hasCurrentQueue = function (queue) {
return _currentQueues.indexOf(queue.name) != -1;
}
/**
* Stop all queues
*
* @param {Boolean} [skipStorageFinish=false] Don't call Zotero.Sync.Storage.finish()
* when done (used when we stopped because of
* an error)
*/
this.cancel = function (skipStorageFinish) {
Zotero.debug("Stopping all storage queues");
for each(var queue in _queues) {
if (queue.isRunning() && !queue.isStopping()) {
queue.stop();
}
}
}
this.finish = function () {
Zotero.debug("All storage queues are finished");
_currentQueues = [];
}
/**
* Calculate the current progress values and trigger a display update
*
* Also detects when all queues have finished and ends sync progress
*/
this.updateProgress = function () {
var activeRequests = 0;
var allFinished = true;
for each(var queue in _queues) {
// Finished or never started
if (!queue.isRunning() && !queue.isStopping()) {
continue;
}
allFinished = false;
activeRequests += queue.activeRequests;
}
if (activeRequests == 0) {
_updateProgressMeters(0);
if (allFinished) {
this.finish();
}
return;
}
var status = {};
for each(var queue in _queues) {
if (!this.hasCurrentQueue(queue)) {
continue;
}
if (!status[queue.libraryID]) {
status[queue.libraryID] = {};
}
if (!status[queue.libraryID][queue.type]) {
status[queue.libraryID][queue.type] = {};
}
status[queue.libraryID][queue.type].statusString = _getQueueStatus(queue);
status[queue.libraryID][queue.type].percentage = queue.percentage;
status[queue.libraryID][queue.type].totalRequests = queue.totalRequests;
status[queue.libraryID][queue.type].finished = queue.finished;
}
_updateProgressMeters(activeRequests, status);
}
/**
* Get a status string for a queue
*
* @param {Zotero.Sync.Storage.Queue} queue
* @return {String}
*/
function _getQueueStatus(queue) {
var remaining = queue.remaining;
var unfinishedRequests = queue.unfinishedRequests;
if (!unfinishedRequests) {
return Zotero.getString('sync.storage.none');
}
if (remaining > 1000) {
var bytesRemaining = Zotero.getString(
'sync.storage.mbRemaining',
Zotero.Utilities.numberFormat(remaining / 1000 / 1000, 1)
);
}
else {
var bytesRemaining = Zotero.getString(
'sync.storage.kbRemaining',
Zotero.Utilities.numberFormat(remaining / 1000, 0)
);
}
var totalRequests = queue.totalRequests;
var filesRemaining = Zotero.getString(
'sync.storage.filesRemaining',
[totalRequests - unfinishedRequests, totalRequests]
);
return bytesRemaining + ' (' + filesRemaining + ')';
}
/**
* Cycle through windows, updating progress meters with new values
*/
function _updateProgressMeters(activeRequests, status) {
// Get overall percentage across queues
var sum = 0, num = 0, percentage, total;
for each(var libraryStatus in status) {
for each(var queueStatus in libraryStatus) {
percentage = queueStatus.percentage;
total = queueStatus.totalRequests;
sum += total * percentage;
num += total;
}
}
var percentage = Math.round(sum / num);
var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
var enumerator = wm.getEnumerator("navigator:browser");
while (enumerator.hasMoreElements()) {
var win = enumerator.getNext();
if (!win.ZoteroPane) continue;
var doc = win.ZoteroPane.document;
var box = doc.getElementById("zotero-tb-sync-progress-box");
var meter = doc.getElementById("zotero-tb-sync-progress");
if (activeRequests == 0) {
box.hidden = true;
continue;
}
meter.setAttribute("value", percentage);
box.hidden = false;
var percentageLabel = doc.getElementById('zotero-tb-sync-progress-tooltip-progress');
percentageLabel.lastChild.setAttribute('value', percentage + "%");
var statusBox = doc.getElementById('zotero-tb-sync-progress-status');
statusBox.data = status;
}
}
var _reconcileConflicts = Zotero.Promise.coroutine(function* (conflicts) {
var objectPairs = [];
for each(var conflict in conflicts) {
var item = Zotero.Sync.Storage.getItemFromRequestName(conflict.name);
var item1 = yield item.clone(false, false, true);
item1.setField('dateModified',
Zotero.Date.dateToSQL(new Date(conflict.localData.modTime), true));
var item2 = yield item.clone(false, false, true);
item2.setField('dateModified',
Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime), true));
objectPairs.push([item1, item2]);
}
var io = {
dataIn: {
type: 'storagefile',
captions: [
Zotero.getString('sync.storage.localFile'),
Zotero.getString('sync.storage.remoteFile'),
Zotero.getString('sync.storage.savedFile')
],
objects: objectPairs
}
};
var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
var lastWin = wm.getMostRecentWindow("navigator:browser");
lastWin.openDialog('chrome://zotero/content/merge.xul', '', 'chrome,modal,centerscreen', io);
if (!io.dataOut) {
return false;
}
// Since we're only putting cloned items into the merge window,
// we have to manually set the ids
for (var i=0; i<conflicts.length; i++) {
io.dataOut[i].id = Zotero.Sync.Storage.getItemFromRequestName(conflicts[i].name).id;
}
return io.dataOut;
});
function _processMergeData(data) {
if (!data.length) {
return false;
}
for each(var mergeItem in data) {
var itemID = mergeItem.id;
var dateModified = mergeItem.ref.getField('dateModified');
// Local
if (dateModified == mergeItem.left.getField('dateModified')) {
Zotero.Sync.Storage.setSyncState(
itemID, Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD
);
}
// Remote
else {
Zotero.Sync.Storage.setSyncState(
itemID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD
);
}
}
}
}

307
chrome/content/zotero/xpcom/storage/storageEngine.js Normal file
View File

@ -0,0 +1,307 @@
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2015 Center for History and New Media
George Mason University, Fairfax, Virginia, USA
http://zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
if (!Zotero.Sync.Storage) {
Zotero.Sync.Storage = {};
}
/**
* An Engine manages file sync processes for a given library
*
* @param {Object} options
* @param {Zotero.Sync.APIClient} options.apiClient
* @param {Integer} options.libraryID
* @param {Function} [onError] - Function to run on error
* @param {Boolean} [stopOnError]
*/
Zotero.Sync.Storage.Engine = function (options) {
if (options.apiClient == undefined) {
throw new Error("options.apiClient not set");
}
if (options.libraryID == undefined) {
throw new Error("options.libraryID not set");
}
this.apiClient = options.apiClient;
this.background = options.background;
this.firstInSession = options.firstInSession;
this.lastFullFileCheck = options.lastFullFileCheck;
this.libraryID = options.libraryID;
this.library = Zotero.Libraries.get(options.libraryID);
this.local = Zotero.Sync.Storage.Local;
this.utils = Zotero.Sync.Storage.Utilities;
this.mode = this.local.getModeForLibrary(this.libraryID);
var modeClass = this.utils.getClassForMode(this.mode);
this.controller = new modeClass(options);
this.setStatus = options.setStatus || function () {};
this.onError = options.onError || function (e) {};
this.stopOnError = options.stopOnError || false;
this.queues = [];
['download', 'upload'].forEach(function (type) {
this.queues[type] = new ConcurrentCaller({
id: `${this.libraryID}/${type}`,
numConcurrent: Zotero.Prefs.get(
'sync.storage.max' + Zotero.Utilities.capitalize(type) + 's'
),
onError: this.onError,
stopOnError: this.stopOnError,
logger: Zotero.debug
});
}.bind(this))
this.maxCheckAge = 10800; // maximum age in seconds for upload modification check (3 hours)
}
Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function* () {
var libraryID = this.libraryID;
if (!Zotero.Prefs.get("sync.storage.enabled")) {
Zotero.debug("File sync is not enabled for " + this.library.name);
return false;
}
Zotero.debug("Starting file sync for " + this.library.name);
if (!this.controller.verified) {
Zotero.debug(`${this.mode} file sync is not active`);
throw new Error("Storage mode verification not implemented");
// TODO: Check server
}
if (this.controller.cacheCredentials) {
yield this.controller.cacheCredentials();
}
// Get library last-sync time for download-on-sync libraries.
var lastSyncTime = null;
var downloadAll = this.local.downloadOnSync(libraryID);
if (downloadAll) {
lastSyncTime = yield this.controller.getLastSyncTime(libraryID);
}
// Check for updated files to upload
if (!Zotero.Libraries.isFilesEditable(libraryID)) {
Zotero.debug("No file editing access -- skipping file modification check for "
+ this.library.name);
}
// If this is a background sync, it's not the first sync of the session, the library has had
// at least one full check this session, and it's been less than maxCheckAge since the last
// full check of this library, check only files that were previously modified or opened
// recently
else if (this.background
&& !this.firstInSession
&& this.local.lastFullFileCheck[libraryID]
&& (this.local.lastFullFileCheck[libraryID]
+ (this.maxCheckAge * 1000)) > new Date().getTime()) {
let itemIDs = this.local.getFilesToCheck(libraryID, this.maxCheckAge);
yield this.local.checkForUpdatedFiles(libraryID, itemIDs);
}
// Otherwise check all files in library
else {
this.local.lastFullFileCheck[libraryID] = new Date().getTime();
yield this.local.checkForUpdatedFiles(libraryID);
}
yield this.local.resolveConflicts(libraryID);
var downloadForced = yield this.local.checkForForcedDownloads(libraryID);
// If we don't have any forced downloads, we can skip downloads if the last sync time hasn't
// changed or doesn't exist on the server (meaning there are no files)
if (downloadAll && !downloadForced) {
if (lastSyncTime) {
if (this.library.lastStorageSync == lastSyncTime) {
Zotero.debug("Last " + this.mode.toUpperCase() + " sync id hasn't changed for "
+ this.library.name + " -- skipping file downloads");
downloadAll = false;
}
}
else {
Zotero.debug(`No last ${this.mode} sync time for ${this.library.name}`
+ " -- skipping file downloads");
downloadAll = false;
}
}
// Get files to download
if (downloadAll || downloadForced) {
let itemIDs = yield this.local.getFilesToDownload(libraryID, !downloadAll);
if (itemIDs.length) {
Zotero.debug(itemIDs.length + " file" + (itemIDs.length == 1 ? '' : 's') + " to "
+ "download for " + this.library.name);
for (let itemID of itemIDs) {
let item = yield Zotero.Items.getAsync(itemID);
yield this.queueItem(item);
}
}
else {
Zotero.debug("No files to download for " + this.library.name);
}
}
// Get files to upload
if (Zotero.Libraries.isFilesEditable(libraryID)) {
let itemIDs = yield this.local.getFilesToUpload(libraryID);
if (itemIDs.length) {
Zotero.debug(itemIDs.length + " file" + (itemIDs.length == 1 ? '' : 's') + " to "
+ "upload for " + this.library.name);
for (let itemID of itemIDs) {
let item = yield Zotero.Items.getAsync(itemID, { noCache: true });
yield this.queueItem(item);
}
}
else {
Zotero.debug("No files to upload for " + this.library.name);
}
}
else {
Zotero.debug("No file editing access -- skipping file uploads for " + this.library.name);
}
var promises = {
download: this.queues.download.runAll(),
upload: this.queues.upload.runAll()
}
// Process the results
var changes = new Zotero.Sync.Storage.Result;
for (let type of ['download', 'upload']) {
let results = yield promises[type];
if (this.stopOnError) {
for (let p of results) {
if (p.isRejected()) {
let e = p.reason();
Zotero.debug(`File ${type} sync failed for ${this.library.name}`);
throw e;
}
}
}
Zotero.debug(`File ${type} sync finished for ${this.library.name}`);
changes.updateFromResults(results.filter(p => p.isFulfilled()).map(p => p.value()));
}
// If files were uploaded, update the remote last-sync time
if (changes.remoteChanges) {
lastSyncTime = yield this.controller.setLastSyncTime(libraryID);
if (!lastSyncTime) {
throw new Error("Last sync time not set after sync");
}
}
// If there's a remote last-sync time from either the check before downloads or when it
// was changed after uploads, store that locally so we know we can skip download checks
// next time
if (lastSyncTime) {
this.library.lastStorageSync = lastSyncTime;
yield this.library.saveTx();
}
// If WebDAV sync, purge deleted and orphaned files
if (this.mode == 'webdav') {
try {
yield this.controller.purgeDeletedStorageFiles(libraryID);
yield this.controller.purgeOrphanedStorageFiles(libraryID);
}
catch (e) {
Zotero.logError(e);
}
}
if (!changes.localChanges) {
Zotero.debug("No local changes made during file sync");
}
Zotero.debug("Done with file sync for " + this.library.name);
return changes;
})
Zotero.Sync.Storage.Engine.prototype.stop = function () {
for (let type in this.queues) {
this.queues[type].stop();
}
}
Zotero.Sync.Storage.Engine.prototype.queueItem = Zotero.Promise.coroutine(function* (item) {
switch (yield this.local.getSyncState(item.id)) {
case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD:
var type = 'download';
var onStart = Zotero.Promise.method(function (request) {
return this.controller.downloadFile(request);
}.bind(this));
break;
case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD:
var type = 'upload';
var onStart = Zotero.Promise.method(function (request) {
return this.controller.uploadFile(request);
}.bind(this));
break;
case false:
Zotero.debug("Sync state for item " + item.id + " not found", 2);
return;
default:
throw new Error("Invalid sync state " + (yield this.local.getSyncState(item.id)));
}
var request = new Zotero.Sync.Storage.Request({
type,
libraryID: this.libraryID,
name: item.libraryKey,
onStart,
onProgress: this.onProgress
});
if (type == 'upload') {
try {
request.setMaxSize(yield Zotero.Attachments.getTotalFileSize(item));
}
// If this fails, ignore it, though we might fail later
catch (e) {
// But if the file doesn't exist yet, don't try to upload it
//
// This isn't a perfect test, because the file could still be in the process of being
// downloaded (e.g., from the web). It'd be better to download files to a temp
// directory and move them into place.
if (!(yield item.getFilePathAsync())) {
Zotero.debug("File " + item.libraryKey + " not yet available to upload -- skipping");
return;
}
Zotero.logError(e);
}
}
this.queues[type].add(request.start.bind(request));
})

1088
chrome/content/zotero/xpcom/storage/storageLocal.js Normal file
View File

File diff suppressed because it is too large Load Diff

184
chrome/content/zotero/xpcom/storage/request.js → chrome/content/zotero/xpcom/storage/storageRequest.js
View File

@ -27,15 +27,25 @@
/**
* Transfer request for storage sync
*
* @param {String} name Identifier for request (e.g., "[libraryID]/[key]")
* @param {Function} onStart Callback to run when request starts
* @param {Object} options
* @param {String} options.type
* @param {Integer} options.libraryID
* @param {String} options.name - Identifier for request (e.g., "[libraryID]/[key]")
* @param {Function|Function[]} [options.onStart]
* @param {Function|Function[]} [options.onProgress]
* @param {Function|Function[]} [options.onStop]
*/
Zotero.Sync.Storage.Request = function (name, callbacks) {
Zotero.debug("Initializing request '" + name + "'");
Zotero.Sync.Storage.Request = function (options) {
if (!options.type) throw new Error("type must be provided");
if (!options.libraryID) throw new Error("libraryID must be provided");
if (!options.name) throw new Error("name must be provided");
['type', 'libraryID', 'name'].forEach(x => this[x] = options[x]);
this.callbacks = ['onStart', 'onProgress'];
Zotero.debug(`Initializing ${this.type} request ${this.name}`);
this.name = name;
this.callbacks = ['onStart', 'onProgress', 'onStop'];
this.Type = Zotero.Utilities.capitalize(this.type);
this.channel = null;
this.queue = null;
this.progress = 0;
@ -48,17 +58,10 @@ Zotero.Sync.Storage.Request = function (name, callbacks) {
this._remaining = null;
this._maxSize = null;
this._finished = false;
this._forceFinish = false;
this._changesMade = false;
for (var func in callbacks) {
if (this.callbacks.indexOf(func) !== -1) {
// Stuff all single functions into arrays
this['_' + func] = typeof callbacks[func] === 'function' ? [callbacks[func]] : callbacks[func];
}
else {
throw new Error("Invalid handler '" + func + "'");
}
for (let name of this.callbacks) {
if (!options[name]) continue;
this['_' + name] = Array.isArray(options[name]) ? options[name] : [options[name]];
}
}
@ -99,11 +102,6 @@ Zotero.Sync.Storage.Request.prototype.importCallbacks = function (request) {
}
Zotero.Sync.Storage.Request.prototype.__defineGetter__('promise', function () {
return this._deferred.promise;
});
Zotero.Sync.Storage.Request.prototype.__defineGetter__('percentage', function () {
if (this._finished) {
return 100;
@ -142,7 +140,7 @@ Zotero.Sync.Storage.Request.prototype.__defineGetter__('remaining', function ()
}
if (!this.progressMax) {
if (this.queue.type == 'upload' && this._maxSize) {
if (this.type == 'upload' && this._maxSize) {
return Math.round(Zotero.Sync.Storage.compressionTracker.ratio * this._maxSize);
}
@ -175,72 +173,47 @@ Zotero.Sync.Storage.Request.prototype.setChannel = function (channel) {
}
Zotero.Sync.Storage.Request.prototype.start = function () {
if (!this.queue) {
throw ("Request " + this.name + " must be added to a queue before starting");
}
Zotero.debug("Starting " + this.queue.name + " request " + this.name);
Zotero.Sync.Storage.Request.prototype.start = Zotero.Promise.coroutine(function* () {
Zotero.debug("Starting " + this.type + " request " + this.name);
if (this._running) {
throw new Error("Request " + this.name + " already running");
throw new Error(this.type + " request " + this.name + " already running");
}
if (!this._onStart) {
throw new Error("onStart not provided -- nothing to do!");
}
this._running = true;
this.queue.activeRequests++;
if (this.queue.type == 'download') {
Zotero.Sync.Storage.setItemDownloadPercentage(this.name, 0);
}
var self = this;
// this._onStart is an array of promises returning changesMade.
// this._onStart is an array of promises for objects of result flags, which are combined
// into a single object here
//
// The main sync logic is triggered here.
Zotero.Promise.all([f(this) for each(f in this._onStart)])
.then(function (results) {
return {
localChanges: results.some(function (val) val && val.localChanges == true),
remoteChanges: results.some(function (val) val && val.remoteChanges == true),
conflict: results.reduce(function (prev, cur) {
return prev.conflict ? prev : cur;
}).conflict
};
})
.then(function (results) {
Zotero.debug(results);
try {
var results = yield Zotero.Promise.all(this._onStart.map(f => f(this)));
if (results.localChanges) {
Zotero.debug("Changes were made by " + self.queue.name
+ " request " + self.name);
}
else {
Zotero.debug("No changes were made by " + self.queue.name
+ " request " + self.name);
}
var result = new Zotero.Sync.Storage.Result;
result.updateFromResults(results);
// This promise updates localChanges/remoteChanges on the queue
self._deferred.resolve(results);
})
.catch(function (e) {
if (self._stopping) {
Zotero.debug("Skipping error for stopping request " + self.name);
return;
Zotero.debug(this.Type + " request " + this.name + " finished");
Zotero.debug(result + "");
return result;
}
catch (e) {
Zotero.logError(this.Type + " request " + this.name + " failed");
throw e;
}
finally {
this._finished = true;
this._running = false;
if (this._onStop) {
this._onStop.forEach(x => x());
}
Zotero.debug(self.queue.Type + " request " + self.name + " failed");
self._deferred.reject(e);
})
// Finish the request (and in turn the queue, if this is the last request)
.finally(function () {
if (!self._finished) {
self._finish();
}
});
return this._deferred.promise;
}
}
});
Zotero.Sync.Storage.Request.prototype.isRunning = function () {
@ -263,7 +236,7 @@ Zotero.Sync.Storage.Request.prototype.isFinished = function () {
* @param {Integer} progressMax Max progress value for this request
* (usually total bytes)
*/
Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress, progressMax) {
Zotero.Sync.Storage.Request.prototype.onProgress = function (progress, progressMax) {
//Zotero.debug(progress + "/" + progressMax + " for request " + this.name);
if (!this._running) {
@ -273,10 +246,6 @@ Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress,
return;
}
if (!this.channel) {
this.channel = channel;
}
// Workaround for invalid progress values (possibly related to
// https://bugzilla.mozilla.org/show_bug.cgi?id=451991 and fixed in 3.1)
if (progress < this.progress) {
@ -292,9 +261,8 @@ Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress,
this.progress = progress;
this.progressMax = progressMax;
this.queue.updateProgress();
if (this.queue.type == 'download') {
if (this.type == 'download') {
Zotero.Sync.Storage.setItemDownloadPercentage(this.name, this.percentage);
}
@ -310,59 +278,15 @@ Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress,
* Stop the request's underlying network request, if there is one
*/
Zotero.Sync.Storage.Request.prototype.stop = function (force) {
if (force) {
this._forceFinish = true;
}
if (this.channel && this.channel.isPending()) {
this._stopping = true;
try {
Zotero.debug("Stopping request '" + this.name + "'");
Zotero.debug(`Stopping ${this.type} request '${this.name} '`);
this.channel.cancel(0x804b0002); // NS_BINDING_ABORTED
}
catch (e) {
Zotero.debug(e);
Zotero.debug(e, 1);
}
}
else {
this._finish();
}
}
/**
* Mark request as finished and notify queue that it's done
*/
Zotero.Sync.Storage.Request.prototype._finish = function () {
// If an error occurred, we wait to finish the request, since doing
// so might end the queue before the error flag has been set on the queue.
// When the queue's error handler stops the queue, it stops the request
// with stop(true) to force the finish to occur, allowing the queue's
// promise to be rejected with the error.
if (!this._forceFinish && this._deferred.promise.isRejected()) {
return;
}
Zotero.debug("Finishing " + this.queue.name + " request '" + this.name + "'");
this._finished = true;
var active = this._running;
this._running = false;
Zotero.Sync.Storage.setItemDownloadPercentage(this.name, false);
if (active) {
this.queue.activeRequests--;
}
// TEMP: mechanism for failures?
try {
this.queue.finishedRequests++;
this.queue.updateProgress();
}
catch (e) {
Zotero.debug(e, 1);
Components.utils.reportError(e);
this._deferred.reject(e);
throw e;
}
}

47
chrome/content/zotero/xpcom/storage/storageResult.js Normal file
View File

@ -0,0 +1,47 @@
"use strict";
/**
* @property {Boolean} localChanges - Changes were made locally. For logging purposes only.
* @property {Boolean} remoteChanges - Changes were made on the server. This causes the
* last-sync time to be updated on the server (WebDAV) or retrieved (ZFS) and stored locally
* to skip additional file syncs until further server changes are made.
* @property {Boolean} syncRequired - A data sync is required to upload local changes
* @propretty {Boolean} fileSyncRequired - Another file sync is required to handle files left in
* conflict
*/
Zotero.Sync.Storage.Result = function (options = {}) {
this._props = ['localChanges', 'remoteChanges', 'syncRequired', 'fileSyncRequired'];
for (let prop of this._props) {
this[prop] = options[prop] || false;
}
}
/**
* Update the properties on this object from multiple Result objects
*
* @param {Zotero.Sync.Storage.Result[]} results
*/
Zotero.Sync.Storage.Result.prototype.updateFromResults = function (results) {
for (let prop of this._props) {
if (!this[prop]) {
for (let result of results) {
if (!(result instanceof Zotero.Sync.Storage.Result)) {
Zotero.debug(result, 1);
throw new Error("'result' is not a storage result");
}
if (result[prop]) {
this[prop] = true;
}
}
}
}
}
Zotero.Sync.Storage.Result.prototype.toString = function () {
var obj = {};
for (let prop of this._props) {
obj[prop] = this[prop] || false;
}
return JSON.stringify(obj, null, " ");
}

67
chrome/content/zotero/xpcom/storage/storageUtilities.js Normal file
View File

@ -0,0 +1,67 @@
Zotero.Sync.Storage.Utilities = {
getClassForMode: function (mode) {
switch (mode) {
case 'zfs':
return Zotero.Sync.Storage.ZFS_Module;
case 'webdav':
return Zotero.Sync.Storage.WebDAV_Module;
default:
throw new Error("Invalid storage mode '" + mode + "'");
}
},
getItemFromRequest: function (request) {
var [libraryID, key] = request.name.split('/');
return Zotero.Items.getByLibraryAndKey(libraryID, key);
},
/**
* Create zip file of attachment directory in the temp directory
*
* @param {Zotero.Sync.Storage.Request} request
* @return {Promise<Boolean>} - True if the zip file was created, false otherwise
*/
createUploadFile: Zotero.Promise.coroutine(function* (request) {
var item = this.getItemFromRequest(request);
Zotero.debug("Creating ZIP file for item " + item.libraryKey);
switch (item.attachmentLinkMode) {
case Zotero.Attachments.LINK_MODE_LINKED_FILE:
case Zotero.Attachments.LINK_MODE_LINKED_URL:
throw new Error("Upload file must be an imported snapshot or file");
}
var zipFile = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.zip');
return Zotero.File.zipDirectory(
Zotero.Attachments.getStorageDirectory(item).path,
zipFile,
{
onStopRequest: function (req, context, status) {
var zipFileName = OS.Path.basename(zipFile);
var originalSize = 0;
for (let entry of context.entries) {
let zipEntry = context.zipWriter.getEntry(entry.name);
if (!zipEntry) {
Zotero.logError("ZIP entry '" + entry.name + "' not found for "
+ "request '" + request.name + "'")
continue;
}
originalSize += zipEntry.realSize;
}
Zotero.debug("Zip of " + zipFileName + " finished with status " + status
+ " (original " + Math.round(originalSize / 1024) + "KB, "
+ "compressed " + Math.round(context.zipWriter.file.fileSize / 1024) + "KB, "
+ Math.round(
((originalSize - context.zipWriter.file.fileSize) / originalSize) * 100
) + "% reduction)");
}
}
);
})
}

79
chrome/content/zotero/xpcom/storage/streamListener.js
View File

@ -30,10 +30,9 @@
* Possible properties of data object:
* - onStart: f(request)
* - onProgress: f(request, progress, progressMax)
* - onStop: f(request, status, response, data)
* - onCancel: f(request, status, data)
* - onStop: f(request, status, response)
* - onCancel: f(request, status)
* - streams: array of streams to close on completion
* - Other values to pass to onStop()
*/
Zotero.Sync.Storage.StreamListener = function (data) {
this._data = data;
@ -110,17 +109,15 @@ Zotero.Sync.Storage.StreamListener.prototype = {
},
onStateChange: function (wp, request, stateFlags, status) {
Zotero.debug("onStateChange");
Zotero.debug(stateFlags);
Zotero.debug(status);
Zotero.debug("onStateChange with " + stateFlags);
if ((stateFlags & Components.interfaces.nsIWebProgressListener.STATE_START)
&& (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_NETWORK)) {
this._onStart(request);
}
else if ((stateFlags & Components.interfaces.nsIWebProgressListener.STATE_STOP)
&& (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_NETWORK)) {
this._onStop(request, status);
if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_REQUEST) {
if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_START) {
this._onStart(request);
}
else if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_STOP) {
this._onStop(request, status);
}
}
},
@ -148,18 +145,38 @@ Zotero.Sync.Storage.StreamListener.prototype = {
},
// nsIChannelEventSink
onChannelRedirect: function (oldChannel, newChannel, flags) {
//
// If this._data.onChannelRedirect exists, it should return a promise resolving to true to
// follow the redirect or false to cancel it
onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) {
Zotero.debug('onChannelRedirect');
if (this._data && this._data.onChannelRedirect) {
let result = yield this._data.onChannelRedirect(oldChannel, newChannel, flags);
if (!result) {
oldChannel.cancel(Components.results.NS_BINDING_ABORTED);
newChannel.cancel(Components.results.NS_BINDING_ABORTED);
return false;
}
}
// if redirecting, store the new channel
this._channel = newChannel;
},
}),
asyncOnChannelRedirect: function (oldChan, newChan, flags, redirectCallback) {
Zotero.debug('asyncOnRedirect');
this.onChannelRedirect(oldChan, newChan, flags);
redirectCallback.onRedirectVerifyCallback(0);
this.onChannelRedirect(oldChan, newChan, flags)
.then(function (result) {
redirectCallback.onRedirectVerifyCallback(
result ? Components.results.NS_SUCCEEDED : Components.results.NS_FAILED
);
})
.catch(function (e) {
Zotero.logError(e);
redirectCallback.onRedirectVerifyCallback(Components.results.NS_FAILED);
});
},
// nsIHttpEventSink
@ -177,8 +194,7 @@ Zotero.Sync.Storage.StreamListener.prototype = {
_onStart: function (request) {
Zotero.debug('Starting request');
if (this._data && this._data.onStart) {
var data = this._getPassData();
this._data.onStart(request, data);
this._data.onStart(request);
}
},
@ -189,7 +205,6 @@ Zotero.Sync.Storage.StreamListener.prototype = {
},
_onStop: function (request, status) {
Zotero.debug('Request ended with status ' + status);
var cancelled = status == 0x804b0002; // NS_BINDING_ABORTED
if (!cancelled && status == 0 && request instanceof Components.interfaces.nsIHttpChannel) {
@ -201,9 +216,11 @@ Zotero.Sync.Storage.StreamListener.prototype = {
Zotero.debug("Request responseStatus not available", 1);
status = 0;
}
Zotero.debug('Request ended with status code ' + status);
request.QueryInterface(Components.interfaces.nsIRequest);
}
else {
Zotero.debug('Request ended with status ' + status);
status = 0;
}
@ -213,38 +230,20 @@ Zotero.Sync.Storage.StreamListener.prototype = {
}
}
var data = this._getPassData();
if (cancelled) {
if (this._data.onCancel) {
this._data.onCancel(request, status, data);
this._data.onCancel(request, status);
}
}
else {
if (this._data.onStop) {
this._data.onStop(request, status, this._response, data);
this._data.onStop(request, status, this._response);
}
}
this._channel = null;
},
_getPassData: function () {
// Make copy of data without callbacks to pass along
var passData = {};
for (var i in this._data) {
switch (i) {
case "onStart":
case "onProgress":
case "onStop":
case "onCancel":
continue;
}
passData[i] = this._data[i];
}
return passData;
},
// nsIInterfaceRequestor
getInterface: function (iid) {
try {

1579
chrome/content/zotero/xpcom/storage/webdav.js
View File

File diff suppressed because it is too large Load Diff

2003
chrome/content/zotero/xpcom/storage/zfs.js
View File

File diff suppressed because it is too large Load Diff

37
chrome/content/zotero/xpcom/sync.js
View File

@ -1488,43 +1488,6 @@ Zotero.Sync.Server = new function () {
Zotero.BufferedInputListener = function (callback) {
this._callback = callback;
}
Zotero.BufferedInputListener.prototype = {
binaryInputStream: null,
size: 0,
data: '',
onStartRequest: function(request, context) {},
onStopRequest: function(request, context, status) {
this.binaryInputStream.close();
delete this.binaryInputStream;
this._callback(this.data);
},
onDataAvailable: function(request, context, inputStream, offset, count) {
this.size += count;
this.binaryInputStream = Components.classes["@mozilla.org/binaryinputstream;1"]
.createInstance(Components.interfaces.nsIBinaryInputStream)
this.binaryInputStream.setInputStream(inputStream);
this.data += this.binaryInputStream.readBytes(this.binaryInputStream.available());
},
QueryInterface: function (iid) {
if (iid.equals(Components.interfaces.nsISupports)
|| iid.equals(Components.interfaces.nsIStreamListener)) {
return this;
}
throw Components.results.NS_ERROR_NO_INTERFACE;
}
}
Zotero.Sync.Server.Data = new function() {
var _noMergeTypes = ['search'];

105
chrome/content/zotero/xpcom/sync/syncAPIClient.js
View File

@ -28,14 +28,15 @@ if (!Zotero.Sync) {
}
Zotero.Sync.APIClient = function (options) {
this.baseURL = options.baseURL;
this.apiKey = options.apiKey;
this.concurrentCaller = options.concurrentCaller;
if (!options.baseURL) throw new Error("baseURL not set");
if (!options.apiVersion) throw new Error("apiVersion not set");
if (!options.apiKey) throw new Error("apiKey not set");
if (!options.caller) throw new Error("caller not set");
if (options.apiVersion == undefined) {
throw new Error("options.apiVersion not set");
}
this.baseURL = options.baseURL;
this.apiVersion = options.apiVersion;
this.apiKey = options.apiKey;
this.caller = options.caller;
}
Zotero.Sync.APIClient.prototype = {
@ -44,7 +45,7 @@ Zotero.Sync.APIClient.prototype = {
getKeyInfo: Zotero.Promise.coroutine(function* () {
var uri = this.baseURL + "keys/" + this.apiKey;
var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 404] });
var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 404] });
if (xmlhttp.status == 404) {
return false;
}
@ -63,7 +64,7 @@ Zotero.Sync.APIClient.prototype = {
if (!userID) throw new Error("User ID not provided");
var uri = this.baseURL + "users/" + userID + "/groups?format=versions";
var xmlhttp = yield this._makeRequest("GET", uri);
var xmlhttp = yield this.makeRequest("GET", uri);
return this._parseJSON(xmlhttp.responseText);
}),
@ -76,7 +77,7 @@ Zotero.Sync.APIClient.prototype = {
if (!groupID) throw new Error("Group ID not provided");
var uri = this.baseURL + "groups/" + groupID;
var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 404] });
var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 404] });
if (xmlhttp.status == 404) {
return false;
}
@ -93,7 +94,7 @@ Zotero.Sync.APIClient.prototype = {
if (since) {
params.since = since;
}
var uri = this._buildRequestURI(params);
var uri = this.buildRequestURI(params);
var options = {
successCodes: [200, 304]
};
@ -102,7 +103,7 @@ Zotero.Sync.APIClient.prototype = {
"If-Modified-Since-Version": since
};
}
var xmlhttp = yield this._makeRequest("GET", uri, options);
var xmlhttp = yield this.makeRequest("GET", uri, options);
if (xmlhttp.status == 304) {
return false;
}
@ -128,8 +129,8 @@ Zotero.Sync.APIClient.prototype = {
libraryTypeID: libraryTypeID,
since: since || 0
};
var uri = this._buildRequestURI(params);
var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 409] });
var uri = this.buildRequestURI(params);
var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 409] });
if (xmlhttp.status == 409) {
Zotero.debug(`'since' value '${since}' is earlier than the beginning of the delete log`);
return false;
@ -154,7 +155,7 @@ Zotero.Sync.APIClient.prototype = {
* @param {String} libraryType 'user' or 'group'
* @param {Integer} libraryTypeID userID or groupID
* @param {String} objectType 'item', 'collection', 'search'
* @param {Object} queryParams Query parameters (see _buildRequestURI())
* @param {Object} queryParams Query parameters (see buildRequestURI())
* @return {Promise<Object>|FALSE} Object with 'libraryVersion' and 'results'
*/
getVersions: Zotero.Promise.coroutine(function* (libraryType, libraryTypeID, objectType, queryParams, libraryVersion) {
@ -176,7 +177,7 @@ Zotero.Sync.APIClient.prototype = {
}
// TODO: Use pagination
var uri = this._buildRequestURI(params);
var uri = this.buildRequestURI(params);
var options = {
successCodes: [200, 304]
@ -186,7 +187,7 @@ Zotero.Sync.APIClient.prototype = {
"If-Modified-Since-Version": libraryVersion
};
}
var xmlhttp = yield this._makeRequest("GET", uri, options);
var xmlhttp = yield this.makeRequest("GET", uri, options);
if (xmlhttp.status == 304) {
return false;
}
@ -256,10 +257,10 @@ Zotero.Sync.APIClient.prototype = {
if (objectType == 'item') {
params.includeTrashed = 1;
}
var uri = this._buildRequestURI(params);
var uri = this.buildRequestURI(params);
return [
this._makeRequest("GET", uri)
this.makeRequest("GET", uri)
.then(function (xmlhttp) {
return this._parseJSON(xmlhttp.responseText)
}.bind(this))
@ -294,9 +295,9 @@ Zotero.Sync.APIClient.prototype = {
libraryType: libraryType,
libraryTypeID: libraryTypeID
};
var uri = this._buildRequestURI(params);
var uri = this.buildRequestURI(params);
var xmlhttp = yield this._makeRequest(method, uri, {
var xmlhttp = yield this.makeRequest(method, uri, {
headers: {
"If-Unmodified-Since-Version": version
},
@ -319,7 +320,7 @@ Zotero.Sync.APIClient.prototype = {
}),
_buildRequestURI: function (params) {
buildRequestURI: function (params) {
var uri = this.baseURL;
switch (params.libraryType) {
@ -332,6 +333,10 @@ Zotero.Sync.APIClient.prototype = {
break;
}
if (params.target === undefined) {
throw new Error("'target' not provided");
}
uri += "/" + params.target;
if (params.objectKey) {
@ -382,30 +387,33 @@ Zotero.Sync.APIClient.prototype = {
},
_makeRequest: function (method, uri, options) {
if (!options) {
options = {};
getHeaders: function (headers = {}) {
headers["Zotero-API-Version"] = this.apiVersion;
if (this.apiKey) {
headers["Zotero-API-Key"] = this.apiKey;
}
if (!options.headers) {
options.headers = {};
}
options.headers["Zotero-API-Version"] = this.apiVersion;
return headers;
},
makeRequest: function (method, uri, options = {}) {
options.headers = this.getHeaders(options.headers);
options.dontCache = true;
options.foreground = !options.background;
options.responseType = options.responseType || 'text';
if (this.apiKey) {
options.headers.Authorization = "Bearer " + this.apiKey;
}
var self = this;
return this.concurrentCaller.fcall(function () {
return Zotero.HTTP.request(method, uri, options)
.catch(function (e) {
if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
self._checkResponse(e.xmlhttp);
}
return this.caller.start(Zotero.Promise.coroutine(function* () {
try {
var xmlhttp = yield Zotero.HTTP.request(method, uri, options);
this._checkBackoff(xmlhttp);
return xmlhttp;
}
catch (e) {
/*if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
this._checkRetry(e.xmlhttp);
}*/
throw e;
});
});
}
}.bind(this)));
},
@ -422,21 +430,6 @@ Zotero.Sync.APIClient.prototype = {
},
_checkResponse: function (xmlhttp) {
this._checkBackoff(xmlhttp);
this._checkAuth(xmlhttp);
},
_checkAuth: function (xmlhttp) {
if (xmlhttp.status == 403) {
var e = new Zotero.Error(Zotero.getString('sync.error.invalidLogin'), "INVALID_SYNC_LOGIN");
e.fatal = true;
throw e;
}
},
_checkBackoff: function (xmlhttp) {
var backoff = xmlhttp.getResponseHeader("Backoff");
if (backoff) {
@ -444,7 +437,7 @@ Zotero.Sync.APIClient.prototype = {
if (backoff > 3600) {
// TODO: Update status?
this.concurrentCaller.pause(backoff * 1000);
this.caller.pause(backoff * 1000);
}
}
}

70
chrome/content/zotero/xpcom/sync/syncEngine.js
View File

@ -76,7 +76,13 @@ Zotero.Sync.Data.Engine = function (options) {
onError: this.onError
}
this.syncCachePromise = Zotero.Promise.resolve().bind(this);
Components.utils.import("resource://zotero/concurrentCaller.js");
this.syncCacheProcessor = new ConcurrentCaller({
id: "Sync Cache Processor",
numConcurrent: 1,
logger: Zotero.debug,
stopOnError: this.stopOnError
});
};
Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_SUCCESS = 1;
@ -167,12 +173,8 @@ Zotero.Sync.Data.Engine.prototype.start = Zotero.Promise.coroutine(function* ()
}
}
// TEMP: make more reliable
while (this.syncCachePromise.isPending()) {
Zotero.debug("Waiting for sync cache to be processed");
yield this.syncCachePromise;
yield Zotero.Promise.delay(50);
}
Zotero.debug("Waiting for sync cache to be processed");
yield this.syncCacheProcessor.wait();
yield Zotero.Libraries.updateLastSyncTime(this.libraryID);
@ -286,12 +288,7 @@ Zotero.Sync.Data.Engine.prototype._startDownload = Zotero.Promise.coroutine(func
}
// Wait for sync process to clear
// TEMP: make more reliable
while (this.syncCachePromise.isPending()) {
Zotero.debug("Waiting for sync cache to be processed");
yield this.syncCachePromise;
yield Zotero.Promise.delay(50);
}
yield this.syncCacheProcessor.wait();
//
// Get deleted objects
@ -671,7 +668,8 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi
if (state == 'successful') {
// Update local object with saved data if necessary
yield obj.fromJSON(current.data);
yield obj.loadAllData();
obj.fromJSON(current.data);
toSave.push(obj);
toCache.push(current);
}
@ -701,8 +699,11 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi
// Handle failed objects
for (let index in json.results.failed) {
let e = json.results.failed[index];
Zotero.logError(e.message);
let { code, message } = json.results.failed[index];
e = new Error(message);
e.name = "ZoteroUploadObjectError";
e.code = code;
Zotero.logError(e);
// This shouldn't happen, because the upload request includes a library
// version and should prevent an outdated upload before the object version is
@ -711,12 +712,11 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi
return this.UPLOAD_RESULT_OBJECT_CONFLICT;
}
if (this.stopOnError) {
Zotero.debug("WE FAILED!!!");
throw new Error(e.message);
}
if (this.onError) {
this.onError(e.message);
this.onError(e);
}
if (this.stopOnError) {
throw new Error(e);
}
batch[index].tries++;
// Mark 400 errors as permanently failed
@ -990,7 +990,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
this._failedCheck();
let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
let ObjectType = objectType[0].toUpperCase() + objectType.substr(1);
let ObjectType = Zotero.Utilities.capitalize(objectType);
// TODO: localize
this.setStatus("Updating " + objectTypePlural + " in " + this.libraryName);
@ -1037,8 +1037,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
let cacheVersions = yield Zotero.Sync.Data.Local.getLatestCacheObjectVersions(
this.libraryID, objectType
);
// Queue objects that are out of date or don't exist locally and aren't up-to-date
// in the cache
// Queue objects that are out of date or don't exist locally
for (let key in results.versions) {
let version = results.versions[key];
let obj = yield objectsClass.getByLibraryAndKeyAsync(this.libraryID, key, {
@ -1060,12 +1059,12 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
}
if (obj) {
Zotero.debug(Zotero.Utilities.capitalize(objectType) + " " + obj.libraryKey
Zotero.debug(ObjectType + " " + obj.libraryKey
+ " is older than version in sync cache");
}
else {
Zotero.debug(Zotero.Utilities.capitalize(objectType) + " "
+ this.libraryID + "/" + key + " in sync cache not found locally");
Zotero.debug(ObjectType + " " + this.libraryID + "/" + key
+ " in sync cache not found locally");
}
toDownload.push(key);
@ -1127,7 +1126,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
break;
}
yield this.syncCachePromise;
yield this.syncCacheProcessor.wait();
yield Zotero.Libraries.setVersion(this.libraryID, lastLibraryVersion);
@ -1145,20 +1144,19 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
* @param {String} objectType
*/
Zotero.Sync.Data.Engine.prototype._processCache = function (objectType) {
var self = this;
this.syncCachePromise = this.syncCachePromise.then(function () {
self._failedCheck();
this.syncCacheProcessor.start(function () {
this._failedCheck();
return Zotero.Sync.Data.Local.processSyncCacheForObjectType(
self.libraryID, objectType, self.options
this.libraryID, objectType, this.options
)
.catch(function (e) {
Zotero.logError(e);
if (self.stopOnError) {
if (this.stopOnError) {
Zotero.debug("WE FAILED!!!");
self.failed = e;
this.failed = e;
}
});
})
}.bind(this));
}.bind(this))
}

56
chrome/content/zotero/xpcom/sync/syncEventListeners.js
View File

@ -39,10 +39,9 @@ Zotero.Sync.EventListeners.ChangeListener = new function () {
var syncSQL = "REPLACE INTO syncDeleteLog (syncObjectTypeID, libraryID, key, synced) "
+ "VALUES (?, ?, ?, 0)";
var storageSQL = "REPLACE INTO storageDeleteLog VALUES (?, ?, 0)";
if (type == 'item' && Zotero.Sync.Storage.WebDAV.includeUserFiles) {
var storageSQL = "REPLACE INTO storageDeleteLog VALUES (?, ?, 0)";
}
var storageForLibrary = {};
return Zotero.DB.executeTransaction(function* () {
for (let i = 0; i < ids.length; i++) {
@ -74,18 +73,25 @@ Zotero.Sync.EventListeners.ChangeListener = new function () {
key
]
);
if (storageSQL && oldItem.itemType == 'attachment' &&
[
Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
Zotero.Attachments.LINK_MODE_IMPORTED_URL
].indexOf(oldItem.linkMode) != -1) {
yield Zotero.DB.queryAsync(
storageSQL,
[
libraryID,
key
]
);
if (type == 'item') {
if (storageForLibrary[libraryID] === undefined) {
storageForLibrary[libraryID] =
Zotero.Sync.Storage.Local.getModeForLibrary(libraryID) == 'webdav';
}
if (storageForLibrary[libraryID] && oldItem.itemType == 'attachment' &&
[
Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
Zotero.Attachments.LINK_MODE_IMPORTED_URL
].indexOf(oldItem.linkMode) != -1) {
yield Zotero.DB.queryAsync(
storageSQL,
[
libraryID,
key
]
);
}
}
}
});
@ -215,3 +221,23 @@ Zotero.Sync.EventListeners.progressListener = {
}
};
Zotero.Sync.EventListeners.StorageFileOpenListener = {
init: function () {
Zotero.Notifier.registerObserver(this, ['file'], 'storageFileOpen');
},
notify: function (event, type, ids, extraData) {
if (event == 'open' && type == 'file') {
let timestamp = new Date().getTime();
for (let i = 0; i < ids.length; i++) {
Zotero.Sync.Storage.Local.uploadCheckFiles.push({
itemID: ids[i],
timestamp: timestamp
});
}
}
}
}

333
chrome/content/zotero/xpcom/sync/syncLocal.js
View File

@ -28,6 +28,8 @@ if (!Zotero.Sync.Data) {
}
Zotero.Sync.Data.Local = {
_loginManagerHost: 'https://api.zotero.org',
_loginManagerRealm: 'Zotero Web API',
_lastSyncTime: null,
_lastClassicSyncTime: null,
@ -39,6 +41,71 @@ Zotero.Sync.Data.Local = {
}),
getAPIKey: function () {
var apiKey = Zotero.Prefs.get('devAPIKey');
if (apiKey) {
return apiKey;
}
var loginManager = Components.classes["@mozilla.org/login-manager;1"]
.getService(Components.interfaces.nsILoginManager);
var logins = loginManager.findLogins(
{}, this._loginManagerHost, null, this._loginManagerRealm
);
// Get API from returned array of nsILoginInfo objects
if (logins.length) {
return logins[0].password;
}
if (!apiKey) {
let username = Zotero.Prefs.get('sync.server.username');
if (username) {
let password = Zotero.Sync.Data.Local.getLegacyPassword(username);
if (!password) {
return false;
}
throw new Error("Unimplemented");
// Get API key from server
// Store API key
// Remove old logins and username pref
}
}
return apiKey;
},
setAPIKey: function (apiKey) {
var loginManager = Components.classes["@mozilla.org/login-manager;1"]
.getService(Components.interfaces.nsILoginManager);
var nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1",
Components.interfaces.nsILoginInfo, "init");
var loginInfo = new nsLoginInfo(
this._loginManagerHost,
null,
this._loginManagerRealm,
'API Key',
apiKey,
"",
""
);
loginManager.addLogin(loginInfo);
},
getLegacyPassword: function (username) {
var loginManager = Components.classes["@mozilla.org/login-manager;1"]
.getService(Components.interfaces.nsILoginManager);
var logins = loginManager.findLogins({}, "chrome://zotero", "Zotero Storage Server", null);
// Find user from returned array of nsILoginInfo objects
for (let login of logins) {
if (login.username == username) {
return login.password;
}
}
return false;
},
getLastSyncTime: function () {
if (_lastSyncTime === null) {
throw new Error("Last sync time not yet loaded");
@ -86,7 +153,7 @@ Zotero.Sync.Data.Local = {
var sql = "SELECT " + objectsClass.idColumn + " FROM " + objectsClass.table
+ " WHERE libraryID=? AND synced=0";
// RETRIEVE PARENT DOWN? EVEN POSSIBLE?
// TODO: RETRIEVE PARENT DOWN? EVEN POSSIBLE?
// items via parent
// collections via getDescendents?
@ -154,6 +221,35 @@ Zotero.Sync.Data.Local = {
}),
getCacheObjects: Zotero.Promise.coroutine(function* (objectType, libraryID, keyVersionPairs) {
if (!keyVersionPairs.length) return [];
var sql = "SELECT data FROM syncCache SC JOIN (SELECT "
+ keyVersionPairs.map(function (pair) {
Zotero.DataObjectUtilities.checkKey(pair[0]);
return "'" + pair[0] + "' AS key, " + parseInt(pair[1]) + " AS version";
}).join(" UNION SELECT ")
+ ") AS pairs ON (pairs.key=SC.key AND pairs.version=SC.version) "
+ "WHERE libraryID=? AND "
+ "syncObjectTypeID IN (SELECT syncObjectTypeID FROM syncObjectTypes WHERE name=?)";
var rows = yield Zotero.DB.columnQueryAsync(sql, [libraryID, objectType]);
return rows.map(row => JSON.parse(row));
}),
saveCacheObject: Zotero.Promise.coroutine(function* (objectType, libraryID, json) {
json = this._checkCacheJSON(json);
Zotero.debug("Saving to sync cache:");
Zotero.debug(json);
var syncObjectTypeID = Zotero.Sync.Data.Utilities.getSyncObjectTypeID(objectType);
var sql = "INSERT OR REPLACE INTO syncCache "
+ "(libraryID, key, syncObjectTypeID, version, data) VALUES (?, ?, ?, ?, ?)";
var params = [libraryID, json.key, syncObjectTypeID, json.version, JSON.stringify(json)];
return Zotero.DB.queryAsync(sql, params);
}),
saveCacheObjects: Zotero.Promise.coroutine(function* (objectType, libraryID, jsonArray) {
if (!Array.isArray(jsonArray)) {
throw new Error("'json' must be an array");
@ -165,20 +261,7 @@ Zotero.Sync.Data.Local = {
return;
}
jsonArray = jsonArray.map(o => {
if (o.key === undefined) {
throw new Error("Missing 'key' property in JSON");
}
if (o.version === undefined) {
throw new Error("Missing 'version' property in JSON");
}
// If direct data object passed, wrap in fake response object
return o.data === undefined ? {
key: o.key,
version: o.version,
data: o
} : o;
});
jsonArray = jsonArray.map(json => this._checkCacheJSON(json));
Zotero.debug("Saving to sync cache:");
Zotero.debug(jsonArray);
@ -206,6 +289,22 @@ Zotero.Sync.Data.Local = {
}),
_checkCacheJSON: function (json) {
if (json.key === undefined) {
throw new Error("Missing 'key' property in JSON");
}
if (json.version === undefined) {
throw new Error("Missing 'version' property in JSON");
}
// If direct data object passed, wrap in fake response object
return json.data === undefined ? {
key: json.key,
version: json.version,
data: json
} : json;
},
processSyncCache: Zotero.Promise.coroutine(function* (libraryID, options) {
for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(libraryID)) {
yield this.processSyncCacheForObjectType(libraryID, objectType, options);
@ -213,8 +312,7 @@ Zotero.Sync.Data.Local = {
}),
processSyncCacheForObjectType: Zotero.Promise.coroutine(function* (libraryID, objectType, options) {
options = options || {};
processSyncCacheForObjectType: Zotero.Promise.coroutine(function* (libraryID, objectType, options = {}) {
var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
var ObjectType = Zotero.Utilities.capitalize(objectType);
@ -227,7 +325,6 @@ Zotero.Sync.Data.Local = {
var numSkipped = 0;
var data = yield this._getUnwrittenData(libraryID, objectType);
if (!data.length) {
Zotero.debug("No unwritten " + objectTypePlural + " in sync cache");
return;
@ -260,9 +357,9 @@ Zotero.Sync.Data.Local = {
for (let i = 0; i < chunk.length; i++) {
let json = chunk[i];
let jsonData = json.data;
let isNewObject;
let objectKey = json.key;
Zotero.debug(`Processing ${objectType} ${libraryID}/${objectKey}`);
Zotero.debug(json);
if (!jsonData) {
@ -302,26 +399,22 @@ Zotero.Sync.Data.Local = {
}*/
}
let isNewObject = false;
let skipCache = false;
let obj = yield objectsClass.getByLibraryAndKeyAsync(
libraryID, objectKey, { noCache: true }
);
if (obj) {
Zotero.debug("Matching local " + objectType + " exists", 4);
isNewObject = false;
// Local object has not been modified since last sync
if (obj.synced) {
// Overwrite local below
}
else {
// Local object has been modified since last sync
if (!obj.synced) {
Zotero.debug("Local " + objectType + " " + obj.libraryKey
+ " has been modified since last sync", 4);
let cachedJSON = yield this.getCacheObject(
objectType, obj.libraryID, obj.key, obj.version
);
Zotero.debug("GOT CACHED");
Zotero.debug(cachedJSON);
let jsonDataLocal = yield obj.toJSON();
@ -333,42 +426,51 @@ Zotero.Sync.Data.Local = {
['dateAdded', 'dateModified']
);
// If no changes, update local version and keep as unsynced
// If no changes, update local version number and mark as synced
if (!result.changes.length && !result.conflicts.length) {
Zotero.debug("No remote changes to apply to local " + objectType
+ " " + obj.libraryKey);
yield obj.updateVersion(json.version);
Zotero.debug("No remote changes to apply to local "
+ objectType + " " + obj.libraryKey);
obj.version = json.version;
obj.synced = true;
yield obj.save();
continue;
}
if (result.conflicts.length) {
if (objectType != 'item') {
throw new Error(`Unexpected conflict on ${objectType} object`);
}
Zotero.debug("Conflict!");
conflicts.push({
left: jsonDataLocal,
right: jsonData,
changes: result.changes,
conflicts: result.conflicts
});
continue;
}
// If no conflicts, apply remote changes automatically
if (!result.conflicts.length) {
Zotero.DataObjectUtilities.applyChanges(
jsonData, result.changes
);
let saved = yield this._saveObjectFromJSON(obj, jsonData, options);
if (saved) numSaved++;
continue;
}
if (objectType != 'item') {
throw new Error(`Unexpected conflict on ${objectType} object`);
}
conflicts.push({
left: jsonDataLocal,
right: jsonData,
changes: result.changes,
conflicts: result.conflicts
});
continue;
Zotero.debug(`Applying remote changes to ${objectType} `
+ obj.libraryKey);
Zotero.debug(result.changes);
Zotero.DataObjectUtilities.applyChanges(
jsonDataLocal, result.changes
);
// Transfer properties that aren't in the changeset
['version', 'dateAdded', 'dateModified'].forEach(x => {
if (jsonDataLocal[x] !== jsonData[x]) {
Zotero.debug(`Applying remote '${x}' value`);
}
jsonDataLocal[x] = jsonData[x];
})
jsonData = jsonDataLocal;
}
let saved = yield this._saveObjectFromJSON(obj, jsonData, options);
if (saved) numSaved++;
}
// Object doesn't exist locally
else {
Zotero.debug(ObjectType + " doesn't exist locally");
isNewObject = true;
// Check if object has been deleted locally
@ -376,6 +478,8 @@ Zotero.Sync.Data.Local = {
objectType, libraryID, objectKey
);
if (dateDeleted) {
Zotero.debug(ObjectType + " was deleted locally");
switch (objectType) {
case 'item':
conflicts.push({
@ -410,24 +514,30 @@ Zotero.Sync.Data.Local = {
obj.key = objectKey;
yield obj.loadPrimaryData();
let saved = yield this._saveObjectFromJSON(obj, jsonData, options, {
// Don't cache new items immediately, which skips reloading after save
skipCache: true
});
if (saved) numSaved++;
// Don't cache new items immediately, which skips reloading after save
skipCache = true;
}
let saved = yield this._saveObjectFromJSON(
obj, jsonData, options, { skipCache }
);
// Mark updated attachments for download
if (saved && objectType == 'item' && obj.isImportedAttachment()) {
yield this._checkAttachmentForDownload(
obj, jsonData.mtime, isNewObject
);
}
if (saved) {
numSaved++;
}
}
}.bind(this));
}.bind(this)
);
// Keep retrying if we skipped any, as long as we're still making progress
if (numSkipped && numSaved != 0) {
Zotero.debug("More " + objectTypePlural + " in cache -- continuing");
yield this.processSyncCacheForObjectType(libraryID, objectType, options);
}
if (conflicts.length) {
// Sort conflicts by local Date Modified/Deleted
conflicts.sort(function (a, b) {
var d1 = a.left.dateDeleted || a.left.dateModified;
var d2 = b.left.dateDeleted || b.left.dateModified;
@ -442,6 +552,7 @@ Zotero.Sync.Data.Local = {
var mergeData = this.resolveConflicts(conflicts);
if (mergeData) {
Zotero.debug("Processing resolved conflicts");
let mergeOptions = {};
Object.assign(mergeOptions, options);
// Tell _saveObjectFromJSON not to save with 'synced' set to true
@ -484,11 +595,55 @@ Zotero.Sync.Data.Local = {
}
}
// Keep retrying if we skipped any, as long as we're still making progress
if (numSkipped && numSaved != 0) {
Zotero.debug("More " + objectTypePlural + " in cache -- continuing");
return this.processSyncCacheForObjectType(libraryID, objectType, options);
}
data = yield this._getUnwrittenData(libraryID, objectType);
Zotero.debug("Skipping " + data.length + " "
+ (data.length == 1 ? objectType : objectTypePlural)
+ " in sync cache");
return data;
if (data.length) {
Zotero.debug(`Skipping ${data.length} `
+ (data.length == 1 ? objectType : objectTypePlural)
+ " in sync cache");
}
}),
_checkAttachmentForDownload: Zotero.Promise.coroutine(function* (item, mtime, isNewObject) {
var markToDownload = false;
if (!isNewObject) {
// Convert previously used Unix timestamps to ms-based timestamps
if (mtime < 10000000000) {
Zotero.debug("Converting Unix timestamp '" + mtime + "' to ms");
mtime = mtime * 1000;
}
var fmtime = null;
try {
fmtime = yield item.attachmentModificationTime;
}
catch (e) {
// This will probably fail later too, but ignore it for now
Zotero.logError(e);
}
if (fmtime) {
let state = Zotero.Sync.Storage.Local.checkFileModTime(item, fmtime, mtime);
if (state !== false) {
markToDownload = true;
}
}
else {
markToDownload = true;
}
}
else {
markToDownload = true;
}
if (markToDownload) {
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
}
}),
@ -501,6 +656,8 @@ Zotero.Sync.Data.Local = {
resolveConflicts: function (conflicts) {
Zotero.debug("Showing conflict resolution window");
var io = {
dataIn: {
captions: [
@ -511,9 +668,7 @@ Zotero.Sync.Data.Local = {
conflicts
}
};
var url = 'chrome://zotero/content/merge.xul';
var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
var lastWin = wm.getMostRecentWindow("navigator:browser");
@ -553,7 +708,8 @@ Zotero.Sync.Data.Local = {
_saveObjectFromJSON: Zotero.Promise.coroutine(function* (obj, json, options) {
try {
yield obj.fromJSON(json);
yield obj.loadAllData();
obj.fromJSON(json);
if (!options.saveAsChanged) {
obj.version = json.version;
obj.synced = true;
@ -611,6 +767,11 @@ Zotero.Sync.Data.Local = {
var changeset1 = Zotero.DataObjectUtilities.diff(originalJSON, currentJSON, ignoreFields);
var changeset2 = Zotero.DataObjectUtilities.diff(originalJSON, newJSON, ignoreFields);
Zotero.debug("CHANGESET1");
Zotero.debug(changeset1);
Zotero.debug("CHANGESET2");
Zotero.debug(changeset2);
var conflicts = [];
for (let i = 0; i < changeset1.length; i++) {
@ -725,27 +886,43 @@ Zotero.Sync.Data.Local = {
var conflicts = [];
for (let i = 0; i < changeset.length; i++) {
let c = changeset[i];
let c2 = changeset[i];
// Member changes are additive only, so ignore removals
if (c.op.endsWith('-remove')) {
if (c2.op.endsWith('-remove')) {
continue;
}
// Record member changes
if (c.op.startsWith('member-') || c.op.startsWith('property-member-')) {
changes.push(c);
if (c2.op.startsWith('member-') || c2.op.startsWith('property-member-')) {
changes.push(c2);
continue;
}
// Automatically apply remote changes for non-items, even if in conflict
if (objectType != 'item') {
changes.push(c);
changes.push(c2);
continue;
}
// Field changes are conflicts
conflicts.push(c);
//
// Since we don't know what changed, use only 'add' and 'delete'
if (c2.op == 'modify') {
c2.op = 'add';
}
let val = currentJSON[c2.field];
let c1 = {
field: c2.field,
op: val !== undefined ? 'add' : 'delete'
};
if (val !== undefined) {
c1.value = val;
}
if (c2.op == 'modify') {
c2.op = 'add';
}
conflicts.push([c1, c2]);
}
return { changes, conflicts };

351
chrome/content/zotero/xpcom/sync/syncRunner.js
View File

@ -29,34 +29,62 @@ if (!Zotero.Sync) {
Zotero.Sync = {};
}
Zotero.Sync.Runner_Module = function () {
// Initialized as Zotero.Sync.Runner in zotero.js
Zotero.Sync.Runner_Module = function (options = {}) {
const stopOnError = true;
Zotero.defineProperty(this, 'background', { get: () => _background });
Zotero.defineProperty(this, 'lastSyncStatus', { get: () => _lastSyncStatus });
const stopOnError = true;
this.baseURL = options.baseURL || ZOTERO_CONFIG.API_URL;
this.apiVersion = options.apiVersion || ZOTERO_CONFIG.API_VERSION;
this.apiKey = options.apiKey || Zotero.Sync.Data.Local.getAPIKey();
Components.utils.import("resource://zotero/concurrentCaller.js");
this.caller = new ConcurrentCaller(4);
this.caller.setLogger(msg => Zotero.debug(msg));
this.caller.stopOnError = stopOnError;
this.caller.onError = function (e) {
this.addError(e);
if (e.fatal) {
this.caller.stop();
throw e;
}
}.bind(this);
var _autoSyncTimer;
var _background;
var _firstInSession = true;
var _syncInProgress = false;
var _syncEngines = [];
var _storageEngines = [];
var _lastSyncStatus;
var _currentSyncStatusLabel;
var _currentLastSyncLabel;
var _errors = [];
this.getAPIClient = function () {
return new Zotero.Sync.APIClient({
baseURL: this.baseURL,
apiVersion: this.apiVersion,
apiKey: this.apiKey,
caller: this.caller
});
}
/**
* Begin a sync session
*
* @param {Object} [options]
* @param {String} [apiKey]
* @param {Boolean} [background=false] - Whether this is a background request, which prevents
* some alerts from being shown
* @param {String} [baseURL]
* @param {Integer[]} [libraries] - IDs of libraries to sync
* @param {Function} [onError] - Function to pass errors to instead of handling internally
* (used for testing)
* @param {Object} [options]
* @param {Boolean} [options.background=false] Whether this is a background request, which
* prevents some alerts from being shown
* @param {Integer[]} [options.libraries] IDs of libraries to sync
* @param {Function} [options.onError] Function to pass errors to instead of
* handling internally (used for testing)
*/
this.sync = Zotero.Promise.coroutine(function* (options = {}) {
// Clear message list
@ -84,14 +112,13 @@ Zotero.Sync.Runner_Module = function () {
// Purge deleted objects so they don't cause sync errors (e.g., long tags)
yield Zotero.purgeDataObjects(true);
options.apiKey = options.apiKey || Zotero.Prefs.get('devAPIKey');
if (!options.apiKey) {
let msg = "API key not provided";
if (!this.apiKey) {
let msg = "API key not set";
let e = new Zotero.Error(msg, 0, { dialogButtonText: null })
this.updateIcons(e);
_syncInProgress = false;
return false;
}
options.baseURL = options.baseURL || ZOTERO_CONFIG.API_URL;
if (_firstInSession) {
options.firstInSession = true;
_firstInSession = false;
@ -102,66 +129,45 @@ Zotero.Sync.Runner_Module = function () {
this.updateIcons('animate');
try {
Components.utils.import("resource://zotero/concurrent-caller.js");
var caller = new ConcurrentCaller(4); // TEMP: one for now
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = stopOnError;
caller.onError = function (e) {
this.addError(e);
if (e.fatal) {
caller.stop();
throw e;
}
}.bind(this);
let client = this.getAPIClient();
// TODO: Use a single client for all operations?
var client = new Zotero.Sync.APIClient({
baseURL: options.baseURL,
apiVersion: ZOTERO_CONFIG.API_VERSION,
apiKey: options.apiKey,
concurrentCaller: caller,
background: options.background
});
var keyInfo = yield this.checkAccess(client, options);
let keyInfo = yield this.checkAccess(client, options);
if (!keyInfo) {
this.stop();
this.end();
Zotero.debug("Syncing cancelled");
return false;
}
var libraries = yield this.checkLibraries(client, options, keyInfo, libraries);
for (let libraryID of libraries) {
try {
let engine = new Zotero.Sync.Data.Engine({
libraryID: libraryID,
apiClient: client,
setStatus: this.setSyncStatus.bind(this),
stopOnError: stopOnError,
onError: this.addError.bind(this)
});
yield engine.start();
}
catch (e) {
Zotero.debug("Sync failed for library " + libraryID);
Zotero.debug(e, 1);
Components.utils.reportError(e);
this.checkError(e);
let engineOptions = {
apiClient: client,
caller: this.caller,
setStatus: this.setSyncStatus.bind(this),
stopOnError,
onError: function (e) {
if (options.onError) {
options.onError(e);
}
else {
this.addError(e);
this.addError.bind(this);
}
if (stopOnError || e.fatal) {
caller.stop();
break;
}
}
}
}.bind(this),
background: _background,
firstInSession: _firstInSession
};
yield Zotero.Sync.Data.Local.updateLastSyncTime();
let nextLibraries = yield this.checkLibraries(
client, options, keyInfo, options.libraries
);
// Sync data, files, and then any data that needs to be uploaded
let attempt = 1;
while (nextLibraries.length) {
if (attempt > 3) {
throw new Error("Too many sync attempts -- stopping");
}
nextLibraries = yield _doDataSync(nextLibraries, engineOptions);
nextLibraries = yield _doFileSync(nextLibraries, engineOptions);
attempt++;
}
}
catch (e) {
if (options.onError) {
@ -171,62 +177,19 @@ Zotero.Sync.Runner_Module = function () {
this.addError(e);
}
}
this.stop();
finally {
this.end();
}
Zotero.debug("Done syncing");
/*if (results.changesMade) {
Zotero.debug("Changes made during file sync "
+ "-- performing additional data sync");
this.sync(options);
}*/
return;
var storageSync = function () {
Zotero.Sync.Runner.setSyncStatus(Zotero.getString('sync.status.syncingFiles'));
Zotero.Sync.Storage.sync(options)
.then(function (results) {
Zotero.debug("File sync is finished");
if (results.errors.length) {
Zotero.debug(results.errors, 1);
for each(var e in results.errors) {
Components.utils.reportError(e);
}
Zotero.Sync.Runner.setErrors(results.errors);
return;
}
if (results.changesMade) {
Zotero.debug("Changes made during file sync "
+ "-- performing additional data sync");
Zotero.Sync.Server.sync(finalCallbacks);
}
else {
Zotero.Sync.Runner.stop();
}
})
.catch(function (e) {
Zotero.debug("File sync failed", 1);
Zotero.Sync.Runner.error(e);
})
.done();
};
Zotero.Sync.Server.sync({
// Sync 1 success
onSuccess: storageSync,
// Sync 1 skip
onSkip: storageSync,
// Sync 1 stop
onStop: function () {
Zotero.Sync.Runner.stop();
},
// Sync 1 error
onError: function (e) {
Zotero.Sync.Runner.error(e);
}
});
});
@ -242,8 +205,9 @@ Zotero.Sync.Runner_Module = function () {
}
// Sanity check
if (!json.userID) throw new Error("userID not found in response");
if (!json.username) throw new Error("username not found in response");
if (!json.userID) throw new Error("userID not found in key response");
if (!json.username) throw new Error("username not found in key response");
if (!json.access) throw new Error("'access' not found in key response");
// Make sure user hasn't changed, and prompt to update database if so
if (!(yield this.checkUser(json.userID, json.username))) {
@ -446,8 +410,6 @@ Zotero.Sync.Runner_Module = function () {
*
* @param {Integer} userID New userID
* @param {Integer} libraryID New libraryID
* @param {Integer} noServerData The server account is empty — this is
* the account after a server clear
* @return {Boolean} - True to continue, false to cancel
*/
this.checkUser = Zotero.Promise.coroutine(function* (userID, username) {
@ -544,7 +506,154 @@ Zotero.Sync.Runner_Module = function () {
});
var _doDataSync = Zotero.Promise.coroutine(function* (libraries, options, skipUpdateLastSyncTime) {
var successfulLibraries = [];
for (let libraryID of libraries) {
try {
let opts = {};
Object.assign(opts, options);
opts.libraryID = libraryID;
let engine = new Zotero.Sync.Data.Engine(opts);
yield engine.start();
successfulLibraries.push(libraryID);
}
catch (e) {
Zotero.debug("Sync failed for library " + libraryID);
Zotero.logError(e);
this.checkError(e);
if (options.onError) {
options.onError(e);
}
else {
this.addError(e);
}
if (stopOnError || e.fatal) {
Zotero.debug("Stopping on error", 1);
options.caller.stop();
break;
}
}
}
// Update last-sync time if any libraries synced
// TEMP: Do we want to show updated time if some libraries haven't synced?
if (!libraries.length || successfulLibraries.length) {
yield Zotero.Sync.Data.Local.updateLastSyncTime();
}
return successfulLibraries;
}.bind(this));
var _doFileSync = Zotero.Promise.coroutine(function* (libraries, options) {
Zotero.debug("Starting file syncing");
this.setSyncStatus(Zotero.getString('sync.status.syncingFiles'));
let librariesToSync = [];
for (let libraryID of libraries) {
try {
let opts = {};
Object.assign(opts, options);
opts.libraryID = libraryID;
let tries = 3;
while (true) {
if (tries == 0) {
throw new Error("Too many file sync attempts for library " + libraryID);
}
tries--;
let engine = new Zotero.Sync.Storage.Engine(opts);
let results = yield engine.start();
if (results.syncRequired) {
librariesToSync.push(libraryID);
}
else if (results.fileSyncRequired) {
Zotero.debug("Another file sync required -- restarting");
continue;
}
break;
}
}
catch (e) {
Zotero.debug("File sync failed for library " + libraryID);
Zotero.debug(e, 1);
Components.utils.reportError(e);
this.checkError(e);
if (options.onError) {
options.onError(e);
}
else {
this.addError(e);
}
if (stopOnError || e.fatal) {
options.caller.stop();
break;
}
}
}
Zotero.debug("Done with file syncing");
return librariesToSync;
}.bind(this));
/**
* Download a single file on demand (not within a sync process)
*/
this.downloadFile = Zotero.Promise.coroutine(function* (item, requestCallbacks) {
if (Zotero.HTTP.browserIsOffline()) {
Zotero.debug("Browser is offline", 2);
return false;
}
// TEMP
var options = {};
var itemID = item.id;
var modeClass = Zotero.Sync.Storage.Local.getClassForLibrary(item.libraryID);
var controller = new modeClass({
apiClient: this.getAPIClient()
});
// TODO: verify WebDAV on-demand?
if (!controller.verified) {
Zotero.debug("File syncing is not active for item's library -- skipping download");
return false;
}
if (!item.isImportedAttachment()) {
throw new Error("Not an imported attachment");
}
if (yield item.getFilePathAsync()) {
Zotero.debug("File already exists -- replacing");
}
// TODO: start sync icon?
// TODO: create queue for cancelling
if (!requestCallbacks) {
requestCallbacks = {};
}
var onStart = function (request) {
return controller.downloadFile(request);
};
var request = new Zotero.Sync.Storage.Request({
type: 'download',
libraryID: item.libraryID,
name: item.libraryKey,
onStart: requestCallbacks.onStart
? [onStart, requestCallbacks.onStart]
: [onStart]
});
return request.start();
});
this.stop = function () {
_syncEngines.forEach(engine => engine.stop());
_storageEngines.forEach(engine => engine.stop());
}
this.end = function () {
this.updateIcons(_errors);
_errors = [];
_syncInProgress = false;
@ -669,7 +778,6 @@ Zotero.Sync.Runner_Module = function () {
if (libraryID) {
e.libraryID = libraryID;
}
Zotero.logError(e);
_errors.push(this.parseError(e));
}
@ -1027,7 +1135,8 @@ Zotero.Sync.Runner_Module = function () {
var lastSyncTime = Zotero.Sync.Data.Local.getLastSyncTime();
if (!lastSyncTime) {
try {
lastSyncTime = Zotero.Sync.Data.Local.getLastClassicSyncTime()
lastSyncTime = Zotero.Sync.Data.Local.getLastClassicSyncTime();
Zotero.debug(lastSyncTime);
}
catch (e) {
Zotero.debug(e, 2);
@ -1052,5 +1161,3 @@ Zotero.Sync.Runner_Module = function () {
_currentLastSyncLabel.hidden = false;
}
}
Zotero.Sync.Runner = new Zotero.Sync.Runner_Module;

4
chrome/content/zotero/xpcom/zotero.js
View File

@ -607,6 +607,7 @@ Components.utils.import("resource://gre/modules/osfile.jsm");
yield Zotero.Sync.Data.Local.init();
yield Zotero.Sync.Data.Utilities.init();
Zotero.Sync.EventListeners.init();
Zotero.Sync.Runner = new Zotero.Sync.Runner_Module;
Zotero.MIMETypeHandler.init();
yield Zotero.Proxies.init();
@ -2706,6 +2707,9 @@ Zotero.Browser = new function() {
if(!win) {
var win = Services.ww.activeWindow;
}
if (!win) {
throw new Error("Parent window not available for hidden browser");
}
}
// Create a hidden browser

54
chrome/content/zotero/zoteroPane.js
View File

@ -1325,6 +1325,7 @@ var ZoteroPane = new function()
else if (item.isAttachment()) {
var attachmentBox = document.getElementById('zotero-attachment-box');
attachmentBox.mode = this.collectionsView.editable ? 'edit' : 'view';
yield item.loadNote();
attachmentBox.item = item;
document.getElementById('zotero-item-pane-content').selectedIndex = 3;
@ -3692,38 +3693,41 @@ var ZoteroPane = new function()
}
}
else {
if (!item.isImportedAttachment() || !Zotero.Sync.Storage.downloadAsNeeded(item.libraryID)) {
if (!item.isImportedAttachment()
|| !Zotero.Sync.Storage.Local.downloadAsNeeded(item.libraryID)) {
this.showAttachmentNotFoundDialog(itemID, noLocateOnMissing);
return;
}
let downloadedItem = item;
yield Zotero.Sync.Storage.downloadFile(
downloadedItem,
{
onProgress: function (progress, progressMax) {}
}
)
.then(function () {
if (!downloadedItem.getFile()) {
ZoteroPane_Local.showAttachmentNotFoundDialog(downloadedItem.id, noLocateOnMissing);
return;
}
// check if unchanged?
// maybe not necessary, since we'll get an error if there's an error
Zotero.Notifier.trigger('redraw', 'item', []);
Zotero.debug('downloaded');
Zotero.debug(downloadedItem.id);
return ZoteroPane_Local.viewAttachment(downloadedItem.id, event, false, forceExternalViewer);
})
.catch(function (e) {
try {
yield Zotero.Sync.Runner.downloadFile(
downloadedItem,
{
onProgress: function (progress, progressMax) {}
}
);
}
catch (e) {
// TODO: show error somewhere else
Zotero.debug(e, 1);
ZoteroPane_Local.syncAlert(e);
});
return;
}
if (!(yield downloadedItem.getFilePathAsync())) {
ZoteroPane_Local.showAttachmentNotFoundDialog(downloadedItem.id, noLocateOnMissing);
return;
}
// check if unchanged?
// maybe not necessary, since we'll get an error if there's an error
Zotero.Notifier.trigger('redraw', 'item', []);
Zotero.debug('downloaded');
Zotero.debug(downloadedItem.id);
return ZoteroPane_Local.viewAttachment(downloadedItem.id, event, false, forceExternalViewer);
}
}
});
@ -3962,7 +3966,7 @@ var ZoteroPane = new function()
this.syncAlert = function (e) {
e = Zotero.Sync.Runner.parseSyncError(e);
e = Zotero.Sync.Runner.parseError(e);
var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"]
.getService(Components.interfaces.nsIPromptService);

5
chrome/content/zotero/zoteroPane.xul
View File

@ -195,9 +195,10 @@
</hbox>
<hbox align="center" pack="end">
<hbox id="zotero-tb-sync-progress-box" hidden="true" align="center">
<!-- TODO: localize -->
<toolbarbutton id="zotero-tb-sync-storage-cancel"
tooltiptext="Cancel Storage Sync"
oncommand="Zotero.Sync.Storage.QueueManager.cancel()"/>
tooltiptext="Stop sync"
oncommand="Zotero.Sync.Runner.stop()"/>
<progressmeter id="zotero-tb-sync-progress" mode="determined"
value="0" tooltip="zotero-tb-sync-progress-tooltip">
</progressmeter>

12
chrome/locale/en-US/zotero/zotero.properties
View File

@ -961,12 +961,12 @@ rtfScan.saveTitle = Select a location in which to save the formatted file
rtfScan.scannedFileSuffix = (Scanned)
file.accessError.theFile = The file '%S'
file.accessError.aFile = A file
file.accessError.cannotBe = cannot be
file.accessError.created = created
file.accessError.updated = updated
file.accessError.deleted = deleted
file.accessError.theFileCannotBeCreated = The file '%S' cannot be created.
file.accessError.theFileCannotBeUpdated = The file '%S' cannot be updated.
file.accessError.theFileCannotBeDeleted = The file '%S' cannot be deleted.
file.accessError.aFileCannotBeCreated = A file cannot be created.
file.accessError.aFileCannotBeUpdated = A file cannot be updated.
file.accessError.aFileCannotBeDeleted = A file cannot be deleted.
file.accessError.message.windows = Check that the file is not currently in use, that its permissions allow write access, and that it has a valid filename.
file.accessError.message.other = Check that the file is not currently in use and that its permissions allow write access.
file.accessError.restart = Restarting your computer or disabling security software may also help.

9
components/zotero-service.js
View File

@ -107,11 +107,12 @@ const xpcomFilesLocal = [
'sync/syncRunner',
'sync/syncUtilities',
'storage',
'storage/storageEngine',
'storage/storageLocal',
'storage/storageRequest',
'storage/storageResult',
'storage/storageUtilities',
'storage/streamListener',
'storage/queueManager',
'storage/queue',
'storage/request',
'storage/mode',
'storage/zfs',
'storage/webdav',
'syncedSettings',

5356
test/resource/httpd.js Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
test/tests/data/snapshot/img.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 42 B

8
test/tests/data/test.html Normal file
View File

@ -0,0 +1,8 @@
<html>
<head>
<meta charset="utf-8"/>
</head>
<body>
<p>This is a test.</p>
</body>
</html>

1
test/tests/data/test.txt Normal file
View File

@ -0,0 +1 @@
This is a test file.

10
test/tests/itemTest.js
View File

@ -537,6 +537,10 @@ describe("Zotero.Item", function () {
file.append(filename);
assert.equal(item.getFilePath(), file.path);
});
it.skip("should get and set a filename for a base-dir-relative file", function* () {
})
})
describe("#attachmentPath", function () {
@ -608,11 +612,13 @@ describe("Zotero.Item", function () {
assert.equal(OS.Path.basename(path), newName)
yield OS.File.exists(path);
// File should be flagged for upload
// DEBUG: Is this necessary?
assert.equal(
(yield Zotero.Sync.Storage.getSyncState(item.id)),
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
);
assert.isNull(yield Zotero.Sync.Storage.getSyncedHash(item.id));
assert.isNull(yield Zotero.Sync.Storage.Local.getSyncedHash(item.id));
})
})

822
test/tests/storageEngineTest.js Normal file
View File

@ -0,0 +1,822 @@
"use strict";
describe("Zotero.Sync.Storage.Engine", function () {
Components.utils.import("resource://zotero-unit/httpd.js");
var win;
var apiKey = Zotero.Utilities.randomString(24);
var port = 16213;
var baseURL = `http://localhost:${port}/`;
var server;
var responses = {};
var setup = Zotero.Promise.coroutine(function* (options = {}) {
server = sinon.fakeServer.create();
server.autoRespond = true;
Components.utils.import("resource://zotero/concurrentCaller.js");
var caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true;
Components.utils.import("resource://zotero/config.js");
var client = new Zotero.Sync.APIClient({
baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
apiKey,
caller,
background: options.background || true
});
var engine = new Zotero.Sync.Storage.Engine({
apiClient: client,
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
stopOnError: true
});
return { engine, client, caller };
});
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
function parseQueryString(str) {
var queryStringParams = str.split('&');
var params = {};
for (let param of queryStringParams) {
let [ key, val ] = param.split('=');
params[key] = decodeURIComponent(val);
}
return params;
}
function assertAPIKey(request) {
assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
}
//
// Tests
//
before(function* () {
})
beforeEach(function* () {
Zotero.debug("BEFORE HERE");
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
Zotero.debug("DONE RESET");
win = yield loadZoteroPane();
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
this.httpd = new HttpServer();
this.httpd.start(port);
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser");
// Set download-on-sync by default
Zotero.Sync.Storage.Local.downloadOnSync(
Zotero.Libraries.userLibraryID, true
);
Zotero.debug("DONE BEFORE");
})
afterEach(function* () {
var defer = new Zotero.Promise.defer();
this.httpd.stop(() => defer.resolve());
yield defer.promise;
win.close();
})
after(function* () {
this.timeout(60000);
//yield resetDB();
win.close();
})
describe("ZFS", function () {
describe("Syncing", function () {
it("should skip downloads if no last storage sync time", function* () {
var { engine, client, caller } = yield setup();
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 404
});
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.isFalse(Zotero.Libraries.userLibrary.lastStorageSync);
})
it("should skip downloads if unchanged last storage sync time", function* () {
var { engine, client, caller } = yield setup();
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
var library = Zotero.Libraries.userLibrary;
library.lastStorageSync = newStorageSyncTime;
yield library.saveTx();
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.equal(library.lastStorageSync, newStorageSyncTime);
})
it("should ignore a remotely missing file", function* () {
var { engine, client, caller } = yield setup();
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 404, null);
}
}
);
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
})
it("should handle a remotely failing file", function* () {
var { engine, client, caller } = yield setup();
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 500, null);
}
}
);
// TODO: In stopOnError mode, this the promise is rejected.
// This should probably test with stopOnError mode turned off instead.
var e = yield getPromiseError(engine.start());
assert.equal(e.message, Zotero.Sync.Storage.defaultError);
})
it("should download a missing file", function* () {
var { engine, client, caller } = yield setup();
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
// TODO: Test binary data
var text = Zotero.Utilities.randomString();
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var mtime = "1441252524905";
var md5 = Zotero.Utilities.Internal.md5(text)
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
var s3Path = `pretend-s3/${item.key}`;
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
if (!request.hasHeader('Zotero-API-Key')) {
response.setStatusLine(null, 403, "Forbidden");
return;
}
var key = request.getHeader('Zotero-API-Key');
if (key != apiKey) {
response.setStatusLine(null, 403, "Invalid key");
return;
}
response.setStatusLine(null, 302, "Found");
response.setHeader("Zotero-File-Modification-Time", mtime, false);
response.setHeader("Zotero-File-MD5", md5, false);
response.setHeader("Zotero-File-Compressed", "No", false);
response.setHeader("Location", baseURL + s3Path, false);
}
}
);
this.httpd.registerPathHandler(
"/" + s3Path,
{
handle: function (request, response) {
response.setStatusLine(null, 200, "OK");
response.write(text);
}
}
);
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync());
assert.equal(contents, text);
})
it("should upload new files", function* () {
var { engine, client, caller } = yield setup();
// Single file
var file1 = getTestDataDirectory();
file1.append('test.png');
var item1 = yield Zotero.Attachments.importFromFile({ file: file1 });
var mtime1 = yield item1.attachmentModificationTime;
var hash1 = yield item1.attachmentHash;
var path1 = item1.getFilePath();
var filename1 = 'test.png';
var size1 = (yield OS.File.stat(path1)).size;
var contentType1 = 'image/png';
var prefix1 = Zotero.Utilities.randomString();
var suffix1 = Zotero.Utilities.randomString();
var uploadKey1 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
// HTML file with auxiliary image
var file2 = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
var parentItem = yield createDataObject('item');
var item2 = yield Zotero.Attachments.importSnapshotFromFile({
file: file2,
url: 'http://example.com/',
parentItemID: parentItem.id,
title: 'Test',
contentType: 'text/html',
charset: 'utf-8'
});
var mtime2 = yield item2.attachmentModificationTime;
var hash2 = yield item2.attachmentHash;
var path2 = item2.getFilePath();
var filename2 = 'index.html';
var size2 = (yield OS.File.stat(path2)).size;
var contentType2 = 'text/html';
var charset2 = 'utf-8';
var prefix2 = Zotero.Utilities.randomString();
var suffix2 = Zotero.Utilities.randomString();
var uploadKey2 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
var deferreds = [];
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 404
});
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.md5, hash1);
assert.equal(params.mtime, mtime1);
assert.equal(params.filename, filename1);
assert.equal(params.filesize, size1);
assert.equal(params.contentType, contentType1);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/1",
contentType: contentType1,
prefix: prefix1,
suffix: suffix1,
uploadKey: uploadKey1
})
);
}
// Get upload authorization for multi-file zip
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
// Verify ZIP hash
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
item2.key + '.zip'
);
deferreds.push({
promise: Zotero.Utilities.Internal.md5Async(tmpZipPath)
.then(function (md5) {
assert.equal(params.zipMD5, md5);
})
});
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
Zotero.debug(params);
assert.equal(params.md5, hash2);
assert.notEqual(params.zipMD5, hash2);
assert.equal(params.mtime, mtime2);
assert.equal(params.filename, filename2);
assert.equal(params.zipFilename, item2.key + ".zip");
assert.isTrue(parseInt(params.filesize) == params.filesize);
assert.equal(params.contentType, contentType2);
assert.equal(params.charset, charset2);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/2",
contentType: 'application/zip',
prefix: prefix2,
suffix: suffix2,
uploadKey: uploadKey2
})
);
}
// Upload single file to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
assert.equal(req.requestHeaders["Content-Type"], contentType1 + fixSinonBug);
assert.equal(req.requestBody.size, (new Blob([prefix1, File(file1), suffix1]).size));
req.respond(201, {}, "");
}
// Upload multi-file ZIP to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/2") {
assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
// Verify uploaded ZIP file
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
Zotero.Utilities.randomString() + '.zip'
);
let deferred = Zotero.Promise.defer();
deferreds.push(deferred);
var reader = new FileReader();
reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () {
try {
let file = yield OS.File.open(tmpZipPath, {
create: true
});
var contents = new Uint8Array(reader.result);
contents = contents.slice(prefix2.length, suffix2.length * -1);
yield file.write(contents);
yield file.close();
var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
.createInstance(Components.interfaces.nsIZipReader);
zr.open(Zotero.File.pathToFile(tmpZipPath));
zr.test(null);
var entries = zr.findEntries('*');
var entryNames = [];
while (entries.hasMore()) {
entryNames.push(entries.getNext());
}
assert.equal(entryNames.length, 2);
assert.sameMembers(entryNames, ['index.html', 'img.gif']);
assert.equal(zr.getEntry('index.html').realSize, size2);
assert.equal(zr.getEntry('img.gif').realSize, 42);
deferred.resolve();
}
catch (e) {
deferred.reject(e);
}
}));
reader.readAsArrayBuffer(req.requestBody);
req.respond(201, {}, "");
}
// Register single-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey1);
req.respond(
204,
{
"Last-Modified-Version": 10
},
""
);
}
// Register multi-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey2);
req.respond(
204,
{
"Last-Modified-Version": 15
},
""
);
}
})
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "POST",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
// TODO: One-step uploads
/*// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file`) {
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/json" + fixSinonBug
);
let params = JSON.parse(req.requestBody);
assert.equal(params.md5, hash);
assert.equal(params.mtime, mtime);
assert.equal(params.filename, filename);
assert.equal(params.size, size);
assert.equal(params.contentType, contentType);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3",
headers: {
"Content-Type": contentType,
"Content-MD5": hash,
//"Content-Length": params.size, process but don't return
//"x-amz-meta-"
},
uploadKey
})
);
}
else if (req.method == "PUT" && req.url == baseURL + "pretend-s3") {
assert.equal(req.requestHeaders["Content-Type"], contentType + fixSinonBug);
assert.instanceOf(req.requestBody, File);
req.respond(201, {}, "");
}
})*/
var result = yield engine.start();
yield Zotero.Promise.all(deferreds.map(d => d.promise));
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item1.id)), mtime1);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item1.id)), hash1);
assert.equal(item1.version, 10);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item2.id)), mtime2);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item2.id)), hash2);
assert.equal(item2.version, 15);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
})
it("should update local info for file that already exists on the server", function* () {
var { engine, client, caller } = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file: file });
item.version = 5;
yield item.saveTx();
var json = yield item.toJSON();
yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var newVersion = 10;
setResponse({
method: "POST",
url: "users/1/laststoragesync",
status: 200,
text: "" + (Math.round(new Date().getTime() / 1000) - 50000)
});
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
req.respond(
200,
{
"Content-Type": "application/json",
"Last-Modified-Version": newVersion
},
JSON.stringify({
exists: 1,
})
);
}
})
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "POST",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
// TODO: One-step uploads
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.equal(item.version, newVersion);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
})
})
describe("#_processUploadFile()", function () {
it("should handle 412 with matching version and hash matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.ZFS_Module({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var itemJSON = yield item.toResponseJSON();
// Set saved hash to a different value, which should be overwritten
//
// We're also testing cases where a hash isn't set for a file (e.g., if the
// storage directory was transferred, the mtime doesn't match, but the file was
// never downloaded), but there's no difference in behavior
var dbHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, dbHash)
});
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-Match"] == dbHash) {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"ETag does not match current version of file"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncedHash(item.id), itemJSON.data.md5
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isFalse(result.fileSyncRequired);
})
it("should handle 412 with matching version and hash not matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.ZFS_Module({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var fileHash = yield item.attachmentHash;
var itemJSON = yield item.toResponseJSON();
itemJSON.data.md5 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"If-None-Match: * set but file exists"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.isNull(Zotero.Sync.Storage.Local.getSyncedHash(item.id));
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item.id),
Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isTrue(result.fileSyncRequired);
})
it("should handle 412 with greater version", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.ZFS_Module({
apiClient: client
})
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.version = 5;
item.synced = true;
yield item.saveTx();
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 10
},
"If-None-Match: * set but file exists"
);
}
})
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
assert.equal(item.version, 5);
assert.equal(item.synced, true);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isTrue(result.syncRequired);
})
})
})
})

329
test/tests/storageLocalTest.js Normal file
View File

@ -0,0 +1,329 @@
"use strict";
describe("Zotero.Sync.Storage.Local", function () {
var win;
before(function* () {
win = yield loadBrowserWindow();
});
beforeEach(function* () {
yield resetDB({
thisArg: this
})
})
after(function () {
if (win) {
win.close();
}
});
describe("#checkForUpdatedFiles()", function () {
it("should flag modified file for upload and return it", function* () {
// Create attachment
let item = yield importFileAttachment('test.txt')
var hash = yield item.attachmentHash;
// Set file mtime to the past (without milliseconds, which aren't used on OS X)
var mtime = (Math.floor(new Date().getTime() / 1000) * 1000) - 1000;
yield OS.File.setDates((yield item.getFilePathAsync()), null, mtime);
// Mark as synced, so it will be checked
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
});
// Update mtime and contents
var path = yield item.getFilePathAsync();
yield OS.File.setDates(path);
yield Zotero.File.putContentsAsync(path, Zotero.Utilities.randomString());
// File should be returned
var libraryID = Zotero.Libraries.userLibraryID;
var changed = yield Zotero.Sync.Storage.Local.checkForUpdatedFiles(libraryID);
yield item.eraseTx();
assert.equal(changed, true);
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
);
})
it("should skip a file if mod time hasn't changed", function* () {
// Create attachment
let item = yield importFileAttachment('test.txt')
var hash = yield item.attachmentHash;
var mtime = yield item.attachmentModificationTime;
// Mark as synced, so it will be checked
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
});
var libraryID = Zotero.Libraries.userLibraryID;
var changed = yield Zotero.Sync.Storage.Local.checkForUpdatedFiles(libraryID);
var syncState = yield Zotero.Sync.Storage.Local.getSyncState(item.id);
yield item.eraseTx();
assert.isFalse(changed);
assert.equal(syncState, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
})
it("should skip a file if mod time has changed but contents haven't", function* () {
// Create attachment
let item = yield importFileAttachment('test.txt')
var hash = yield item.attachmentHash;
// Set file mtime to the past (without milliseconds, which aren't used on OS X)
var mtime = (Math.floor(new Date().getTime() / 1000) * 1000) - 1000;
yield OS.File.setDates((yield item.getFilePathAsync()), null, mtime);
// Mark as synced, so it will be checked
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
});
// Update mtime, but not contents
var path = yield item.getFilePathAsync();
yield OS.File.setDates(path);
var libraryID = Zotero.Libraries.userLibraryID;
var changed = yield Zotero.Sync.Storage.Local.checkForUpdatedFiles(libraryID);
var syncState = yield Zotero.Sync.Storage.Local.getSyncState(item.id);
var syncedModTime = yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id);
var newModTime = yield item.attachmentModificationTime;
yield item.eraseTx();
assert.isFalse(changed);
assert.equal(syncState, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
assert.equal(syncedModTime, newModTime);
})
})
describe("#processDownload()", function () {
var file1Name = 'index.html';
var file1Contents = '<html><body>Test</body></html>';
var file2Name = 'test.txt';
var file2Contents = 'Test';
var createZIP = Zotero.Promise.coroutine(function* (zipFile) {
var tmpDir = Zotero.getTempDirectory().path;
var zipDir = OS.Path.join(tmpDir, Zotero.Utilities.randomString());
yield OS.File.makeDir(zipDir);
yield Zotero.File.putContentsAsync(OS.Path.join(zipDir, file1Name), file1Contents);
yield Zotero.File.putContentsAsync(OS.Path.join(zipDir, file2Name), file2Contents);
yield Zotero.File.zipDirectory(zipDir, zipFile);
yield OS.File.removeDir(zipDir);
});
it("should download and extract a ZIP file into the attachment directory", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var parentItem = yield createDataObject('item');
var key = Zotero.DataObjectUtilities.generateKey();
var tmpDir = Zotero.getTempDirectory().path;
var zipFile = OS.Path.join(tmpDir, key + '.tmp');
yield createZIP(zipFile);
var md5 = Zotero.Utilities.Internal.md5(Zotero.File.pathToFile(zipFile));
var mtime = 1445667239000;
var json = {
key,
version: 10,
itemType: 'attachment',
linkMode: 'imported_url',
url: 'https://example.com',
filename: file1Name,
contentType: 'text/html',
charset: 'utf-8',
md5,
mtime
};
yield Zotero.Sync.Data.Local.saveCacheObjects(
'item', libraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, 'item', { stopOnError: true }
);
var item = yield Zotero.Items.getByLibraryAndKeyAsync(libraryID, key);
yield Zotero.Sync.Storage.Local.processDownload({
item,
md5,
mtime,
compressed: true
});
yield OS.File.remove(zipFile);
yield assert.eventually.equal(
item.attachmentHash, Zotero.Utilities.Internal.md5(file1Contents)
);
yield assert.eventually.equal(item.attachmentModificationTime, mtime);
})
})
describe("#_deleteExistingAttachmentFiles()", function () {
it("should delete all files", function* () {
var item = yield importFileAttachment('test.html');
var path = OS.Path.dirname(item.getFilePath());
var files = ['a', 'b', 'c', 'd'];
for (let file of files) {
yield Zotero.File.putContentsAsync(OS.Path.join(path, file), file);
}
yield Zotero.Sync.Storage.Local._deleteExistingAttachmentFiles(item);
for (let file of files) {
assert.isFalse(
(yield OS.File.exists(OS.Path.join(path, file))),
`File '${file}' doesn't exist`
);
}
})
})
describe("#getConflicts()", function () {
it("should return an array of objects for attachments in conflict", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var item1 = yield importFileAttachment('test.png');
item1.version = 10;
yield item1.saveTx();
var item2 = yield importFileAttachment('test.txt');
var item3 = yield importFileAttachment('test.html');
item3.version = 11;
yield item3.saveTx();
var json1 = yield item1.toJSON();
var json3 = yield item3.toJSON();
// Change remote mtimes
// Round to nearest second because OS X doesn't support ms resolution
var now = Math.round(new Date().getTime() / 1000) * 1000;
json1.mtime = now - 10000;
json3.mtime = now - 20000;
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]);
yield Zotero.Sync.Storage.Local.setSyncState(
item1.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
yield Zotero.Sync.Storage.Local.setSyncState(
item3.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
var conflicts = yield Zotero.Sync.Storage.Local.getConflicts(libraryID);
assert.lengthOf(conflicts, 2);
var item1Conflict = conflicts.find(x => x.left.key == item1.key);
assert.equal(
item1Conflict.left.dateModified,
Zotero.Date.dateToISO(new Date(yield item1.attachmentModificationTime))
);
assert.equal(
item1Conflict.right.dateModified,
Zotero.Date.dateToISO(new Date(json1.mtime))
);
var item3Conflict = conflicts.find(x => x.left.key == item3.key);
assert.equal(
item3Conflict.left.dateModified,
Zotero.Date.dateToISO(new Date(yield item3.attachmentModificationTime))
);
assert.equal(
item3Conflict.right.dateModified,
Zotero.Date.dateToISO(new Date(json3.mtime))
);
})
})
describe("#resolveConflicts()", function () {
it("should show the conflict resolution window on attachment conflicts", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var item1 = yield importFileAttachment('test.png');
item1.version = 10;
yield item1.saveTx();
var item2 = yield importFileAttachment('test.txt');
var item3 = yield importFileAttachment('test.html');
item3.version = 11;
yield item3.saveTx();
var json1 = yield item1.toJSON();
var json3 = yield item3.toJSON();
// Change remote mtimes
json1.mtime = new Date().getTime() + 10000;
json3.mtime = new Date().getTime() - 10000;
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]);
yield Zotero.Sync.Storage.Local.setSyncState(
item1.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
yield Zotero.Sync.Storage.Local.setSyncState(
item3.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
var promise = waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
var doc = dialog.document;
var wizard = doc.documentElement;
var mergeGroup = wizard.getElementsByTagName('zoteromergegroup')[0];
// 1 (remote)
// Later remote version should be selected
assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
// Check checkbox text
assert.equal(
doc.getElementById('resolve-all').label,
Zotero.getString('sync.conflict.resolveAllRemote')
);
// Select local object
mergeGroup.leftpane.click();
assert.equal(mergeGroup.leftpane.getAttribute('selected'), 'true');
wizard.getButton('next').click();
// 2 (local)
// Later local version should be selected
assert.equal(mergeGroup.leftpane.getAttribute('selected'), 'true');
// Select remote object
mergeGroup.rightpane.click();
assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
if (Zotero.isMac) {
assert.isTrue(wizard.getButton('next').hidden);
assert.isFalse(wizard.getButton('finish').hidden);
}
else {
// TODO
}
wizard.getButton('finish').click();
})
yield Zotero.Sync.Storage.Local.resolveConflicts(libraryID);
yield promise;
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item1.id),
Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD
);
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item3.id),
Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD
);
})
})
})

22
test/tests/storageRequestTest.js Normal file
View File

@ -0,0 +1,22 @@
"use strict";
describe("Zotero.Sync.Storage.Request", function () {
describe("#run()", function () {
it("should run a request and wait for it to complete", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var count = 0;
var request = new Zotero.Sync.Storage.Request({
type: 'download',
libraryID,
name: "1/AAAAAAAA",
onStart: Zotero.Promise.coroutine(function* () {
yield Zotero.Promise.delay(25);
count++;
return new Zotero.Sync.Storage.Result;
})
});
var results = yield request.start();
assert.equal(count, 1);
})
})
})

20
test/tests/syncEngineTest.js
View File

@ -19,28 +19,20 @@ describe("Zotero.Sync.Data.Engine", function () {
var caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true;
caller.onError = function (e) {
Zotero.logError(e);
if (options.onError) {
options.onError(e);
}
if (e.fatal) {
caller.stop();
throw e;
}
};
Components.utils.import("resource://zotero/config.js");
var client = new Zotero.Sync.APIClient({
baseURL: baseURL,
baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
apiKey: apiKey,
concurrentCaller: caller,
apiKey,
caller,
background: options.background || true
});
var engine = new Zotero.Sync.Data.Engine({
apiClient: client,
libraryID: options.libraryID || Zotero.Libraries.userLibraryID
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
stopOnError: true
});
return { engine, client, caller };

264
test/tests/syncLocalTest.js
View File

@ -4,7 +4,7 @@ describe("Zotero.Sync.Data.Local", function() {
describe("#processSyncCacheForObjectType()", function () {
var types = Zotero.DataObjectUtilities.getTypes();
it("should update local version number if remote version is identical", function* () {
it("should update local version number and mark as synced if remote version is identical", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
for (let type of types) {
@ -24,11 +24,167 @@ describe("Zotero.Sync.Data.Local", function() {
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
);
assert.equal(
objectsClass.getByLibraryAndKey(libraryID, obj.key).version, 10
);
let localObj = objectsClass.getByLibraryAndKey(libraryID, obj.key);
assert.equal(localObj.version, 10);
assert.isTrue(localObj.synced);
}
})
it("should keep local item changes while applying non-conflicting remote changes", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var type = 'item';
let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(type);
let obj = yield createDataObject(type, { version: 5 });
let data = yield obj.toJSON();
yield Zotero.Sync.Data.Local.saveCacheObjects(
type, libraryID, [data]
);
// Change local title
yield modifyDataObject(obj)
var changedTitle = obj.getField('title');
// Save remote version to cache without title but with changed place
data.key = obj.key;
data.version = 10;
var changedPlace = data.place = 'New York';
let json = {
key: obj.key,
version: 10,
data: data
};
yield Zotero.Sync.Data.Local.saveCacheObjects(
type, libraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
);
assert.equal(obj.version, 10);
assert.equal(obj.getField('title'), changedTitle);
assert.equal(obj.getField('place'), changedPlace);
})
it("should mark new attachment items for download", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
Zotero.Sync.Storage.Local.setModeForLibrary(libraryID, 'zfs');
var key = Zotero.DataObjectUtilities.generateKey();
var version = 10;
var json = {
key,
version,
data: {
key,
version,
itemType: 'attachment',
linkMode: 'imported_file',
md5: '57f8a4fda823187b91e1191487b87fe6',
mtime: 1442261130615
}
};
yield Zotero.Sync.Data.Local.saveCacheObjects(
'item', Zotero.Libraries.userLibraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, 'item', { stopOnError: true }
);
var id = Zotero.Items.getIDFromLibraryAndKey(libraryID, key);
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(id)),
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
})
it("should mark updated attachment items for download", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
Zotero.Sync.Storage.Local.setModeForLibrary(libraryID, 'zfs');
var item = yield importFileAttachment('test.png');
item.version = 5;
item.synced = true;
yield item.saveTx();
// Set file as synced
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
item.id, (yield item.attachmentModificationTime)
);
yield Zotero.Sync.Storage.Local.setSyncedHash(
item.id, (yield item.attachmentHash)
);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
});
// Simulate download of version with updated attachment
var json = yield item.toResponseJSON();
json.version = 10;
json.data.version = 10;
json.data.md5 = '57f8a4fda823187b91e1191487b87fe6';
json.data.mtime = new Date().getTime() + 10000;
yield Zotero.Sync.Data.Local.saveCacheObjects(
'item', Zotero.Libraries.userLibraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, 'item', { stopOnError: true }
);
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
})
it("should ignore attachment metadata when resolving metadata conflict", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
Zotero.Sync.Storage.Local.setModeForLibrary(libraryID, 'zfs');
var item = yield importFileAttachment('test.png');
item.version = 5;
yield item.saveTx();
var json = yield item.toResponseJSON();
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json]);
// Set file as synced
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
item.id, (yield item.attachmentModificationTime)
);
yield Zotero.Sync.Storage.Local.setSyncedHash(
item.id, (yield item.attachmentHash)
);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
});
// Modify title locally, leaving item unsynced
var newTitle = Zotero.Utilities.randomString();
item.setField('title', newTitle);
yield item.saveTx();
// Simulate download of version with original title but updated attachment
json.version = 10;
json.data.version = 10;
json.data.md5 = '57f8a4fda823187b91e1191487b87fe6';
json.data.mtime = new Date().getTime() + 10000;
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json]);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, 'item', { stopOnError: true }
);
assert.equal(item.getField('title'), newTitle);
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
})
})
describe("Conflict Resolution", function () {
@ -232,7 +388,10 @@ describe("Zotero.Sync.Data.Local", function() {
jsonData.title = Zotero.Utilities.randomString();
yield Zotero.Sync.Data.Local.saveCacheObjects(type, libraryID, [json]);
var windowOpened = false;
waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
windowOpened = true;
var doc = dialog.document;
var wizard = doc.documentElement;
var mergeGroup = wizard.getElementsByTagName('zoteromergegroup')[0];
@ -240,12 +399,14 @@ describe("Zotero.Sync.Data.Local", function() {
// Remote version should be selected by default
assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
assert.ok(mergeGroup.leftpane.pane.onclick);
// Select local deleted version
mergeGroup.leftpane.pane.click();
wizard.getButton('finish').click();
})
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
);
assert.isTrue(windowOpened);
obj = objectsClass.getByLibraryAndKey(libraryID, key);
assert.isFalse(obj);
@ -825,15 +986,28 @@ describe("Zotero.Sync.Data.Local", function() {
assert.sameDeepMembers(
result.conflicts,
[
{
field: "place",
op: "delete"
},
{
field: "date",
op: "add",
value: "2015-05-15"
}
[
{
field: "place",
op: "add",
value: "Place"
},
{
field: "place",
op: "delete"
}
],
[
{
field: "date",
op: "delete"
},
{
field: "date",
op: "add",
value: "2015-05-15"
}
]
]
);
})
@ -1296,4 +1470,68 @@ describe("Zotero.Sync.Data.Local", function() {
})
})
})
describe("#reconcileChangesWithoutCache()", function () {
it("should return conflict for conflicting fields", function () {
var json1 = {
key: "AAAAAAAA",
version: 1234,
title: "Title 1",
pages: 10,
dateModified: "2015-05-14 14:12:34"
};
var json2 = {
key: "AAAAAAAA",
version: 1235,
title: "Title 2",
place: "New York",
dateModified: "2015-05-14 13:45:12"
};
var ignoreFields = ['dateAdded', 'dateModified'];
var result = Zotero.Sync.Data.Local._reconcileChangesWithoutCache(
'item', json1, json2, ignoreFields
);
assert.lengthOf(result.changes, 0);
assert.sameDeepMembers(
result.conflicts,
[
[
{
field: "title",
op: "add",
value: "Title 1"
},
{
field: "title",
op: "add",
value: "Title 2"
}
],
[
{
field: "pages",
op: "add",
value: 10
},
{
field: "pages",
op: "delete"
}
],
[
{
field: "place",
op: "delete"
},
{
field: "place",
op: "add",
value: "New York"
}
]
]
);
})
})
})

47
test/tests/syncRunnerTest.js
View File

@ -5,7 +5,7 @@ describe("Zotero.Sync.Runner", function () {
var apiKey = Zotero.Utilities.randomString(24);
var baseURL = "http://local.zotero/";
var userLibraryID, publicationsLibraryID, runner, caller, server, client, stub, spy;
var userLibraryID, publicationsLibraryID, runner, caller, server, stub, spy;
var responses = {
keyInfo: {
@ -129,15 +129,7 @@ describe("Zotero.Sync.Runner", function () {
}
};
var client = new Zotero.Sync.APIClient({
baseURL: baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
apiKey: apiKey,
concurrentCaller: caller,
background: options.background || true
});
return { runner, caller, client };
return { runner, caller };
})
function setResponse(response) {
@ -160,7 +152,7 @@ describe("Zotero.Sync.Runner", function () {
server = sinon.fakeServer.create();
server.autoRespond = true;
({ runner, caller, client } = yield setup());
({ runner, caller } = yield setup());
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("A");
@ -180,7 +172,7 @@ describe("Zotero.Sync.Runner", function () {
it("should check key access", function* () {
spy = sinon.spy(runner, "checkUser");
setResponse('keyInfo.fullAccess');
var json = yield runner.checkAccess(client);
var json = yield runner.checkAccess(runner.getAPIClient());
sinon.assert.calledWith(spy, 1, "Username");
var compare = {};
Object.assign(compare, responses.keyInfo.fullAccess.json);
@ -216,7 +208,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('userGroups.groupVersions');
var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json
runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 4);
assert.sameMembers(
@ -240,19 +232,25 @@ describe("Zotero.Sync.Runner", function () {
setResponse('userGroups.groupVersions');
var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json, [userLibraryID]
runner.getAPIClient(), false, responses.keyInfo.fullAccess.json, [userLibraryID]
);
assert.lengthOf(libraries, 1);
assert.sameMembers(libraries, [userLibraryID]);
var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json, [userLibraryID, publicationsLibraryID]
runner.getAPIClient(),
false,
responses.keyInfo.fullAccess.json,
[userLibraryID, publicationsLibraryID]
);
assert.lengthOf(libraries, 2);
assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID]);
var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json, [group1.libraryID]
runner.getAPIClient(),
false,
responses.keyInfo.fullAccess.json,
[group1.libraryID]
);
assert.lengthOf(libraries, 1);
assert.sameMembers(libraries, [group1.libraryID]);
@ -277,7 +275,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('groups.ownerGroup');
setResponse('groups.memberGroup');
var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json
runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 4);
assert.sameMembers(
@ -318,7 +316,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('groups.ownerGroup');
setResponse('groups.memberGroup');
var libraries = yield runner.checkLibraries(
client,
runner.getAPIClient(),
false,
responses.keyInfo.fullAccess.json,
[group1.libraryID, group2.libraryID]
@ -339,7 +337,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('groups.ownerGroup');
setResponse('groups.memberGroup');
var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json
runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 4);
var groupData1 = responses.groups.ownerGroup;
@ -370,7 +368,7 @@ describe("Zotero.Sync.Runner", function () {
assert.include(text, group1.name);
});
var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json
runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 3);
assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID, group2.libraryID]);
@ -388,7 +386,7 @@ describe("Zotero.Sync.Runner", function () {
assert.include(text, group.name);
}, "extra1");
var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json
runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 3);
assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID, group.libraryID]);
@ -405,7 +403,7 @@ describe("Zotero.Sync.Runner", function () {
assert.include(text, group.name);
}, "cancel");
var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json
runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 0);
assert.isTrue(Zotero.Groups.exists(groupData.json.id));
@ -656,6 +654,11 @@ describe("Zotero.Sync.Runner", function () {
Zotero.Libraries.getVersion(Zotero.Groups.getLibraryIDFromGroupID(2694172)),
20
);
// Last sync time should be within the last second
var lastSyncTime = Zotero.Sync.Data.Local.getLastSyncTime();
assert.isAbove(lastSyncTime, new Date().getTime() - 1000);
assert.isBelow(lastSyncTime, new Date().getTime());
})
})
})

94
test/tests/zoteroPaneTest.js
View File

@ -1,3 +1,5 @@
"use strict";
describe("ZoteroPane", function() {
var win, doc, zp;
@ -90,4 +92,96 @@ describe("ZoteroPane", function() {
);
})
})
describe("#viewAttachment", function () {
Components.utils.import("resource://zotero-unit/httpd.js");
var apiKey = Zotero.Utilities.randomString(24);
var port = 16213;
var baseURL = `http://localhost:${port}/`;
var server;
var responses = {};
var setup = Zotero.Promise.coroutine(function* (options = {}) {
server = sinon.fakeServer.create();
server.autoRespond = true;
});
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
before(function () {
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
Zotero.Sync.Runner.apiKey = apiKey;
Zotero.Sync.Runner.baseURL = baseURL;
})
beforeEach(function* () {
this.httpd = new HttpServer();
this.httpd.start(port);
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser");
})
afterEach(function* () {
var defer = new Zotero.Promise.defer();
this.httpd.stop(() => defer.resolve());
yield defer.promise;
})
it("should download an attachment on-demand", function* () {
yield setup();
Zotero.Sync.Storage.Local.downloadAsNeeded(Zotero.Libraries.userLibraryID, true);
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
// TODO: Test binary data
var text = Zotero.Utilities.randomString();
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var mtime = "1441252524000";
var md5 = Zotero.Utilities.Internal.md5(text)
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
var s3Path = `pretend-s3/${item.key}`;
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 302, "Found");
response.setHeader("Zotero-File-Modification-Time", mtime, false);
response.setHeader("Zotero-File-MD5", md5, false);
response.setHeader("Zotero-File-Compressed", "No", false);
response.setHeader("Location", baseURL + s3Path, false);
}
}
);
this.httpd.registerPathHandler(
"/" + s3Path,
{
handle: function (request, response) {
response.setStatusLine(null, 200, "OK");
response.write(text);
}
}
);
yield zp.viewAttachment(item.id);
assert.equal((yield item.attachmentHash), md5);
assert.equal((yield item.attachmentModificationTime), mtime);
var path = yield item.getFilePathAsync();
assert.equal((yield Zotero.File.getContentsAsync(path)), text);
})
})
})