suzanne.soy/zotero
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ZFS file sync overhaul for API syncing

Browse Source 
This mostly gets ZFS file syncing and file conflict resolution working
with the API sync process. WebDAV will need to be updated separately.

Known issues:

- File sync progress is temporarily gone
- File uploads can result in an unnecessary 412 loop on the next data
  sync
- This causes Firefox to crash on one of my computers during tests,
  which would be easier to debug if it produced a crash log.

Also:

- Adds httpd.js for use in tests when FakeXMLHttpRequest can't be used
  (e.g., saveURI()).
- Adds some additional test data files for attachment tests
This commit is contained in:
Dan Stillman 2015-10-29 03:41:54 -04:00
parent 6d46b06617
commit 73f4d28ab2
44 changed files with 11226 additions and 5239 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

371
chrome/content/zotero/bindings/attachmentbox.xml
View File

@ -57,6 +57,7 @@
Zotero.debug("Setting mode to '" + val + "'"); Zotero.debug("Setting mode to '" + val + "'");
this.editable = false; this.editable = false;
this.synchronous = false;
this.displayURL = false; this.displayURL = false;
this.displayFileName = false; this.displayFileName = false;
this.clickableLink = false; this.clickableLink = false;
@ -93,6 +94,7 @@
break; break;
case 'merge': case 'merge':
this.synchronous = true;
this.displayURL = true; this.displayURL = true;
this.displayFileName = true; this.displayFileName = true;
this.displayAccessed = true; this.displayAccessed = true;
@ -102,6 +104,7 @@
break; break;
case 'mergeedit': case 'mergeedit':
this.synchronous = true;
this.editable = true; this.editable = true;
this.displayURL = true; this.displayURL = true;
this.displayFileName = true; this.displayFileName = true;
@ -112,6 +115,13 @@
this.displayDateModified = true; this.displayDateModified = true;
break; break;
case 'filemerge':
this.synchronous = true;
this.displayURL = true;
this.displayFileName = true;
this.displayDateModified = true;
break;
default: default:
throw ("Invalid mode '" + val + "' in attachmentbox.xml"); throw ("Invalid mode '" + val + "' in attachmentbox.xml");
} }
@ -123,18 +133,16 @@
</property> </property>
<field name="_item"/> <field name="_item"/>
<property name="item" <property name="item" onget="return this._item;">
onget="return this._item;" <setter><![CDATA[
onset="this._item = val; this.refresh();"> if (!(val instanceof Zotero.Item)) {
throw new Error("'item' must be a Zotero.Item");
}
this._item = val;
this.refresh();
]]></setter>
</property> </property>
<!-- .ref is an alias for .item -->
<property name="ref"
onget="return this._item;"
onset="this._item = val; this.refresh();">
</property>
<!-- Methods --> <!-- Methods -->
<constructor> <constructor>
@ -167,125 +175,122 @@
<method name="refresh"> <method name="refresh">
<body><![CDATA[ <body><![CDATA[
Zotero.spawn(function* () { Zotero.debug('Refreshing attachment box');
Zotero.debug('Refreshing attachment box');
var attachmentBox = document.getAnonymousNodes(this)[0];
var title = this._id('title');
var fileNameRow = this._id('fileNameRow');
var urlField = this._id('url');
var accessed = this._id('accessedRow');
var pagesRow = this._id('pagesRow');
var dateModifiedRow = this._id('dateModifiedRow');
var indexStatusRow = this._id('indexStatusRow');
var selectButton = this._id('select-button');
// DEBUG: this is annoying -- we really want to use an abstracted
// version of createValueElement() from itemPane.js
// (ideally in an XBL binding)
// Wrap title to multiple lines if necessary
while (title.hasChildNodes()) {
title.removeChild(title.firstChild);
}
var val = this.item.getField('title');
if (typeof val != 'string') {
val += "";
}
var firstSpace = val.indexOf(" ");
// Crop long uninterrupted text
if ((firstSpace == -1 && val.length > 29 ) || firstSpace > 29) {
title.setAttribute('crop', 'end');
title.setAttribute('value', val);
}
// Create a <description> element, essentially
else {
title.removeAttribute('value');
title.appendChild(document.createTextNode(val));
}
if (this.editable) {
title.className = 'zotero-clicky';
yield Zotero.Promise.all([this.item.loadItemData(), this.item.loadNote()]) // For the time being, use a silly little popup
.tap(() => Zotero.Promise.check(this.item)); title.addEventListener('click', this.editTitle, false);
}
var isImportedURL = this.item.attachmentLinkMode ==
Zotero.Attachments.LINK_MODE_IMPORTED_URL;
// Metadata for URL's
if (this.item.attachmentLinkMode == Zotero.Attachments.LINK_MODE_LINKED_URL
|| isImportedURL) {
var attachmentBox = document.getAnonymousNodes(this)[0]; // URL
var title = this._id('title'); if (this.displayURL) {
var fileNameRow = this._id('fileNameRow'); var urlSpec = this.item.getField('url');
var urlField = this._id('url'); urlField.setAttribute('value', urlSpec);
var accessed = this._id('accessedRow'); urlField.setAttribute('hidden', false);
var pagesRow = this._id('pagesRow'); if (this.clickableLink) {
var dateModifiedRow = this._id('dateModifiedRow'); urlField.onclick = function (event) {
var indexStatusRow = this._id('indexStatusRow'); ZoteroPane_Local.loadURI(this.value, event)
var selectButton = this._id('select-button'); };
urlField.className = 'zotero-text-link';
// DEBUG: this is annoying -- we really want to use an abstracted
// version of createValueElement() from itemPane.js
// (ideally in an XBL binding)
// Wrap title to multiple lines if necessary
while (title.hasChildNodes()) {
title.removeChild(title.firstChild);
}
var val = this.item.getField('title');
if (typeof val != 'string') {
val += "";
}
var firstSpace = val.indexOf(" ");
// Crop long uninterrupted text
if ((firstSpace == -1 && val.length > 29 ) || firstSpace > 29) {
title.setAttribute('crop', 'end');
title.setAttribute('value', val);
}
// Create a <description> element, essentially
else {
title.removeAttribute('value');
title.appendChild(document.createTextNode(val));
}
if (this.editable) {
title.className = 'zotero-clicky';
// For the time being, use a silly little popup
title.addEventListener('click', this.editTitle, false);
}
var isImportedURL = this.item.attachmentLinkMode ==
Zotero.Attachments.LINK_MODE_IMPORTED_URL;
// Metadata for URL's
if (this.item.attachmentLinkMode == Zotero.Attachments.LINK_MODE_LINKED_URL
|| isImportedURL) {
// URL
if (this.displayURL) {
var urlSpec = this.item.getField('url');
urlField.setAttribute('value', urlSpec);
urlField.setAttribute('hidden', false);
if (this.clickableLink) {
urlField.onclick = function (event) {
ZoteroPane_Local.loadURI(this.value, event)
};
urlField.className = 'zotero-text-link';
}
else {
urlField.className = '';
}
urlField.hidden = false;
} }
else { else {
urlField.hidden = true; urlField.className = '';
}
// Access date
if (this.displayAccessed) {
this._id("accessed-label").value = Zotero.getString('itemFields.accessDate')
+ Zotero.getString('punctuation.colon');
this._id("accessed").value = Zotero.Date.sqlToDate(
this.item.getField('accessDate'), true
).toLocaleString();
accessed.hidden = false;
}
else {
accessed.hidden = true;
} }
urlField.hidden = false;
} }
// Metadata for files
else { else {
urlField.hidden = true; urlField.hidden = true;
accessed.hidden = true;
} }
if (this.item.attachmentLinkMode // Access date
!= Zotero.Attachments.LINK_MODE_LINKED_URL if (this.displayAccessed) {
&& this.displayFileName) { this._id("accessed-label").value = Zotero.getString('itemFields.accessDate')
var fileName = this.item.getFilename(); + Zotero.getString('punctuation.colon');
this._id("accessed").value = Zotero.Date.sqlToDate(
if (fileName) { this.item.getField('accessDate'), true
this._id("fileName-label").value = Zotero.getString('pane.item.attachments.filename') ).toLocaleString();
+ Zotero.getString('punctuation.colon'); accessed.hidden = false;
this._id("fileName").value = fileName; }
fileNameRow.hidden = false; else {
} accessed.hidden = true;
else { }
fileNameRow.hidden = true; }
} // Metadata for files
else {
urlField.hidden = true;
accessed.hidden = true;
}
if (this.item.attachmentLinkMode
!= Zotero.Attachments.LINK_MODE_LINKED_URL
&& this.displayFileName) {
var fileName = this.item.attachmentFilename;
if (fileName) {
this._id("fileName-label").value = Zotero.getString('pane.item.attachments.filename')
+ Zotero.getString('punctuation.colon');
this._id("fileName").value = fileName;
fileNameRow.hidden = false;
} }
else { else {
fileNameRow.hidden = true; fileNameRow.hidden = true;
} }
}
// Page count else {
if (this.displayPages) { fileNameRow.hidden = true;
var pages = yield Zotero.Fulltext.getPages(this.item.id) }
.tap(() => Zotero.Promise.check(this.item));
var pages = pages ? pages.total : null; // Page count
if (this.displayPages) {
Zotero.Fulltext.getPages(this.item.id)
.tap(() => Zotero.Promise.check(this.item))
.then(function (pages) {
pages = pages ? pages.total : null;
if (pages) { if (pages) {
this._id("pages-label").value = Zotero.getString('itemFields.pages') this._id("pages-label").value = Zotero.getString('itemFields.pages')
+ Zotero.getString('punctuation.colon'); + Zotero.getString('punctuation.colon');
@ -295,77 +300,85 @@
else { else {
pagesRow.hidden = true; pagesRow.hidden = true;
} }
} });
else { }
pagesRow.hidden = true; else {
} pagesRow.hidden = true;
}
if (this.displayDateModified) {
this._id("dateModified-label").value = Zotero.getString('itemFields.dateModified') if (this.displayDateModified) {
+ Zotero.getString('punctuation.colon'); this._id("dateModified-label").value = Zotero.getString('itemFields.dateModified')
var mtime = yield this.item.attachmentModificationTime + Zotero.getString('punctuation.colon');
.tap(() => Zotero.Promise.check(this.item)); // Conflict resolution uses a modal window, so promises won't work, but
if (mtime) { // the sync process passes in the file mod time as dateModified
this._id("dateModified").value = new Date(mtime).toLocaleString(); if (this.synchronous) {
} this._id("dateModified").value = Zotero.Date.sqlToDate(
// Use the item's mod time as a backup (e.g., when sync this.item.getField('dateModified'), true
// passes in the mod time for the nonexistent remote file) ).toLocaleString();
else {
this._id("dateModified").value = Zotero.Date.sqlToDate(
this.item.getField('dateModified'), true
).toLocaleString();
}
dateModifiedRow.hidden = false; dateModifiedRow.hidden = false;
} }
else { else {
dateModifiedRow.hidden = true; this.item.attachmentModificationTime
.tap(() => Zotero.Promise.check(this._id))
.then(function (mtime) {
if (!this._id) return;
if (mtime) {
this._id("dateModified").value = new Date(mtime).toLocaleString();
}
dateModifiedRow.hidden = false;
});
} }
}
// Full-text index information else {
if (this.displayIndexed) { dateModifiedRow.hidden = true;
yield this.updateItemIndexedState() }
.tap(() => Zotero.Promise.check(this.item));
// Full-text index information
if (this.displayIndexed) {
this.updateItemIndexedState()
.tap(() => Zotero.Promise.check(this.item))
.then(function () {
indexStatusRow.hidden = false; indexStatusRow.hidden = false;
} });
else { }
indexStatusRow.hidden = true; else {
} indexStatusRow.hidden = true;
}
// Note editor
var noteEditor = this._id('attachment-note-editor'); // Note editor
if (this.displayNote) { var noteEditor = this._id('attachment-note-editor');
if (this.displayNoteIfEmpty || this.item.getNote() != '') { if (this.displayNote) {
Zotero.debug("setting links on top"); if (this.displayNoteIfEmpty || this.item.getNote() != '') {
noteEditor.linksOnTop = true; Zotero.debug("setting links on top");
noteEditor.hidden = false; noteEditor.linksOnTop = true;
noteEditor.hidden = false;
// Don't make note editable (at least for now)
if (this.mode == 'merge' || this.mode == 'mergeedit') {
noteEditor.mode = 'merge';
noteEditor.displayButton = false;
}
else {
noteEditor.mode = this.mode;
}
noteEditor.parent = null;
noteEditor.item = this.item;
}
}
else {
noteEditor.hidden = true;
}
// Don't make note editable (at least for now)
if (this.mode == 'merge' || this.mode == 'mergeedit') {
noteEditor.mode = 'merge';
noteEditor.displayButton = false;
}
else {
noteEditor.mode = this.mode;
}
noteEditor.parent = null;
noteEditor.item = this.item;
}
}
else {
noteEditor.hidden = true;
}
if (this.displayButton) {
selectButton.label = this.buttonCaption; if (this.displayButton) {
selectButton.hidden = false; selectButton.label = this.buttonCaption;
selectButton.setAttribute('oncommand', selectButton.hidden = false;
'document.getBindingParent(this).clickHandler(this)'); selectButton.setAttribute('oncommand',
} 'document.getBindingParent(this).clickHandler(this)');
else { }
selectButton.hidden = true; else {
} selectButton.hidden = true;
}, this); }
]]></body> ]]></body>
</method> </method>

9
chrome/content/zotero/bindings/itembox.xml
View File

@ -71,6 +71,7 @@
switch (val) { switch (val) {
case 'view': case 'view':
case 'merge':
break; break;
case 'edit': case 'edit':
@ -99,10 +100,9 @@
<field name="_item"/> <field name="_item"/>
<property name="item" onget="return this._item;"> <property name="item" onget="return this._item;">
<setter> <setter><![CDATA[
<![CDATA[
if (!(val instanceof Zotero.Item)) { if (!(val instanceof Zotero.Item)) {
throw ("<zoteroitembox>.item must be a Zotero.Item"); throw new Error("'item' must be a Zotero.Item");
} }
// When changing items, reset truncation of creator list // When changing items, reset truncation of creator list
@ -112,8 +112,7 @@
this._item = val; this._item = val;
this.refresh(); this.refresh();
]]> ]]></setter>
</setter>
</property> </property>
<!-- .ref is an alias for .item --> <!-- .ref is an alias for .item -->

22
chrome/content/zotero/bindings/merge.xml
View File

@ -99,9 +99,11 @@
} }
// Check for note or attachment // Check for note or attachment
this.type = this._getTypeFromObject( if (!this.type) {
this._data.left.deleted ? this._data.right : this._data.left this.type = this._getTypeFromObject(
); this._data.left.deleted ? this._data.right : this._data.left
);
}
var showButton = this.type != 'item'; var showButton = this.type != 'item';
@ -109,7 +111,6 @@
this._rightpane.showButton = showButton; this._rightpane.showButton = showButton;
this._leftpane.data = this._data.left; this._leftpane.data = this._data.left;
this._rightpane.data = this._data.right; this._rightpane.data = this._data.right;
this._mergepane.type = this.type;
this._mergepane.data = this._data.merge; this._mergepane.data = this._data.merge;
if (this._data.selected == 'left') { if (this._data.selected == 'left') {
@ -313,6 +314,7 @@
break; break;
case 'attachment': case 'attachment':
case 'file':
elementName = 'zoteroattachmentbox'; elementName = 'zoteroattachmentbox';
break; break;
@ -320,13 +322,8 @@
elementName = 'zoteronoteeditor'; elementName = 'zoteronoteeditor';
break; break;
case 'file':
elementName = 'zoterostoragefilebox';
break;
default: default:
throw ("Object type '" + this.type throw new Error("Object type '" + this.type + "' not supported");
+ "' not supported in <zoteromergepane>.ref");
} }
var objbox = document.createElement(elementName); var objbox = document.createElement(elementName);
@ -342,8 +339,7 @@
objbox.setAttribute("anonid", "objectbox"); objbox.setAttribute("anonid", "objectbox");
objbox.setAttribute("flex", "1"); objbox.setAttribute("flex", "1");
objbox.mode = this.type == 'file' ? 'filemerge' : 'merge';
objbox.mode = 'view';
var button = this._id('choose-button'); var button = this._id('choose-button');
if (this.showButton) { if (this.showButton) {
@ -363,7 +359,7 @@
// Create item from JSON for metadata box // Create item from JSON for metadata box
var item = new Zotero.Item(val.itemType); var item = new Zotero.Item(val.itemType);
item.fromJSON(val); item.fromJSON(val);
objbox.ref = item; objbox.item = item;
]]> ]]>
</setter> </setter>
</property> </property>

1
chrome/content/zotero/bindings/noteeditor.xml
View File

@ -64,6 +64,7 @@
switch (val) { switch (val) {
case 'view': case 'view':
case 'merge':
break; break;
case 'edit': case 'edit':

1
chrome/content/zotero/bindings/relatedbox.xml
View File

@ -109,6 +109,7 @@
]]> ]]>
</destructor> </destructor>
<!-- TODO: Asyncify -->
<method name="notify"> <method name="notify">
<parameter name="event"/> <parameter name="event"/>
<parameter name="type"/> <parameter name="type"/>

32
chrome/content/zotero/merge.js
View File

@ -47,13 +47,20 @@ var Zotero_Merge_Window = new function () {
_wizard.getButton('cancel').setAttribute('label', Zotero.getString('sync.cancel')); _wizard.getButton('cancel').setAttribute('label', Zotero.getString('sync.cancel'));
_io = window.arguments[0].wrappedJSObject; _io = window.arguments[0];
// Not totally clear when this is necessary
if (window.arguments[0].wrappedJSObject) {
_io = window.arguments[0].wrappedJSObject;
}
_conflicts = _io.dataIn.conflicts; _conflicts = _io.dataIn.conflicts;
if (!_conflicts.length) { if (!_conflicts.length) {
// TODO: handle no conflicts // TODO: handle no conflicts
return; return;
} }
if (_io.dataIn.type) {
_mergeGroup.type = _io.dataIn.type;
}
_mergeGroup.leftCaption = _io.dataIn.captions[0]; _mergeGroup.leftCaption = _io.dataIn.captions[0];
_mergeGroup.rightCaption = _io.dataIn.captions[1]; _mergeGroup.rightCaption = _io.dataIn.captions[1];
_mergeGroup.mergeCaption = _io.dataIn.captions[2]; _mergeGroup.mergeCaption = _io.dataIn.captions[2];
@ -240,7 +247,7 @@ var Zotero_Merge_Window = new function () {
} }
// Apply changes from each side and pick most recent version for conflicting fields // Apply changes from each side and pick most recent version for conflicting fields
var mergeInfo = { var mergeInfo = {
data: {} data: {}
}; };
Object.assign(mergeInfo.data, _conflicts[pos].left) Object.assign(mergeInfo.data, _conflicts[pos].left)
Zotero.DataObjectUtilities.applyChanges(mergeInfo.data, _conflicts[pos].changes); Zotero.DataObjectUtilities.applyChanges(mergeInfo.data, _conflicts[pos].changes);
@ -251,7 +258,9 @@ var Zotero_Merge_Window = new function () {
else { else {
var side = 1; var side = 1;
} }
Zotero.DataObjectUtilities.applyChanges(mergeInfo.data, _conflicts[pos].conflicts.map(x => x[side])); Zotero.DataObjectUtilities.applyChanges(
mergeInfo.data, _conflicts[pos].conflicts.map(x => x[side])
);
mergeInfo.selected = side ? 'right' : 'left'; mergeInfo.selected = side ? 'right' : 'left';
return mergeInfo; return mergeInfo;
} }
@ -284,13 +293,22 @@ var Zotero_Merge_Window = new function () {
function _updateResolveAllCheckbox() { function _updateResolveAllCheckbox() {
if (_mergeGroup.rightpane.getAttribute("selected") == 'true') { if (_mergeGroup.type == 'file') {
var label = 'resolveAllRemoteFields'; if (_mergeGroup.rightpane.getAttribute("selected") == 'true') {
var label = 'resolveAllRemote';
}
else {
var label = 'resolveAllLocal';
}
} }
else { else {
var label = 'resolveAllLocalFields'; if (_mergeGroup.rightpane.getAttribute("selected") == 'true') {
var label = 'resolveAllRemoteFields';
}
else {
var label = 'resolveAllLocalFields';
}
} }
// TODO: files
_resolveAllCheckbox.label = Zotero.getString('sync.conflict.' + label); _resolveAllCheckbox.label = Zotero.getString('sync.conflict.' + label);
} }

28
chrome/content/zotero/xpcom/data/item.js
View File

@ -50,7 +50,6 @@ Zotero.Item = function(itemTypeOrID) {
this._attachmentLinkMode = null; this._attachmentLinkMode = null;
this._attachmentContentType = null; this._attachmentContentType = null;
this._attachmentPath = null; this._attachmentPath = null;
this._attachmentSyncState = null;
// loadCreators // loadCreators
this._creators = []; this._creators = [];
@ -1453,14 +1452,13 @@ Zotero.Item.prototype._saveData = Zotero.Promise.coroutine(function* (env) {
if (this._changed.attachmentData) { if (this._changed.attachmentData) {
let sql = "REPLACE INTO itemAttachments (itemID, parentItemID, linkMode, " let sql = "REPLACE INTO itemAttachments (itemID, parentItemID, linkMode, "
+ "contentType, charsetID, path, syncState) VALUES (?,?,?,?,?,?,?)"; + "contentType, charsetID, path) VALUES (?,?,?,?,?,?)";
let linkMode = this.attachmentLinkMode; let linkMode = this.attachmentLinkMode;
let contentType = this.attachmentContentType; let contentType = this.attachmentContentType;
let charsetID = this.attachmentCharset let charsetID = this.attachmentCharset
? Zotero.CharacterSets.getID(this.attachmentCharset) ? Zotero.CharacterSets.getID(this.attachmentCharset)
: null; : null;
let path = this.attachmentPath; let path = this.attachmentPath;
let syncState = this.attachmentSyncState;
if (linkMode == Zotero.Attachments.LINK_MODE_LINKED_FILE && libraryType != 'user') { if (linkMode == Zotero.Attachments.LINK_MODE_LINKED_FILE && libraryType != 'user') {
throw new Error("Linked files can only be added to user library"); throw new Error("Linked files can only be added to user library");
@ -1472,8 +1470,7 @@ Zotero.Item.prototype._saveData = Zotero.Promise.coroutine(function* (env) {
{ int: linkMode }, { int: linkMode },
contentType ? { string: contentType } : null, contentType ? { string: contentType } : null,
charsetID ? { int: charsetID } : null, charsetID ? { int: charsetID } : null,
path ? { string: path } : null, path ? { string: path } : null
syncState ? { int: syncState } : 0
]; ];
yield Zotero.DB.queryAsync(sql, params); yield Zotero.DB.queryAsync(sql, params);
@ -2295,8 +2292,10 @@ Zotero.Item.prototype.renameAttachmentFile = Zotero.Promise.coroutine(function*
yield this.relinkAttachmentFile(destPath); yield this.relinkAttachmentFile(destPath);
yield Zotero.DB.executeTransaction(function* () { yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.setSyncedHash(this.id, null, false); yield Zotero.Sync.Storage.Local.setSyncedHash(this.id, null, false);
yield Zotero.Sync.Storage.setSyncState(this.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD); yield Zotero.Sync.Storage.Local.setSyncState(
this.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
);
}.bind(this)); }.bind(this));
return true; return true;
@ -2317,11 +2316,10 @@ Zotero.Item.prototype.renameAttachmentFile = Zotero.Promise.coroutine(function*
/** /**
* @param {string} path File path * @param {string} path File path
* @param {Boolean} [skipItemUpdate] Don't update attachment item mod time, * @param {Boolean} [skipItemUpdate] Don't update attachment item mod time, so that item doesn't
* so that item doesn't sync. Used when a file * sync. Used when a file needs to be renamed to be accessible but the user doesn't have
* needs to be renamed to be accessible but the * access to modify the attachment metadata. This also allows a save when the library is
* user doesn't have access to modify the * read-only.
* attachment metadata
*/ */
Zotero.Item.prototype.relinkAttachmentFile = Zotero.Promise.coroutine(function* (path, skipItemUpdate) { Zotero.Item.prototype.relinkAttachmentFile = Zotero.Promise.coroutine(function* (path, skipItemUpdate) {
if (path instanceof Components.interfaces.nsIFile) { if (path instanceof Components.interfaces.nsIFile) {
@ -2382,7 +2380,8 @@ Zotero.Item.prototype.relinkAttachmentFile = Zotero.Promise.coroutine(function*
yield this.saveTx({ yield this.saveTx({
skipDateModifiedUpdate: true, skipDateModifiedUpdate: true,
skipClientDateModifiedUpdate: skipItemUpdate skipClientDateModifiedUpdate: skipItemUpdate,
skipEditCheck: skipItemUpdate
}); });
return true; return true;
@ -3606,9 +3605,6 @@ Zotero.Item.prototype.clone = Zotero.Promise.coroutine(function* (libraryID, ski
if (this.attachmentPath) { if (this.attachmentPath) {
newItem.attachmentPath = this.attachmentPath; newItem.attachmentPath = this.attachmentPath;
} }
if (this.attachmentSyncState) {
newItem.attachmentSyncState = this.attachmentSyncState;
}
} }
} }
} }

3
chrome/content/zotero/xpcom/data/items.js
View File

@ -84,8 +84,7 @@ Zotero.Items = function() {
attachmentCharset: "CS.charset AS attachmentCharset", attachmentCharset: "CS.charset AS attachmentCharset",
attachmentLinkMode: "IA.linkMode AS attachmentLinkMode", attachmentLinkMode: "IA.linkMode AS attachmentLinkMode",
attachmentContentType: "IA.contentType AS attachmentContentType", attachmentContentType: "IA.contentType AS attachmentContentType",
attachmentPath: "IA.path AS attachmentPath", attachmentPath: "IA.path AS attachmentPath"
attachmentSyncState: "IA.syncState AS attachmentSyncState"
}; };
} }
}, {lazy: true}); }, {lazy: true});

90
chrome/content/zotero/xpcom/file.js
View File

@ -48,7 +48,7 @@ Zotero.File = new function(){
else if (pathOrFile instanceof Ci.nsIFile) { else if (pathOrFile instanceof Ci.nsIFile) {
return pathOrFile; return pathOrFile;
} }
throw new Error('Unexpected value provided to Zotero.File.pathToFile() (' + pathOrFile + ')'); throw new Error("Unexpected value '" + pathOrFile + "'");
} }
@ -348,7 +348,7 @@ Zotero.File = new function(){
* @param {String} [charset] - The character set; defaults to UTF-8 * @param {String} [charset] - The character set; defaults to UTF-8
* @return {Promise} - A promise that is resolved when the file has been written * @return {Promise} - A promise that is resolved when the file has been written
*/ */
this.putContentsAsync = function putContentsAsync(path, data, charset) { this.putContentsAsync = function (path, data, charset) {
if (path instanceof Ci.nsIFile) { if (path instanceof Ci.nsIFile) {
path = path.path; path = path.path;
} }
@ -424,18 +424,17 @@ Zotero.File = new function(){
* iterator when done * iterator when done
* *
* The DirectoryInterator is passed as the first parameter to the generator. * The DirectoryInterator is passed as the first parameter to the generator.
* A StopIteration error will be caught automatically.
* *
* Zotero.File.iterateDirectory(path, function* (iterator) { * Zotero.File.iterateDirectory(path, function* (iterator) {
* while (true) { * while (true) {
* var entry = yield iterator.next(); * var entry = yield iterator.next();
* [...] * [...]
* } * }
* }).done() * })
* *
* @return {Promise} * @return {Promise}
*/ */
this.iterateDirectory = function iterateDirectory(path, generator) { this.iterateDirectory = function (path, generator) {
var iterator = new OS.File.DirectoryIterator(path); var iterator = new OS.File.DirectoryIterator(path);
return Zotero.Promise.coroutine(generator)(iterator) return Zotero.Promise.coroutine(generator)(iterator)
.catch(function (e) { .catch(function (e) {
@ -470,6 +469,8 @@ Zotero.File = new function(){
this.createShortened = function (file, type, mode, maxBytes) { this.createShortened = function (file, type, mode, maxBytes) {
file = this.pathToFile(file);
if (!maxBytes) { if (!maxBytes) {
maxBytes = 255; maxBytes = 255;
} }
@ -575,6 +576,8 @@ Zotero.File = new function(){
} }
break; break;
} }
return file.leafName;
} }
@ -902,29 +905,28 @@ Zotero.File = new function(){
this.checkFileAccessError = function (e, file, operation) { this.checkFileAccessError = function (e, file, operation) {
var str = 'file.accessError.';
if (file) { if (file) {
var str = Zotero.getString('file.accessError.theFile', file.path); str += 'theFile'
} }
else { else {
var str = Zotero.getString('file.accessError.aFile'); str += 'aFile'
} }
str += 'CannotBe';
switch (operation) { switch (operation) {
case 'create': case 'create':
var opWord = Zotero.getString('file.accessError.created'); str += 'Created';
break;
case 'update':
var opWord = Zotero.getString('file.accessError.updated');
break; break;
case 'delete': case 'delete':
var opWord = Zotero.getString('file.accessError.deleted'); str += 'Deleted';
break; break;
default: default:
var opWord = Zotero.getString('file.accessError.updated'); str += 'Updated';
} }
str = Zotero.getString(str, file.path ? file.path : undefined);
Zotero.debug(file.path); Zotero.debug(file.path);
Zotero.debug(e, 1); Zotero.debug(e, 1);
@ -962,4 +964,64 @@ Zotero.File = new function(){
throw (e); throw (e);
} }
this.checkPathAccessError = function (e, path, operation) {
var str = 'file.accessError.';
if (path) {
str += 'theFile'
}
else {
str += 'aFile'
}
str += 'CannotBe';
switch (operation) {
case 'create':
str += 'Created';
break;
case 'delete':
str += 'Deleted';
break;
default:
str += 'Updated';
}
str = Zotero.getString(str, path ? path : undefined);
Zotero.debug(path);
Zotero.debug(e, 1);
Components.utils.reportError(e);
// TODO: Check for specific errors?
if (e instanceof OS.File.Error) {
let checkFileWindows = Zotero.getString('file.accessError.message.windows');
let checkFileOther = Zotero.getString('file.accessError.message.other');
var msg = str + "\n\n"
+ (Zotero.isWin ? checkFileWindows : checkFileOther)
+ "\n\n"
+ Zotero.getString('file.accessError.restart');
var e = new Zotero.Error(
msg,
0,
{
dialogButtonText: Zotero.getString('file.accessError.showParentDir'),
dialogButtonCallback: function () {
try {
file.parent.QueryInterface(Components.interfaces.nsILocalFile);
file.parent.reveal();
}
// Unsupported on some platforms
catch (e2) {
Zotero.launchFile(file.parent);
}
}
}
);
}
throw e;
}
} }

1661
chrome/content/zotero/xpcom/storage.js
View File

File diff suppressed because it is too large Load Diff

87
chrome/content/zotero/xpcom/storage/mode.js
View File

@ -1,87 +0,0 @@
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2009 Center for History and New Media
George Mason University, Fairfax, Virginia, USA
http://zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
Zotero.Sync.Storage.Mode = function () {};
Zotero.Sync.Storage.Mode.prototype.__defineGetter__('verified', function () {
return this._verified;
});
Zotero.Sync.Storage.Mode.prototype.__defineGetter__('username', function () {
return this._username;
});
Zotero.Sync.Storage.Mode.prototype.__defineGetter__('password', function () {
return this._password;
});
Zotero.Sync.Storage.Mode.prototype.__defineSetter__('password', function (val) {
this._password = val;
});
Zotero.Sync.Storage.Mode.prototype.init = function () {
return this._init();
}
Zotero.Sync.Storage.Mode.prototype.sync = function (observer) {
return Zotero.Sync.Storage.sync(this.name, observer);
}
Zotero.Sync.Storage.Mode.prototype.downloadFile = function (request) {
return this._downloadFile(request);
}
Zotero.Sync.Storage.Mode.prototype.uploadFile = function (request) {
return this._uploadFile(request);
}
Zotero.Sync.Storage.Mode.prototype.getLastSyncTime = function (libraryID) {
return this._getLastSyncTime(libraryID);
}
Zotero.Sync.Storage.Mode.prototype.setLastSyncTime = function (callback, useLastSyncTime) {
return this._setLastSyncTime(callback, useLastSyncTime);
}
Zotero.Sync.Storage.Mode.prototype.checkServer = function (callback) {
return this._checkServer(callback);
}
Zotero.Sync.Storage.Mode.prototype.checkServerCallback = function (uri, status, window, skipSuccessMessage) {
return this._checkServerCallback(uri, status, window, skipSuccessMessage);
}
Zotero.Sync.Storage.Mode.prototype.cacheCredentials = function () {
return this._cacheCredentials();
}
Zotero.Sync.Storage.Mode.prototype.purgeDeletedStorageFiles = function (callback) {
return this._purgeDeletedStorageFiles(callback);
}
Zotero.Sync.Storage.Mode.prototype.purgeOrphanedStorageFiles = function (callback) {
return this._purgeOrphanedStorageFiles(callback);
}

427
chrome/content/zotero/xpcom/storage/queue.js
View File

@ -1,427 +0,0 @@
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2009 Center for History and New Media
George Mason University, Fairfax, Virginia, USA
http://zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
/**
* Queue for storage sync transfer requests
*
* @param {String} type Queue type (e.g., 'download' or 'upload')
*/
Zotero.Sync.Storage.Queue = function (type, libraryID) {
Zotero.debug("Initializing " + type + " queue for library " + libraryID);
// Public properties
this.type = type;
this.libraryID = libraryID;
this.maxConcurrentRequests = 1;
this.activeRequests = 0;
this.totalRequests = 0;
// Private properties
this._requests = {};
this._highPriority = [];
this._running = false;
this._stopping = false;
this._finished = false;
this._error = false;
this._finishedReqs = 0;
this._localChanges = false;
this._remoteChanges = false;
this._conflicts = [];
this._cachedPercentage;
this._cachedPercentageTime;
}
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('name', function () {
return this.type + "/" + this.libraryID;
});
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('Type', function () {
return this.type[0].toUpperCase() + this.type.substr(1);
});
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('running', function () this._running);
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('stopping', function () this._stopping);
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('finished', function () this._finished);
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('unfinishedRequests', function () {
return this.totalRequests - this.finishedRequests;
});
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('finishedRequests', function () {
return this._finishedReqs;
});
Zotero.Sync.Storage.Queue.prototype.__defineSetter__('finishedRequests', function (val) {
Zotero.debug("Finished requests: " + val);
Zotero.debug("Total requests: " + this.totalRequests);
this._finishedReqs = val;
if (val == 0) {
return;
}
// Last request
if (val == this.totalRequests) {
Zotero.debug(this.Type + " queue is done for library " + this.libraryID);
// DEBUG info
Zotero.debug("Active requests: " + this.activeRequests);
if (this.activeRequests) {
throw new Error(this.Type + " queue for library " + this.libraryID
+ " can't be done if there are active requests");
}
this._running = false;
this._stopping = false;
this._finished = true;
this._requests = {};
this._highPriority = [];
var localChanges = this._localChanges;
var remoteChanges = this._remoteChanges;
var conflicts = this._conflicts.concat();
var deferred = this._deferred;
this._localChanges = false;
this._remoteChanges = false;
this._conflicts = [];
this._deferred = null;
if (!this._error) {
Zotero.debug("Resolving promise for queue " + this.name);
Zotero.debug(this._localChanges);
Zotero.debug(this._remoteChanges);
Zotero.debug(this._conflicts);
deferred.resolve({
libraryID: this.libraryID,
type: this.type,
localChanges: localChanges,
remoteChanges: remoteChanges,
conflicts: conflicts
});
}
else {
Zotero.debug("Rejecting promise for queue " + this.name);
var e = this._error;
this._error = false;
e.libraryID = this.libraryID;
e.type = this.type;
deferred.reject(e);
}
return;
}
if (this._stopping) {
return;
}
this.advance();
});
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('queuedRequests', function () {
return this.unfinishedRequests - this.activeRequests;
});
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('remaining', function () {
var remaining = 0;
for each(var request in this._requests) {
remaining += request.remaining;
}
return remaining;
});
Zotero.Sync.Storage.Queue.prototype.__defineGetter__('percentage', function () {
if (this.totalRequests == 0) {
return 0;
}
if (this._finished) {
return 100;
}
// Cache percentage for a second
if (this._cachedPercentage && (new Date() - this._cachedPercentageTime) < 1000) {
return this._cachedPercentage;
}
var completedRequests = 0;
for each(var request in this._requests) {
completedRequests += request.percentage / 100;
}
this._cachedPercentage = Math.round((completedRequests / this.totalRequests) * 100);
this._cachedPercentageTime = new Date();
return this._cachedPercentage;
});
Zotero.Sync.Storage.Queue.prototype.isRunning = function () {
return this._running;
}
Zotero.Sync.Storage.Queue.prototype.isStopping = function () {
return this._stopping;
}
/**
* Add a request to this queue
*
* @param {Zotero.Sync.Storage.Request} request
* @param {Boolean} highPriority Add or move request to high priority queue
*/
Zotero.Sync.Storage.Queue.prototype.addRequest = function (request, highPriority) {
if (this._finished) {
this.reset();
}
request.queue = this;
var name = request.name;
Zotero.debug("Queuing " + this.type + " request '" + name + "' for library " + this.libraryID);
if (this._requests[name]) {
if (highPriority) {
Zotero.debug("Moving " + name + " to high-priority queue");
this._requests[name].importCallbacks(request);
this._highPriority.push(name);
return;
}
Zotero.debug("Request '" + name + "' already exists");
return;
}
this._requests[name] = request;
this.totalRequests++;
if (highPriority) {
this._highPriority.push(name);
}
}
Zotero.Sync.Storage.Queue.prototype.start = function () {
if (!this._deferred || this._deferred.promise.isFulfilled()) {
Zotero.debug("Creating deferred for queue " + this.name);
this._deferred = Zotero.Promise.defer();
}
// The queue manager needs to know what queues were running in the
// current session
Zotero.Sync.Storage.QueueManager.addCurrentQueue(this);
var self = this;
setTimeout(function () {
self.advance();
}, 0);
return this._deferred.promise;
}
/**
* Start another request in this queue if there's an available slot
*/
Zotero.Sync.Storage.Queue.prototype.advance = function () {
this._running = true;
this._finished = false;
if (this._stopping) {
Zotero.debug(this.Type + " queue for library " + this.libraryID
+ "is being stopped in Zotero.Sync.Storage.Queue.advance()", 2);
return;
}
if (!this.queuedRequests) {
Zotero.debug("No remaining requests in " + this.type
+ " queue for library " + this.libraryID + " ("
+ this.activeRequests + " active, "
+ this.finishedRequests + " finished)");
return;
}
if (this.activeRequests >= this.maxConcurrentRequests) {
Zotero.debug(this.Type + " queue for library " + this.libraryID
+ " is busy (" + this.activeRequests + "/"
+ this.maxConcurrentRequests + ")");
return;
}
// Start the first unprocessed request
// Try the high-priority queue first
var self = this;
var request, name;
while (name = this._highPriority.shift()) {
request = this._requests[name];
if (request.isRunning() || request.isFinished()) {
continue;
}
let requestName = name;
Zotero.Promise.try(function () {
var promise = request.start();
self.advance();
return promise;
})
.then(function (result) {
if (result.localChanges) {
self._localChanges = true;
}
if (result.remoteChanges) {
self._remoteChanges = true;
}
if (result.conflict) {
self.addConflict(
requestName,
result.conflict.local,
result.conflict.remote
);
}
})
.catch(function (e) {
self.error(e);
});
return;
}
// And then others
for each(var request in this._requests) {
if (request.isRunning() || request.isFinished()) {
continue;
}
let requestName = request.name;
// This isn't in a Zotero.Promise.try() because the request needs to get marked
// as running immediately so that it doesn't get run again by a
// subsequent advance() call.
try {
var promise = request.start();
self.advance();
}
catch (e) {
self.error(e);
}
promise.then(function (result) {
if (result.localChanges) {
self._localChanges = true;
}
if (result.remoteChanges) {
self._remoteChanges = true;
}
if (result.conflict) {
self.addConflict(
requestName,
result.conflict.local,
result.conflict.remote
);
}
})
.catch(function (e) {
self.error(e);
});
return;
}
}
Zotero.Sync.Storage.Queue.prototype.updateProgress = function () {
Zotero.Sync.Storage.QueueManager.updateProgress();
}
Zotero.Sync.Storage.Queue.prototype.addConflict = function (requestName, localData, remoteData) {
Zotero.debug('===========');
Zotero.debug(localData);
Zotero.debug(remoteData);
this._conflicts.push({
name: requestName,
localData: localData,
remoteData: remoteData
});
}
Zotero.Sync.Storage.Queue.prototype.error = function (e) {
if (!this._error) {
if (this.isRunning()) {
this._error = e;
}
else {
Zotero.debug("Queue " + this.name + " was no longer running -- not assigning error", 2);
}
}
Zotero.debug(e, 1);
this.stop();
}
/**
* Stops all requests in this queue
*/
Zotero.Sync.Storage.Queue.prototype.stop = function () {
if (!this._running) {
Zotero.debug(this.Type + " queue for library " + this.libraryID
+ " is not running");
return;
}
if (this._stopping) {
Zotero.debug("Already stopping " + this.type + " queue for library "
+ this.libraryID);
return;
}
Zotero.debug("Stopping " + this.type + " queue for library " + this.libraryID);
// If no requests, finish manually
/*if (this.activeRequests == 0) {
this._finishedRequests = this._finishedRequests;
return;
}*/
this._stopping = true;
for each(var request in this._requests) {
if (!request.isFinished()) {
request.stop(true);
}
}
Zotero.debug("Queue is stopped");
}
Zotero.Sync.Storage.Queue.prototype.reset = function () {
this._finished = false;
this._finishedReqs = 0;
this.totalRequests = 0;
}

370
chrome/content/zotero/xpcom/storage/queueManager.js
View File

@ -1,370 +0,0 @@
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2009 Center for History and New Media
George Mason University, Fairfax, Virginia, USA
http://zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
Zotero.Sync.Storage.QueueManager = new function () {
var _queues = {};
var _currentQueues = [];
this.start = Zotero.Promise.coroutine(function* (libraryID) {
if (libraryID) {
var queues = this.getAll(libraryID);
var suffix = " for library " + libraryID;
}
else {
var queues = this.getAll();
var suffix = "";
}
Zotero.debug("Starting file sync queues" + suffix);
var promises = [];
for each(var queue in queues) {
if (!queue.unfinishedRequests) {
continue;
}
Zotero.debug("Starting queue " + queue.name);
promises.push(queue.start());
}
if (!promises.length) {
Zotero.debug("No files to sync" + suffix);
}
var results = yield Zotero.Promise.allSettled(promises);
Zotero.debug("All storage queues are finished" + suffix);
for (let i = 0; i < results.length; i++) {
let result = results[i];
// Check for conflicts to resolve
if (result.state == "fulfilled") {
result = result.value;
if (result.conflicts.length) {
Zotero.debug("Reconciling conflicts for library " + result.libraryID);
Zotero.debug(result.conflicts);
var data = yield _reconcileConflicts(result.conflicts);
if (data) {
_processMergeData(data);
}
}
}
}
return promises;
});
this.stop = function (libraryID) {
if (libraryID) {
var queues = this.getAll(libraryID);
}
else {
var queues = this.getAll();
}
for (var queue in queues) {
queue.stop();
}
};
/**
* Retrieving a queue, creating a new one if necessary
*
* @param {String} queueName
*/
this.get = function (queueName, libraryID, noInit) {
if (typeof libraryID == 'undefined') {
throw new Error("libraryID not specified");
}
var hash = queueName + "/" + libraryID;
// Initialize the queue if it doesn't exist yet
if (!_queues[hash]) {
if (noInit) {
return false;
}
var queue = new Zotero.Sync.Storage.Queue(queueName, libraryID);
switch (queueName) {
case 'download':
queue.maxConcurrentRequests =
Zotero.Prefs.get('sync.storage.maxDownloads')
break;
case 'upload':
queue.maxConcurrentRequests =
Zotero.Prefs.get('sync.storage.maxUploads')
break;
default:
throw ("Invalid queue '" + queueName + "' in Zotero.Sync.Storage.QueueManager.get()");
}
_queues[hash] = queue;
}
return _queues[hash];
};
this.getAll = function (libraryID) {
if (typeof libraryID == 'string') {
throw new Error("libraryID must be a number or undefined");
}
var queues = [];
for each(var queue in _queues) {
if (typeof libraryID == 'undefined' || queue.libraryID === libraryID) {
queues.push(queue);
}
}
return queues;
};
this.addCurrentQueue = function (queue) {
if (!this.hasCurrentQueue(queue)) {
_currentQueues.push(queue.name);
}
}
this.hasCurrentQueue = function (queue) {
return _currentQueues.indexOf(queue.name) != -1;
}
/**
* Stop all queues
*
* @param {Boolean} [skipStorageFinish=false] Don't call Zotero.Sync.Storage.finish()
* when done (used when we stopped because of
* an error)
*/
this.cancel = function (skipStorageFinish) {
Zotero.debug("Stopping all storage queues");
for each(var queue in _queues) {
if (queue.isRunning() && !queue.isStopping()) {
queue.stop();
}
}
}
this.finish = function () {
Zotero.debug("All storage queues are finished");
_currentQueues = [];
}
/**
* Calculate the current progress values and trigger a display update
*
* Also detects when all queues have finished and ends sync progress
*/
this.updateProgress = function () {
var activeRequests = 0;
var allFinished = true;
for each(var queue in _queues) {
// Finished or never started
if (!queue.isRunning() && !queue.isStopping()) {
continue;
}
allFinished = false;
activeRequests += queue.activeRequests;
}
if (activeRequests == 0) {
_updateProgressMeters(0);
if (allFinished) {
this.finish();
}
return;
}
var status = {};
for each(var queue in _queues) {
if (!this.hasCurrentQueue(queue)) {
continue;
}
if (!status[queue.libraryID]) {
status[queue.libraryID] = {};
}
if (!status[queue.libraryID][queue.type]) {
status[queue.libraryID][queue.type] = {};
}
status[queue.libraryID][queue.type].statusString = _getQueueStatus(queue);
status[queue.libraryID][queue.type].percentage = queue.percentage;
status[queue.libraryID][queue.type].totalRequests = queue.totalRequests;
status[queue.libraryID][queue.type].finished = queue.finished;
}
_updateProgressMeters(activeRequests, status);
}
/**
* Get a status string for a queue
*
* @param {Zotero.Sync.Storage.Queue} queue
* @return {String}
*/
function _getQueueStatus(queue) {
var remaining = queue.remaining;
var unfinishedRequests = queue.unfinishedRequests;
if (!unfinishedRequests) {
return Zotero.getString('sync.storage.none');
}
if (remaining > 1000) {
var bytesRemaining = Zotero.getString(
'sync.storage.mbRemaining',
Zotero.Utilities.numberFormat(remaining / 1000 / 1000, 1)
);
}
else {
var bytesRemaining = Zotero.getString(
'sync.storage.kbRemaining',
Zotero.Utilities.numberFormat(remaining / 1000, 0)
);
}
var totalRequests = queue.totalRequests;
var filesRemaining = Zotero.getString(
'sync.storage.filesRemaining',
[totalRequests - unfinishedRequests, totalRequests]
);
return bytesRemaining + ' (' + filesRemaining + ')';
}
/**
* Cycle through windows, updating progress meters with new values
*/
function _updateProgressMeters(activeRequests, status) {
// Get overall percentage across queues
var sum = 0, num = 0, percentage, total;
for each(var libraryStatus in status) {
for each(var queueStatus in libraryStatus) {
percentage = queueStatus.percentage;
total = queueStatus.totalRequests;
sum += total * percentage;
num += total;
}
}
var percentage = Math.round(sum / num);
var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
var enumerator = wm.getEnumerator("navigator:browser");
while (enumerator.hasMoreElements()) {
var win = enumerator.getNext();
if (!win.ZoteroPane) continue;
var doc = win.ZoteroPane.document;
var box = doc.getElementById("zotero-tb-sync-progress-box");
var meter = doc.getElementById("zotero-tb-sync-progress");
if (activeRequests == 0) {
box.hidden = true;
continue;
}
meter.setAttribute("value", percentage);
box.hidden = false;
var percentageLabel = doc.getElementById('zotero-tb-sync-progress-tooltip-progress');
percentageLabel.lastChild.setAttribute('value', percentage + "%");
var statusBox = doc.getElementById('zotero-tb-sync-progress-status');
statusBox.data = status;
}
}
var _reconcileConflicts = Zotero.Promise.coroutine(function* (conflicts) {
var objectPairs = [];
for each(var conflict in conflicts) {
var item = Zotero.Sync.Storage.getItemFromRequestName(conflict.name);
var item1 = yield item.clone(false, false, true);
item1.setField('dateModified',
Zotero.Date.dateToSQL(new Date(conflict.localData.modTime), true));
var item2 = yield item.clone(false, false, true);
item2.setField('dateModified',
Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime), true));
objectPairs.push([item1, item2]);
}
var io = {
dataIn: {
type: 'storagefile',
captions: [
Zotero.getString('sync.storage.localFile'),
Zotero.getString('sync.storage.remoteFile'),
Zotero.getString('sync.storage.savedFile')
],
objects: objectPairs
}
};
var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
var lastWin = wm.getMostRecentWindow("navigator:browser");
lastWin.openDialog('chrome://zotero/content/merge.xul', '', 'chrome,modal,centerscreen', io);
if (!io.dataOut) {
return false;
}
// Since we're only putting cloned items into the merge window,
// we have to manually set the ids
for (var i=0; i<conflicts.length; i++) {
io.dataOut[i].id = Zotero.Sync.Storage.getItemFromRequestName(conflicts[i].name).id;
}
return io.dataOut;
});
function _processMergeData(data) {
if (!data.length) {
return false;
}
for each(var mergeItem in data) {
var itemID = mergeItem.id;
var dateModified = mergeItem.ref.getField('dateModified');
// Local
if (dateModified == mergeItem.left.getField('dateModified')) {
Zotero.Sync.Storage.setSyncState(
itemID, Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD
);
}
// Remote
else {
Zotero.Sync.Storage.setSyncState(
itemID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD
);
}
}
}
}

307
chrome/content/zotero/xpcom/storage/storageEngine.js Normal file
View File

@ -0,0 +1,307 @@
/*
***** BEGIN LICENSE BLOCK *****
Copyright © 2015 Center for History and New Media
George Mason University, Fairfax, Virginia, USA
http://zotero.org
This file is part of Zotero.
Zotero is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Zotero is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
***** END LICENSE BLOCK *****
*/
if (!Zotero.Sync.Storage) {
Zotero.Sync.Storage = {};
}
/**
* An Engine manages file sync processes for a given library
*
* @param {Object} options
* @param {Zotero.Sync.APIClient} options.apiClient
* @param {Integer} options.libraryID
* @param {Function} [onError] - Function to run on error
* @param {Boolean} [stopOnError]
*/
Zotero.Sync.Storage.Engine = function (options) {
if (options.apiClient == undefined) {
throw new Error("options.apiClient not set");
}
if (options.libraryID == undefined) {
throw new Error("options.libraryID not set");
}
this.apiClient = options.apiClient;
this.background = options.background;
this.firstInSession = options.firstInSession;
this.lastFullFileCheck = options.lastFullFileCheck;
this.libraryID = options.libraryID;
this.library = Zotero.Libraries.get(options.libraryID);
this.local = Zotero.Sync.Storage.Local;
this.utils = Zotero.Sync.Storage.Utilities;
this.mode = this.local.getModeForLibrary(this.libraryID);
var modeClass = this.utils.getClassForMode(this.mode);
this.controller = new modeClass(options);
this.setStatus = options.setStatus || function () {};
this.onError = options.onError || function (e) {};
this.stopOnError = options.stopOnError || false;
this.queues = [];
['download', 'upload'].forEach(function (type) {
this.queues[type] = new ConcurrentCaller({
id: `${this.libraryID}/${type}`,
numConcurrent: Zotero.Prefs.get(
'sync.storage.max' + Zotero.Utilities.capitalize(type) + 's'
),
onError: this.onError,
stopOnError: this.stopOnError,
logger: Zotero.debug
});
}.bind(this))
this.maxCheckAge = 10800; // maximum age in seconds for upload modification check (3 hours)
}
Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function* () {
var libraryID = this.libraryID;
if (!Zotero.Prefs.get("sync.storage.enabled")) {
Zotero.debug("File sync is not enabled for " + this.library.name);
return false;
}
Zotero.debug("Starting file sync for " + this.library.name);
if (!this.controller.verified) {
Zotero.debug(`${this.mode} file sync is not active`);
throw new Error("Storage mode verification not implemented");
// TODO: Check server
}
if (this.controller.cacheCredentials) {
yield this.controller.cacheCredentials();
}
// Get library last-sync time for download-on-sync libraries.
var lastSyncTime = null;
var downloadAll = this.local.downloadOnSync(libraryID);
if (downloadAll) {
lastSyncTime = yield this.controller.getLastSyncTime(libraryID);
}
// Check for updated files to upload
if (!Zotero.Libraries.isFilesEditable(libraryID)) {
Zotero.debug("No file editing access -- skipping file modification check for "
+ this.library.name);
}
// If this is a background sync, it's not the first sync of the session, the library has had
// at least one full check this session, and it's been less than maxCheckAge since the last
// full check of this library, check only files that were previously modified or opened
// recently
else if (this.background
&& !this.firstInSession
&& this.local.lastFullFileCheck[libraryID]
&& (this.local.lastFullFileCheck[libraryID]
+ (this.maxCheckAge * 1000)) > new Date().getTime()) {
let itemIDs = this.local.getFilesToCheck(libraryID, this.maxCheckAge);
yield this.local.checkForUpdatedFiles(libraryID, itemIDs);
}
// Otherwise check all files in library
else {
this.local.lastFullFileCheck[libraryID] = new Date().getTime();
yield this.local.checkForUpdatedFiles(libraryID);
}
yield this.local.resolveConflicts(libraryID);
var downloadForced = yield this.local.checkForForcedDownloads(libraryID);
// If we don't have any forced downloads, we can skip downloads if the last sync time hasn't
// changed or doesn't exist on the server (meaning there are no files)
if (downloadAll && !downloadForced) {
if (lastSyncTime) {
if (this.library.lastStorageSync == lastSyncTime) {
Zotero.debug("Last " + this.mode.toUpperCase() + " sync id hasn't changed for "
+ this.library.name + " -- skipping file downloads");
downloadAll = false;
}
}
else {
Zotero.debug(`No last ${this.mode} sync time for ${this.library.name}`
+ " -- skipping file downloads");
downloadAll = false;
}
}
// Get files to download
if (downloadAll || downloadForced) {
let itemIDs = yield this.local.getFilesToDownload(libraryID, !downloadAll);
if (itemIDs.length) {
Zotero.debug(itemIDs.length + " file" + (itemIDs.length == 1 ? '' : 's') + " to "
+ "download for " + this.library.name);
for (let itemID of itemIDs) {
let item = yield Zotero.Items.getAsync(itemID);
yield this.queueItem(item);
}
}
else {
Zotero.debug("No files to download for " + this.library.name);
}
}
// Get files to upload
if (Zotero.Libraries.isFilesEditable(libraryID)) {
let itemIDs = yield this.local.getFilesToUpload(libraryID);
if (itemIDs.length) {
Zotero.debug(itemIDs.length + " file" + (itemIDs.length == 1 ? '' : 's') + " to "
+ "upload for " + this.library.name);
for (let itemID of itemIDs) {
let item = yield Zotero.Items.getAsync(itemID, { noCache: true });
yield this.queueItem(item);
}
}
else {
Zotero.debug("No files to upload for " + this.library.name);
}
}
else {
Zotero.debug("No file editing access -- skipping file uploads for " + this.library.name);
}
var promises = {
download: this.queues.download.runAll(),
upload: this.queues.upload.runAll()
}
// Process the results
var changes = new Zotero.Sync.Storage.Result;
for (let type of ['download', 'upload']) {
let results = yield promises[type];
if (this.stopOnError) {
for (let p of results) {
if (p.isRejected()) {
let e = p.reason();
Zotero.debug(`File ${type} sync failed for ${this.library.name}`);
throw e;
}
}
}
Zotero.debug(`File ${type} sync finished for ${this.library.name}`);
changes.updateFromResults(results.filter(p => p.isFulfilled()).map(p => p.value()));
}
// If files were uploaded, update the remote last-sync time
if (changes.remoteChanges) {
lastSyncTime = yield this.controller.setLastSyncTime(libraryID);
if (!lastSyncTime) {
throw new Error("Last sync time not set after sync");
}
}
// If there's a remote last-sync time from either the check before downloads or when it
// was changed after uploads, store that locally so we know we can skip download checks
// next time
if (lastSyncTime) {
this.library.lastStorageSync = lastSyncTime;
yield this.library.saveTx();
}
// If WebDAV sync, purge deleted and orphaned files
if (this.mode == 'webdav') {
try {
yield this.controller.purgeDeletedStorageFiles(libraryID);
yield this.controller.purgeOrphanedStorageFiles(libraryID);
}
catch (e) {
Zotero.logError(e);
}
}
if (!changes.localChanges) {
Zotero.debug("No local changes made during file sync");
}
Zotero.debug("Done with file sync for " + this.library.name);
return changes;
})
Zotero.Sync.Storage.Engine.prototype.stop = function () {
for (let type in this.queues) {
this.queues[type].stop();
}
}
Zotero.Sync.Storage.Engine.prototype.queueItem = Zotero.Promise.coroutine(function* (item) {
switch (yield this.local.getSyncState(item.id)) {
case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD:
var type = 'download';
var onStart = Zotero.Promise.method(function (request) {
return this.controller.downloadFile(request);
}.bind(this));
break;
case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD:
var type = 'upload';
var onStart = Zotero.Promise.method(function (request) {
return this.controller.uploadFile(request);
}.bind(this));
break;
case false:
Zotero.debug("Sync state for item " + item.id + " not found", 2);
return;
default:
throw new Error("Invalid sync state " + (yield this.local.getSyncState(item.id)));
}
var request = new Zotero.Sync.Storage.Request({
type,
libraryID: this.libraryID,
name: item.libraryKey,
onStart,
onProgress: this.onProgress
});
if (type == 'upload') {
try {
request.setMaxSize(yield Zotero.Attachments.getTotalFileSize(item));
}
// If this fails, ignore it, though we might fail later
catch (e) {
// But if the file doesn't exist yet, don't try to upload it
//
// This isn't a perfect test, because the file could still be in the process of being
// downloaded (e.g., from the web). It'd be better to download files to a temp
// directory and move them into place.
if (!(yield item.getFilePathAsync())) {
Zotero.debug("File " + item.libraryKey + " not yet available to upload -- skipping");
return;
}
Zotero.logError(e);
}
}
this.queues[type].add(request.start.bind(request));
})

1088
chrome/content/zotero/xpcom/storage/storageLocal.js Normal file
View File

File diff suppressed because it is too large Load Diff

184
chrome/content/zotero/xpcom/storage/request.js → chrome/content/zotero/xpcom/storage/storageRequest.js
View File

@ -27,15 +27,25 @@
/** /**
* Transfer request for storage sync * Transfer request for storage sync
* *
* @param {String} name Identifier for request (e.g., "[libraryID]/[key]") * @param {Object} options
* @param {Function} onStart Callback to run when request starts * @param {String} options.type
* @param {Integer} options.libraryID
* @param {String} options.name - Identifier for request (e.g., "[libraryID]/[key]")
* @param {Function|Function[]} [options.onStart]
* @param {Function|Function[]} [options.onProgress]
* @param {Function|Function[]} [options.onStop]
*/ */
Zotero.Sync.Storage.Request = function (name, callbacks) { Zotero.Sync.Storage.Request = function (options) {
Zotero.debug("Initializing request '" + name + "'"); if (!options.type) throw new Error("type must be provided");
if (!options.libraryID) throw new Error("libraryID must be provided");
if (!options.name) throw new Error("name must be provided");
['type', 'libraryID', 'name'].forEach(x => this[x] = options[x]);
this.callbacks = ['onStart', 'onProgress']; Zotero.debug(`Initializing ${this.type} request ${this.name}`);
this.name = name; this.callbacks = ['onStart', 'onProgress', 'onStop'];
this.Type = Zotero.Utilities.capitalize(this.type);
this.channel = null; this.channel = null;
this.queue = null; this.queue = null;
this.progress = 0; this.progress = 0;
@ -48,17 +58,10 @@ Zotero.Sync.Storage.Request = function (name, callbacks) {
this._remaining = null; this._remaining = null;
this._maxSize = null; this._maxSize = null;
this._finished = false; this._finished = false;
this._forceFinish = false;
this._changesMade = false;
for (var func in callbacks) { for (let name of this.callbacks) {
if (this.callbacks.indexOf(func) !== -1) { if (!options[name]) continue;
// Stuff all single functions into arrays this['_' + name] = Array.isArray(options[name]) ? options[name] : [options[name]];
this['_' + func] = typeof callbacks[func] === 'function' ? [callbacks[func]] : callbacks[func];
}
else {
throw new Error("Invalid handler '" + func + "'");
}
} }
} }
@ -99,11 +102,6 @@ Zotero.Sync.Storage.Request.prototype.importCallbacks = function (request) {
} }
Zotero.Sync.Storage.Request.prototype.__defineGetter__('promise', function () {
return this._deferred.promise;
});
Zotero.Sync.Storage.Request.prototype.__defineGetter__('percentage', function () { Zotero.Sync.Storage.Request.prototype.__defineGetter__('percentage', function () {
if (this._finished) { if (this._finished) {
return 100; return 100;
@ -142,7 +140,7 @@ Zotero.Sync.Storage.Request.prototype.__defineGetter__('remaining', function ()
} }
if (!this.progressMax) { if (!this.progressMax) {
if (this.queue.type == 'upload' && this._maxSize) { if (this.type == 'upload' && this._maxSize) {
return Math.round(Zotero.Sync.Storage.compressionTracker.ratio * this._maxSize); return Math.round(Zotero.Sync.Storage.compressionTracker.ratio * this._maxSize);
} }
@ -175,72 +173,47 @@ Zotero.Sync.Storage.Request.prototype.setChannel = function (channel) {
} }
Zotero.Sync.Storage.Request.prototype.start = function () { Zotero.Sync.Storage.Request.prototype.start = Zotero.Promise.coroutine(function* () {
if (!this.queue) { Zotero.debug("Starting " + this.type + " request " + this.name);
throw ("Request " + this.name + " must be added to a queue before starting");
}
Zotero.debug("Starting " + this.queue.name + " request " + this.name);
if (this._running) { if (this._running) {
throw new Error("Request " + this.name + " already running"); throw new Error(this.type + " request " + this.name + " already running");
}
if (!this._onStart) {
throw new Error("onStart not provided -- nothing to do!");
} }
this._running = true; this._running = true;
this.queue.activeRequests++;
if (this.queue.type == 'download') { // this._onStart is an array of promises for objects of result flags, which are combined
Zotero.Sync.Storage.setItemDownloadPercentage(this.name, 0); // into a single object here
}
var self = this;
// this._onStart is an array of promises returning changesMade.
// //
// The main sync logic is triggered here. // The main sync logic is triggered here.
try {
Zotero.Promise.all([f(this) for each(f in this._onStart)]) var results = yield Zotero.Promise.all(this._onStart.map(f => f(this)));
.then(function (results) {
return {
localChanges: results.some(function (val) val && val.localChanges == true),
remoteChanges: results.some(function (val) val && val.remoteChanges == true),
conflict: results.reduce(function (prev, cur) {
return prev.conflict ? prev : cur;
}).conflict
};
})
.then(function (results) {
Zotero.debug(results);
if (results.localChanges) { var result = new Zotero.Sync.Storage.Result;
Zotero.debug("Changes were made by " + self.queue.name result.updateFromResults(results);
+ " request " + self.name);
}
else {
Zotero.debug("No changes were made by " + self.queue.name
+ " request " + self.name);
}
// This promise updates localChanges/remoteChanges on the queue Zotero.debug(this.Type + " request " + this.name + " finished");
self._deferred.resolve(results); Zotero.debug(result + "");
})
.catch(function (e) { return result;
if (self._stopping) { }
Zotero.debug("Skipping error for stopping request " + self.name); catch (e) {
return; Zotero.logError(this.Type + " request " + this.name + " failed");
throw e;
}
finally {
this._finished = true;
this._running = false;
if (this._onStop) {
this._onStop.forEach(x => x());
} }
Zotero.debug(self.queue.Type + " request " + self.name + " failed"); }
self._deferred.reject(e); });
})
// Finish the request (and in turn the queue, if this is the last request)
.finally(function () {
if (!self._finished) {
self._finish();
}
});
return this._deferred.promise;
}
Zotero.Sync.Storage.Request.prototype.isRunning = function () { Zotero.Sync.Storage.Request.prototype.isRunning = function () {
@ -263,7 +236,7 @@ Zotero.Sync.Storage.Request.prototype.isFinished = function () {
* @param {Integer} progressMax Max progress value for this request * @param {Integer} progressMax Max progress value for this request
* (usually total bytes) * (usually total bytes)
*/ */
Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress, progressMax) { Zotero.Sync.Storage.Request.prototype.onProgress = function (progress, progressMax) {
//Zotero.debug(progress + "/" + progressMax + " for request " + this.name); //Zotero.debug(progress + "/" + progressMax + " for request " + this.name);
if (!this._running) { if (!this._running) {
@ -273,10 +246,6 @@ Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress,
return; return;
} }
if (!this.channel) {
this.channel = channel;
}
// Workaround for invalid progress values (possibly related to // Workaround for invalid progress values (possibly related to
// https://bugzilla.mozilla.org/show_bug.cgi?id=451991 and fixed in 3.1) // https://bugzilla.mozilla.org/show_bug.cgi?id=451991 and fixed in 3.1)
if (progress < this.progress) { if (progress < this.progress) {
@ -292,9 +261,8 @@ Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress,
this.progress = progress; this.progress = progress;
this.progressMax = progressMax; this.progressMax = progressMax;
this.queue.updateProgress();
if (this.queue.type == 'download') { if (this.type == 'download') {
Zotero.Sync.Storage.setItemDownloadPercentage(this.name, this.percentage); Zotero.Sync.Storage.setItemDownloadPercentage(this.name, this.percentage);
} }
@ -310,59 +278,15 @@ Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress,
* Stop the request's underlying network request, if there is one * Stop the request's underlying network request, if there is one
*/ */
Zotero.Sync.Storage.Request.prototype.stop = function (force) { Zotero.Sync.Storage.Request.prototype.stop = function (force) {
if (force) {
this._forceFinish = true;
}
if (this.channel && this.channel.isPending()) { if (this.channel && this.channel.isPending()) {
this._stopping = true; this._stopping = true;
try { try {
Zotero.debug("Stopping request '" + this.name + "'"); Zotero.debug(`Stopping ${this.type} request '${this.name} '`);
this.channel.cancel(0x804b0002); // NS_BINDING_ABORTED this.channel.cancel(0x804b0002); // NS_BINDING_ABORTED
} }
catch (e) { catch (e) {
Zotero.debug(e); Zotero.debug(e, 1);
} }
} }
else {
this._finish();
}
}
/**
* Mark request as finished and notify queue that it's done
*/
Zotero.Sync.Storage.Request.prototype._finish = function () {
// If an error occurred, we wait to finish the request, since doing
// so might end the queue before the error flag has been set on the queue.
// When the queue's error handler stops the queue, it stops the request
// with stop(true) to force the finish to occur, allowing the queue's
// promise to be rejected with the error.
if (!this._forceFinish && this._deferred.promise.isRejected()) {
return;
}
Zotero.debug("Finishing " + this.queue.name + " request '" + this.name + "'");
this._finished = true;
var active = this._running;
this._running = false;
Zotero.Sync.Storage.setItemDownloadPercentage(this.name, false);
if (active) {
this.queue.activeRequests--;
}
// TEMP: mechanism for failures?
try {
this.queue.finishedRequests++;
this.queue.updateProgress();
}
catch (e) {
Zotero.debug(e, 1);
Components.utils.reportError(e);
this._deferred.reject(e);
throw e;
}
} }

47
chrome/content/zotero/xpcom/storage/storageResult.js Normal file
View File

@ -0,0 +1,47 @@
"use strict";
/**
* @property {Boolean} localChanges - Changes were made locally. For logging purposes only.
* @property {Boolean} remoteChanges - Changes were made on the server. This causes the
* last-sync time to be updated on the server (WebDAV) or retrieved (ZFS) and stored locally
* to skip additional file syncs until further server changes are made.
* @property {Boolean} syncRequired - A data sync is required to upload local changes
* @propretty {Boolean} fileSyncRequired - Another file sync is required to handle files left in
* conflict
*/
Zotero.Sync.Storage.Result = function (options = {}) {
this._props = ['localChanges', 'remoteChanges', 'syncRequired', 'fileSyncRequired'];
for (let prop of this._props) {
this[prop] = options[prop] || false;
}
}
/**
* Update the properties on this object from multiple Result objects
*
* @param {Zotero.Sync.Storage.Result[]} results
*/
Zotero.Sync.Storage.Result.prototype.updateFromResults = function (results) {
for (let prop of this._props) {
if (!this[prop]) {
for (let result of results) {
if (!(result instanceof Zotero.Sync.Storage.Result)) {
Zotero.debug(result, 1);
throw new Error("'result' is not a storage result");
}
if (result[prop]) {
this[prop] = true;
}
}
}
}
}
Zotero.Sync.Storage.Result.prototype.toString = function () {
var obj = {};
for (let prop of this._props) {
obj[prop] = this[prop] || false;
}
return JSON.stringify(obj, null, " ");
}

67
chrome/content/zotero/xpcom/storage/storageUtilities.js Normal file
View File

@ -0,0 +1,67 @@
Zotero.Sync.Storage.Utilities = {
getClassForMode: function (mode) {
switch (mode) {
case 'zfs':
return Zotero.Sync.Storage.ZFS_Module;
case 'webdav':
return Zotero.Sync.Storage.WebDAV_Module;
default:
throw new Error("Invalid storage mode '" + mode + "'");
}
},
getItemFromRequest: function (request) {
var [libraryID, key] = request.name.split('/');
return Zotero.Items.getByLibraryAndKey(libraryID, key);
},
/**
* Create zip file of attachment directory in the temp directory
*
* @param {Zotero.Sync.Storage.Request} request
* @return {Promise<Boolean>} - True if the zip file was created, false otherwise
*/
createUploadFile: Zotero.Promise.coroutine(function* (request) {
var item = this.getItemFromRequest(request);
Zotero.debug("Creating ZIP file for item " + item.libraryKey);
switch (item.attachmentLinkMode) {
case Zotero.Attachments.LINK_MODE_LINKED_FILE:
case Zotero.Attachments.LINK_MODE_LINKED_URL:
throw new Error("Upload file must be an imported snapshot or file");
}
var zipFile = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.zip');
return Zotero.File.zipDirectory(
Zotero.Attachments.getStorageDirectory(item).path,
zipFile,
{
onStopRequest: function (req, context, status) {
var zipFileName = OS.Path.basename(zipFile);
var originalSize = 0;
for (let entry of context.entries) {
let zipEntry = context.zipWriter.getEntry(entry.name);
if (!zipEntry) {
Zotero.logError("ZIP entry '" + entry.name + "' not found for "
+ "request '" + request.name + "'")
continue;
}
originalSize += zipEntry.realSize;
}
Zotero.debug("Zip of " + zipFileName + " finished with status " + status
+ " (original " + Math.round(originalSize / 1024) + "KB, "
+ "compressed " + Math.round(context.zipWriter.file.fileSize / 1024) + "KB, "
+ Math.round(
((originalSize - context.zipWriter.file.fileSize) / originalSize) * 100
) + "% reduction)");
}
}
);
})
}

79
chrome/content/zotero/xpcom/storage/streamListener.js
View File

@ -30,10 +30,9 @@
* Possible properties of data object: * Possible properties of data object:
* - onStart: f(request) * - onStart: f(request)
* - onProgress: f(request, progress, progressMax) * - onProgress: f(request, progress, progressMax)
* - onStop: f(request, status, response, data) * - onStop: f(request, status, response)
* - onCancel: f(request, status, data) * - onCancel: f(request, status)
* - streams: array of streams to close on completion * - streams: array of streams to close on completion
* - Other values to pass to onStop()
*/ */
Zotero.Sync.Storage.StreamListener = function (data) { Zotero.Sync.Storage.StreamListener = function (data) {
this._data = data; this._data = data;
@ -110,17 +109,15 @@ Zotero.Sync.Storage.StreamListener.prototype = {
}, },
onStateChange: function (wp, request, stateFlags, status) { onStateChange: function (wp, request, stateFlags, status) {
Zotero.debug("onStateChange"); Zotero.debug("onStateChange with " + stateFlags);
Zotero.debug(stateFlags);
Zotero.debug(status);
if ((stateFlags & Components.interfaces.nsIWebProgressListener.STATE_START) if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_REQUEST) {
&& (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_NETWORK)) { if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_START) {
this._onStart(request); this._onStart(request);
} }
else if ((stateFlags & Components.interfaces.nsIWebProgressListener.STATE_STOP) else if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_STOP) {
&& (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_NETWORK)) { this._onStop(request, status);
this._onStop(request, status); }
} }
}, },
@ -148,18 +145,38 @@ Zotero.Sync.Storage.StreamListener.prototype = {
}, },
// nsIChannelEventSink // nsIChannelEventSink
onChannelRedirect: function (oldChannel, newChannel, flags) { //
// If this._data.onChannelRedirect exists, it should return a promise resolving to true to
// follow the redirect or false to cancel it
onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) {
Zotero.debug('onChannelRedirect'); Zotero.debug('onChannelRedirect');
if (this._data && this._data.onChannelRedirect) {
let result = yield this._data.onChannelRedirect(oldChannel, newChannel, flags);
if (!result) {
oldChannel.cancel(Components.results.NS_BINDING_ABORTED);
newChannel.cancel(Components.results.NS_BINDING_ABORTED);
return false;
}
}
// if redirecting, store the new channel // if redirecting, store the new channel
this._channel = newChannel; this._channel = newChannel;
}, }),
asyncOnChannelRedirect: function (oldChan, newChan, flags, redirectCallback) { asyncOnChannelRedirect: function (oldChan, newChan, flags, redirectCallback) {
Zotero.debug('asyncOnRedirect'); Zotero.debug('asyncOnRedirect');
this.onChannelRedirect(oldChan, newChan, flags); this.onChannelRedirect(oldChan, newChan, flags)
redirectCallback.onRedirectVerifyCallback(0); .then(function (result) {
redirectCallback.onRedirectVerifyCallback(
result ? Components.results.NS_SUCCEEDED : Components.results.NS_FAILED
);
})
.catch(function (e) {
Zotero.logError(e);
redirectCallback.onRedirectVerifyCallback(Components.results.NS_FAILED);
});
}, },
// nsIHttpEventSink // nsIHttpEventSink
@ -177,8 +194,7 @@ Zotero.Sync.Storage.StreamListener.prototype = {
_onStart: function (request) { _onStart: function (request) {
Zotero.debug('Starting request'); Zotero.debug('Starting request');
if (this._data && this._data.onStart) { if (this._data && this._data.onStart) {
var data = this._getPassData(); this._data.onStart(request);
this._data.onStart(request, data);
} }
}, },
@ -189,7 +205,6 @@ Zotero.Sync.Storage.StreamListener.prototype = {
}, },
_onStop: function (request, status) { _onStop: function (request, status) {
Zotero.debug('Request ended with status ' + status);
var cancelled = status == 0x804b0002; // NS_BINDING_ABORTED var cancelled = status == 0x804b0002; // NS_BINDING_ABORTED
if (!cancelled && status == 0 && request instanceof Components.interfaces.nsIHttpChannel) { if (!cancelled && status == 0 && request instanceof Components.interfaces.nsIHttpChannel) {
@ -201,9 +216,11 @@ Zotero.Sync.Storage.StreamListener.prototype = {
Zotero.debug("Request responseStatus not available", 1); Zotero.debug("Request responseStatus not available", 1);
status = 0; status = 0;
} }
Zotero.debug('Request ended with status code ' + status);
request.QueryInterface(Components.interfaces.nsIRequest); request.QueryInterface(Components.interfaces.nsIRequest);
} }
else { else {
Zotero.debug('Request ended with status ' + status);
status = 0; status = 0;
} }
@ -213,38 +230,20 @@ Zotero.Sync.Storage.StreamListener.prototype = {
} }
} }
var data = this._getPassData();
if (cancelled) { if (cancelled) {
if (this._data.onCancel) { if (this._data.onCancel) {
this._data.onCancel(request, status, data); this._data.onCancel(request, status);
} }
} }
else { else {
if (this._data.onStop) { if (this._data.onStop) {
this._data.onStop(request, status, this._response, data); this._data.onStop(request, status, this._response);
} }
} }
this._channel = null; this._channel = null;
}, },
_getPassData: function () {
// Make copy of data without callbacks to pass along
var passData = {};
for (var i in this._data) {
switch (i) {
case "onStart":
case "onProgress":
case "onStop":
case "onCancel":
continue;
}
passData[i] = this._data[i];
}
return passData;
},
// nsIInterfaceRequestor // nsIInterfaceRequestor
getInterface: function (iid) { getInterface: function (iid) {
try { try {

1579
chrome/content/zotero/xpcom/storage/webdav.js
View File

File diff suppressed because it is too large Load Diff

2003
chrome/content/zotero/xpcom/storage/zfs.js
View File

File diff suppressed because it is too large Load Diff

37
chrome/content/zotero/xpcom/sync.js
View File

@ -1488,43 +1488,6 @@ Zotero.Sync.Server = new function () {
Zotero.BufferedInputListener = function (callback) {
this._callback = callback;
}
Zotero.BufferedInputListener.prototype = {
binaryInputStream: null,
size: 0,
data: '',
onStartRequest: function(request, context) {},
onStopRequest: function(request, context, status) {
this.binaryInputStream.close();
delete this.binaryInputStream;
this._callback(this.data);
},
onDataAvailable: function(request, context, inputStream, offset, count) {
this.size += count;
this.binaryInputStream = Components.classes["@mozilla.org/binaryinputstream;1"]
.createInstance(Components.interfaces.nsIBinaryInputStream)
this.binaryInputStream.setInputStream(inputStream);
this.data += this.binaryInputStream.readBytes(this.binaryInputStream.available());
},
QueryInterface: function (iid) {
if (iid.equals(Components.interfaces.nsISupports)
|| iid.equals(Components.interfaces.nsIStreamListener)) {
return this;
}
throw Components.results.NS_ERROR_NO_INTERFACE;
}
}
Zotero.Sync.Server.Data = new function() { Zotero.Sync.Server.Data = new function() {
var _noMergeTypes = ['search']; var _noMergeTypes = ['search'];

105
chrome/content/zotero/xpcom/sync/syncAPIClient.js
View File

@ -28,14 +28,15 @@ if (!Zotero.Sync) {
} }
Zotero.Sync.APIClient = function (options) { Zotero.Sync.APIClient = function (options) {
this.baseURL = options.baseURL; if (!options.baseURL) throw new Error("baseURL not set");
this.apiKey = options.apiKey; if (!options.apiVersion) throw new Error("apiVersion not set");
this.concurrentCaller = options.concurrentCaller; if (!options.apiKey) throw new Error("apiKey not set");
if (!options.caller) throw new Error("caller not set");
if (options.apiVersion == undefined) { this.baseURL = options.baseURL;
throw new Error("options.apiVersion not set");
}
this.apiVersion = options.apiVersion; this.apiVersion = options.apiVersion;
this.apiKey = options.apiKey;
this.caller = options.caller;
} }
Zotero.Sync.APIClient.prototype = { Zotero.Sync.APIClient.prototype = {
@ -44,7 +45,7 @@ Zotero.Sync.APIClient.prototype = {
getKeyInfo: Zotero.Promise.coroutine(function* () { getKeyInfo: Zotero.Promise.coroutine(function* () {
var uri = this.baseURL + "keys/" + this.apiKey; var uri = this.baseURL + "keys/" + this.apiKey;
var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 404] }); var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 404] });
if (xmlhttp.status == 404) { if (xmlhttp.status == 404) {
return false; return false;
} }
@ -63,7 +64,7 @@ Zotero.Sync.APIClient.prototype = {
if (!userID) throw new Error("User ID not provided"); if (!userID) throw new Error("User ID not provided");
var uri = this.baseURL + "users/" + userID + "/groups?format=versions"; var uri = this.baseURL + "users/" + userID + "/groups?format=versions";
var xmlhttp = yield this._makeRequest("GET", uri); var xmlhttp = yield this.makeRequest("GET", uri);
return this._parseJSON(xmlhttp.responseText); return this._parseJSON(xmlhttp.responseText);
}), }),
@ -76,7 +77,7 @@ Zotero.Sync.APIClient.prototype = {
if (!groupID) throw new Error("Group ID not provided"); if (!groupID) throw new Error("Group ID not provided");
var uri = this.baseURL + "groups/" + groupID; var uri = this.baseURL + "groups/" + groupID;
var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 404] }); var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 404] });
if (xmlhttp.status == 404) { if (xmlhttp.status == 404) {
return false; return false;
} }
@ -93,7 +94,7 @@ Zotero.Sync.APIClient.prototype = {
if (since) { if (since) {
params.since = since; params.since = since;
} }
var uri = this._buildRequestURI(params); var uri = this.buildRequestURI(params);
var options = { var options = {
successCodes: [200, 304] successCodes: [200, 304]
}; };
@ -102,7 +103,7 @@ Zotero.Sync.APIClient.prototype = {
"If-Modified-Since-Version": since "If-Modified-Since-Version": since
}; };
} }
var xmlhttp = yield this._makeRequest("GET", uri, options); var xmlhttp = yield this.makeRequest("GET", uri, options);
if (xmlhttp.status == 304) { if (xmlhttp.status == 304) {
return false; return false;
} }
@ -128,8 +129,8 @@ Zotero.Sync.APIClient.prototype = {
libraryTypeID: libraryTypeID, libraryTypeID: libraryTypeID,
since: since || 0 since: since || 0
}; };
var uri = this._buildRequestURI(params); var uri = this.buildRequestURI(params);
var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 409] }); var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 409] });
if (xmlhttp.status == 409) { if (xmlhttp.status == 409) {
Zotero.debug(`'since' value '${since}' is earlier than the beginning of the delete log`); Zotero.debug(`'since' value '${since}' is earlier than the beginning of the delete log`);
return false; return false;
@ -154,7 +155,7 @@ Zotero.Sync.APIClient.prototype = {
* @param {String} libraryType 'user' or 'group' * @param {String} libraryType 'user' or 'group'
* @param {Integer} libraryTypeID userID or groupID * @param {Integer} libraryTypeID userID or groupID
* @param {String} objectType 'item', 'collection', 'search' * @param {String} objectType 'item', 'collection', 'search'
* @param {Object} queryParams Query parameters (see _buildRequestURI()) * @param {Object} queryParams Query parameters (see buildRequestURI())
* @return {Promise<Object>|FALSE} Object with 'libraryVersion' and 'results' * @return {Promise<Object>|FALSE} Object with 'libraryVersion' and 'results'
*/ */
getVersions: Zotero.Promise.coroutine(function* (libraryType, libraryTypeID, objectType, queryParams, libraryVersion) { getVersions: Zotero.Promise.coroutine(function* (libraryType, libraryTypeID, objectType, queryParams, libraryVersion) {
@ -176,7 +177,7 @@ Zotero.Sync.APIClient.prototype = {
} }
// TODO: Use pagination // TODO: Use pagination
var uri = this._buildRequestURI(params); var uri = this.buildRequestURI(params);
var options = { var options = {
successCodes: [200, 304] successCodes: [200, 304]
@ -186,7 +187,7 @@ Zotero.Sync.APIClient.prototype = {
"If-Modified-Since-Version": libraryVersion "If-Modified-Since-Version": libraryVersion
}; };
} }
var xmlhttp = yield this._makeRequest("GET", uri, options); var xmlhttp = yield this.makeRequest("GET", uri, options);
if (xmlhttp.status == 304) { if (xmlhttp.status == 304) {
return false; return false;
} }
@ -256,10 +257,10 @@ Zotero.Sync.APIClient.prototype = {
if (objectType == 'item') { if (objectType == 'item') {
params.includeTrashed = 1; params.includeTrashed = 1;
} }
var uri = this._buildRequestURI(params); var uri = this.buildRequestURI(params);
return [ return [
this._makeRequest("GET", uri) this.makeRequest("GET", uri)
.then(function (xmlhttp) { .then(function (xmlhttp) {
return this._parseJSON(xmlhttp.responseText) return this._parseJSON(xmlhttp.responseText)
}.bind(this)) }.bind(this))
@ -294,9 +295,9 @@ Zotero.Sync.APIClient.prototype = {
libraryType: libraryType, libraryType: libraryType,
libraryTypeID: libraryTypeID libraryTypeID: libraryTypeID
}; };
var uri = this._buildRequestURI(params); var uri = this.buildRequestURI(params);
var xmlhttp = yield this._makeRequest(method, uri, { var xmlhttp = yield this.makeRequest(method, uri, {
headers: { headers: {
"If-Unmodified-Since-Version": version "If-Unmodified-Since-Version": version
}, },
@ -319,7 +320,7 @@ Zotero.Sync.APIClient.prototype = {
}), }),
_buildRequestURI: function (params) { buildRequestURI: function (params) {
var uri = this.baseURL; var uri = this.baseURL;
switch (params.libraryType) { switch (params.libraryType) {
@ -332,6 +333,10 @@ Zotero.Sync.APIClient.prototype = {
break; break;
} }
if (params.target === undefined) {
throw new Error("'target' not provided");
}
uri += "/" + params.target; uri += "/" + params.target;
if (params.objectKey) { if (params.objectKey) {
@ -382,30 +387,33 @@ Zotero.Sync.APIClient.prototype = {
}, },
_makeRequest: function (method, uri, options) { getHeaders: function (headers = {}) {
if (!options) { headers["Zotero-API-Version"] = this.apiVersion;
options = {}; if (this.apiKey) {
headers["Zotero-API-Key"] = this.apiKey;
} }
if (!options.headers) { return headers;
options.headers = {}; },
}
options.headers["Zotero-API-Version"] = this.apiVersion;
makeRequest: function (method, uri, options = {}) {
options.headers = this.getHeaders(options.headers);
options.dontCache = true; options.dontCache = true;
options.foreground = !options.background; options.foreground = !options.background;
options.responseType = options.responseType || 'text'; options.responseType = options.responseType || 'text';
if (this.apiKey) { return this.caller.start(Zotero.Promise.coroutine(function* () {
options.headers.Authorization = "Bearer " + this.apiKey; try {
} var xmlhttp = yield Zotero.HTTP.request(method, uri, options);
var self = this; this._checkBackoff(xmlhttp);
return this.concurrentCaller.fcall(function () { return xmlhttp;
return Zotero.HTTP.request(method, uri, options) }
.catch(function (e) { catch (e) {
if (e instanceof Zotero.HTTP.UnexpectedStatusException) { /*if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
self._checkResponse(e.xmlhttp); this._checkRetry(e.xmlhttp);
} }*/
throw e; throw e;
}); }
}); }.bind(this)));
}, },
@ -422,21 +430,6 @@ Zotero.Sync.APIClient.prototype = {
}, },
_checkResponse: function (xmlhttp) {
this._checkBackoff(xmlhttp);
this._checkAuth(xmlhttp);
},
_checkAuth: function (xmlhttp) {
if (xmlhttp.status == 403) {
var e = new Zotero.Error(Zotero.getString('sync.error.invalidLogin'), "INVALID_SYNC_LOGIN");
e.fatal = true;
throw e;
}
},
_checkBackoff: function (xmlhttp) { _checkBackoff: function (xmlhttp) {
var backoff = xmlhttp.getResponseHeader("Backoff"); var backoff = xmlhttp.getResponseHeader("Backoff");
if (backoff) { if (backoff) {
@ -444,7 +437,7 @@ Zotero.Sync.APIClient.prototype = {
if (backoff > 3600) { if (backoff > 3600) {
// TODO: Update status? // TODO: Update status?
this.concurrentCaller.pause(backoff * 1000); this.caller.pause(backoff * 1000);
} }
} }
} }

70
chrome/content/zotero/xpcom/sync/syncEngine.js
View File

@ -76,7 +76,13 @@ Zotero.Sync.Data.Engine = function (options) {
onError: this.onError onError: this.onError
} }
this.syncCachePromise = Zotero.Promise.resolve().bind(this); Components.utils.import("resource://zotero/concurrentCaller.js");
this.syncCacheProcessor = new ConcurrentCaller({
id: "Sync Cache Processor",
numConcurrent: 1,
logger: Zotero.debug,
stopOnError: this.stopOnError
});
}; };
Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_SUCCESS = 1; Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_SUCCESS = 1;
@ -167,12 +173,8 @@ Zotero.Sync.Data.Engine.prototype.start = Zotero.Promise.coroutine(function* ()
} }
} }
// TEMP: make more reliable Zotero.debug("Waiting for sync cache to be processed");
while (this.syncCachePromise.isPending()) { yield this.syncCacheProcessor.wait();
Zotero.debug("Waiting for sync cache to be processed");
yield this.syncCachePromise;
yield Zotero.Promise.delay(50);
}
yield Zotero.Libraries.updateLastSyncTime(this.libraryID); yield Zotero.Libraries.updateLastSyncTime(this.libraryID);
@ -286,12 +288,7 @@ Zotero.Sync.Data.Engine.prototype._startDownload = Zotero.Promise.coroutine(func
} }
// Wait for sync process to clear // Wait for sync process to clear
// TEMP: make more reliable yield this.syncCacheProcessor.wait();
while (this.syncCachePromise.isPending()) {
Zotero.debug("Waiting for sync cache to be processed");
yield this.syncCachePromise;
yield Zotero.Promise.delay(50);
}
// //
// Get deleted objects // Get deleted objects
@ -671,7 +668,8 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi
if (state == 'successful') { if (state == 'successful') {
// Update local object with saved data if necessary // Update local object with saved data if necessary
yield obj.fromJSON(current.data); yield obj.loadAllData();
obj.fromJSON(current.data);
toSave.push(obj); toSave.push(obj);
toCache.push(current); toCache.push(current);
} }
@ -701,8 +699,11 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi
// Handle failed objects // Handle failed objects
for (let index in json.results.failed) { for (let index in json.results.failed) {
let e = json.results.failed[index]; let { code, message } = json.results.failed[index];
Zotero.logError(e.message); e = new Error(message);
e.name = "ZoteroUploadObjectError";
e.code = code;
Zotero.logError(e);
// This shouldn't happen, because the upload request includes a library // This shouldn't happen, because the upload request includes a library
// version and should prevent an outdated upload before the object version is // version and should prevent an outdated upload before the object version is
@ -711,12 +712,11 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi
return this.UPLOAD_RESULT_OBJECT_CONFLICT; return this.UPLOAD_RESULT_OBJECT_CONFLICT;
} }
if (this.stopOnError) {
Zotero.debug("WE FAILED!!!");
throw new Error(e.message);
}
if (this.onError) { if (this.onError) {
this.onError(e.message); this.onError(e);
}
if (this.stopOnError) {
throw new Error(e);
} }
batch[index].tries++; batch[index].tries++;
// Mark 400 errors as permanently failed // Mark 400 errors as permanently failed
@ -990,7 +990,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
this._failedCheck(); this._failedCheck();
let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
let ObjectType = objectType[0].toUpperCase() + objectType.substr(1); let ObjectType = Zotero.Utilities.capitalize(objectType);
// TODO: localize // TODO: localize
this.setStatus("Updating " + objectTypePlural + " in " + this.libraryName); this.setStatus("Updating " + objectTypePlural + " in " + this.libraryName);
@ -1037,8 +1037,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
let cacheVersions = yield Zotero.Sync.Data.Local.getLatestCacheObjectVersions( let cacheVersions = yield Zotero.Sync.Data.Local.getLatestCacheObjectVersions(
this.libraryID, objectType this.libraryID, objectType
); );
// Queue objects that are out of date or don't exist locally and aren't up-to-date // Queue objects that are out of date or don't exist locally
// in the cache
for (let key in results.versions) { for (let key in results.versions) {
let version = results.versions[key]; let version = results.versions[key];
let obj = yield objectsClass.getByLibraryAndKeyAsync(this.libraryID, key, { let obj = yield objectsClass.getByLibraryAndKeyAsync(this.libraryID, key, {
@ -1060,12 +1059,12 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
} }
if (obj) { if (obj) {
Zotero.debug(Zotero.Utilities.capitalize(objectType) + " " + obj.libraryKey Zotero.debug(ObjectType + " " + obj.libraryKey
+ " is older than version in sync cache"); + " is older than version in sync cache");
} }
else { else {
Zotero.debug(Zotero.Utilities.capitalize(objectType) + " " Zotero.debug(ObjectType + " " + this.libraryID + "/" + key
+ this.libraryID + "/" + key + " in sync cache not found locally"); + " in sync cache not found locally");
} }
toDownload.push(key); toDownload.push(key);
@ -1127,7 +1126,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
break; break;
} }
yield this.syncCachePromise; yield this.syncCacheProcessor.wait();
yield Zotero.Libraries.setVersion(this.libraryID, lastLibraryVersion); yield Zotero.Libraries.setVersion(this.libraryID, lastLibraryVersion);
@ -1145,20 +1144,19 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
* @param {String} objectType * @param {String} objectType
*/ */
Zotero.Sync.Data.Engine.prototype._processCache = function (objectType) { Zotero.Sync.Data.Engine.prototype._processCache = function (objectType) {
var self = this; this.syncCacheProcessor.start(function () {
this.syncCachePromise = this.syncCachePromise.then(function () { this._failedCheck();
self._failedCheck();
return Zotero.Sync.Data.Local.processSyncCacheForObjectType( return Zotero.Sync.Data.Local.processSyncCacheForObjectType(
self.libraryID, objectType, self.options this.libraryID, objectType, this.options
) )
.catch(function (e) { .catch(function (e) {
Zotero.logError(e); Zotero.logError(e);
if (self.stopOnError) { if (this.stopOnError) {
Zotero.debug("WE FAILED!!!"); Zotero.debug("WE FAILED!!!");
self.failed = e; this.failed = e;
} }
}); }.bind(this));
}) }.bind(this))
} }

56
chrome/content/zotero/xpcom/sync/syncEventListeners.js
View File

@ -39,10 +39,9 @@ Zotero.Sync.EventListeners.ChangeListener = new function () {
var syncSQL = "REPLACE INTO syncDeleteLog (syncObjectTypeID, libraryID, key, synced) " var syncSQL = "REPLACE INTO syncDeleteLog (syncObjectTypeID, libraryID, key, synced) "
+ "VALUES (?, ?, ?, 0)"; + "VALUES (?, ?, ?, 0)";
var storageSQL = "REPLACE INTO storageDeleteLog VALUES (?, ?, 0)";
if (type == 'item' && Zotero.Sync.Storage.WebDAV.includeUserFiles) { var storageForLibrary = {};
var storageSQL = "REPLACE INTO storageDeleteLog VALUES (?, ?, 0)";
}
return Zotero.DB.executeTransaction(function* () { return Zotero.DB.executeTransaction(function* () {
for (let i = 0; i < ids.length; i++) { for (let i = 0; i < ids.length; i++) {
@ -74,18 +73,25 @@ Zotero.Sync.EventListeners.ChangeListener = new function () {
key key
] ]
); );
if (storageSQL && oldItem.itemType == 'attachment' &&
[ if (type == 'item') {
Zotero.Attachments.LINK_MODE_IMPORTED_FILE, if (storageForLibrary[libraryID] === undefined) {
Zotero.Attachments.LINK_MODE_IMPORTED_URL storageForLibrary[libraryID] =
].indexOf(oldItem.linkMode) != -1) { Zotero.Sync.Storage.Local.getModeForLibrary(libraryID) == 'webdav';
yield Zotero.DB.queryAsync( }
storageSQL, if (storageForLibrary[libraryID] && oldItem.itemType == 'attachment' &&
[ [
libraryID, Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
key Zotero.Attachments.LINK_MODE_IMPORTED_URL
] ].indexOf(oldItem.linkMode) != -1) {
); yield Zotero.DB.queryAsync(
storageSQL,
[
libraryID,
key
]
);
}
} }
} }
}); });
@ -215,3 +221,23 @@ Zotero.Sync.EventListeners.progressListener = {
} }
}; };
Zotero.Sync.EventListeners.StorageFileOpenListener = {
init: function () {
Zotero.Notifier.registerObserver(this, ['file'], 'storageFileOpen');
},
notify: function (event, type, ids, extraData) {
if (event == 'open' && type == 'file') {
let timestamp = new Date().getTime();
for (let i = 0; i < ids.length; i++) {
Zotero.Sync.Storage.Local.uploadCheckFiles.push({
itemID: ids[i],
timestamp: timestamp
});
}
}
}
}

333
chrome/content/zotero/xpcom/sync/syncLocal.js
View File

@ -28,6 +28,8 @@ if (!Zotero.Sync.Data) {
} }
Zotero.Sync.Data.Local = { Zotero.Sync.Data.Local = {
_loginManagerHost: 'https://api.zotero.org',
_loginManagerRealm: 'Zotero Web API',
_lastSyncTime: null, _lastSyncTime: null,
_lastClassicSyncTime: null, _lastClassicSyncTime: null,
@ -39,6 +41,71 @@ Zotero.Sync.Data.Local = {
}), }),
getAPIKey: function () {
var apiKey = Zotero.Prefs.get('devAPIKey');
if (apiKey) {
return apiKey;
}
var loginManager = Components.classes["@mozilla.org/login-manager;1"]
.getService(Components.interfaces.nsILoginManager);
var logins = loginManager.findLogins(
{}, this._loginManagerHost, null, this._loginManagerRealm
);
// Get API from returned array of nsILoginInfo objects
if (logins.length) {
return logins[0].password;
}
if (!apiKey) {
let username = Zotero.Prefs.get('sync.server.username');
if (username) {
let password = Zotero.Sync.Data.Local.getLegacyPassword(username);
if (!password) {
return false;
}
throw new Error("Unimplemented");
// Get API key from server
// Store API key
// Remove old logins and username pref
}
}
return apiKey;
},
setAPIKey: function (apiKey) {
var loginManager = Components.classes["@mozilla.org/login-manager;1"]
.getService(Components.interfaces.nsILoginManager);
var nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1",
Components.interfaces.nsILoginInfo, "init");
var loginInfo = new nsLoginInfo(
this._loginManagerHost,
null,
this._loginManagerRealm,
'API Key',
apiKey,
"",
""
);
loginManager.addLogin(loginInfo);
},
getLegacyPassword: function (username) {
var loginManager = Components.classes["@mozilla.org/login-manager;1"]
.getService(Components.interfaces.nsILoginManager);
var logins = loginManager.findLogins({}, "chrome://zotero", "Zotero Storage Server", null);
// Find user from returned array of nsILoginInfo objects
for (let login of logins) {
if (login.username == username) {
return login.password;
}
}
return false;
},
getLastSyncTime: function () { getLastSyncTime: function () {
if (_lastSyncTime === null) { if (_lastSyncTime === null) {
throw new Error("Last sync time not yet loaded"); throw new Error("Last sync time not yet loaded");
@ -86,7 +153,7 @@ Zotero.Sync.Data.Local = {
var sql = "SELECT " + objectsClass.idColumn + " FROM " + objectsClass.table var sql = "SELECT " + objectsClass.idColumn + " FROM " + objectsClass.table
+ " WHERE libraryID=? AND synced=0"; + " WHERE libraryID=? AND synced=0";
// RETRIEVE PARENT DOWN? EVEN POSSIBLE? // TODO: RETRIEVE PARENT DOWN? EVEN POSSIBLE?
// items via parent // items via parent
// collections via getDescendents? // collections via getDescendents?
@ -154,6 +221,35 @@ Zotero.Sync.Data.Local = {
}), }),
getCacheObjects: Zotero.Promise.coroutine(function* (objectType, libraryID, keyVersionPairs) {
if (!keyVersionPairs.length) return [];
var sql = "SELECT data FROM syncCache SC JOIN (SELECT "
+ keyVersionPairs.map(function (pair) {
Zotero.DataObjectUtilities.checkKey(pair[0]);
return "'" + pair[0] + "' AS key, " + parseInt(pair[1]) + " AS version";
}).join(" UNION SELECT ")
+ ") AS pairs ON (pairs.key=SC.key AND pairs.version=SC.version) "
+ "WHERE libraryID=? AND "
+ "syncObjectTypeID IN (SELECT syncObjectTypeID FROM syncObjectTypes WHERE name=?)";
var rows = yield Zotero.DB.columnQueryAsync(sql, [libraryID, objectType]);
return rows.map(row => JSON.parse(row));
}),
saveCacheObject: Zotero.Promise.coroutine(function* (objectType, libraryID, json) {
json = this._checkCacheJSON(json);
Zotero.debug("Saving to sync cache:");
Zotero.debug(json);
var syncObjectTypeID = Zotero.Sync.Data.Utilities.getSyncObjectTypeID(objectType);
var sql = "INSERT OR REPLACE INTO syncCache "
+ "(libraryID, key, syncObjectTypeID, version, data) VALUES (?, ?, ?, ?, ?)";
var params = [libraryID, json.key, syncObjectTypeID, json.version, JSON.stringify(json)];
return Zotero.DB.queryAsync(sql, params);
}),
saveCacheObjects: Zotero.Promise.coroutine(function* (objectType, libraryID, jsonArray) { saveCacheObjects: Zotero.Promise.coroutine(function* (objectType, libraryID, jsonArray) {
if (!Array.isArray(jsonArray)) { if (!Array.isArray(jsonArray)) {
throw new Error("'json' must be an array"); throw new Error("'json' must be an array");
@ -165,20 +261,7 @@ Zotero.Sync.Data.Local = {
return; return;
} }
jsonArray = jsonArray.map(o => { jsonArray = jsonArray.map(json => this._checkCacheJSON(json));
if (o.key === undefined) {
throw new Error("Missing 'key' property in JSON");
}
if (o.version === undefined) {
throw new Error("Missing 'version' property in JSON");
}
// If direct data object passed, wrap in fake response object
return o.data === undefined ? {
key: o.key,
version: o.version,
data: o
} : o;
});
Zotero.debug("Saving to sync cache:"); Zotero.debug("Saving to sync cache:");
Zotero.debug(jsonArray); Zotero.debug(jsonArray);
@ -206,6 +289,22 @@ Zotero.Sync.Data.Local = {
}), }),
_checkCacheJSON: function (json) {
if (json.key === undefined) {
throw new Error("Missing 'key' property in JSON");
}
if (json.version === undefined) {
throw new Error("Missing 'version' property in JSON");
}
// If direct data object passed, wrap in fake response object
return json.data === undefined ? {
key: json.key,
version: json.version,
data: json
} : json;
},
processSyncCache: Zotero.Promise.coroutine(function* (libraryID, options) { processSyncCache: Zotero.Promise.coroutine(function* (libraryID, options) {
for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(libraryID)) { for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(libraryID)) {
yield this.processSyncCacheForObjectType(libraryID, objectType, options); yield this.processSyncCacheForObjectType(libraryID, objectType, options);
@ -213,8 +312,7 @@ Zotero.Sync.Data.Local = {
}), }),
processSyncCacheForObjectType: Zotero.Promise.coroutine(function* (libraryID, objectType, options) { processSyncCacheForObjectType: Zotero.Promise.coroutine(function* (libraryID, objectType, options = {}) {
options = options || {};
var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType); var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType); var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
var ObjectType = Zotero.Utilities.capitalize(objectType); var ObjectType = Zotero.Utilities.capitalize(objectType);
@ -227,7 +325,6 @@ Zotero.Sync.Data.Local = {
var numSkipped = 0; var numSkipped = 0;
var data = yield this._getUnwrittenData(libraryID, objectType); var data = yield this._getUnwrittenData(libraryID, objectType);
if (!data.length) { if (!data.length) {
Zotero.debug("No unwritten " + objectTypePlural + " in sync cache"); Zotero.debug("No unwritten " + objectTypePlural + " in sync cache");
return; return;
@ -260,9 +357,9 @@ Zotero.Sync.Data.Local = {
for (let i = 0; i < chunk.length; i++) { for (let i = 0; i < chunk.length; i++) {
let json = chunk[i]; let json = chunk[i];
let jsonData = json.data; let jsonData = json.data;
let isNewObject;
let objectKey = json.key; let objectKey = json.key;
Zotero.debug(`Processing ${objectType} ${libraryID}/${objectKey}`);
Zotero.debug(json); Zotero.debug(json);
if (!jsonData) { if (!jsonData) {
@ -302,26 +399,22 @@ Zotero.Sync.Data.Local = {
}*/ }*/
} }
let isNewObject = false;
let skipCache = false;
let obj = yield objectsClass.getByLibraryAndKeyAsync( let obj = yield objectsClass.getByLibraryAndKeyAsync(
libraryID, objectKey, { noCache: true } libraryID, objectKey, { noCache: true }
); );
if (obj) { if (obj) {
Zotero.debug("Matching local " + objectType + " exists", 4); Zotero.debug("Matching local " + objectType + " exists", 4);
isNewObject = false;
// Local object has not been modified since last sync // Local object has been modified since last sync
if (obj.synced) { if (!obj.synced) {
// Overwrite local below
}
else {
Zotero.debug("Local " + objectType + " " + obj.libraryKey Zotero.debug("Local " + objectType + " " + obj.libraryKey
+ " has been modified since last sync", 4); + " has been modified since last sync", 4);
let cachedJSON = yield this.getCacheObject( let cachedJSON = yield this.getCacheObject(
objectType, obj.libraryID, obj.key, obj.version objectType, obj.libraryID, obj.key, obj.version
); );
Zotero.debug("GOT CACHED");
Zotero.debug(cachedJSON);
let jsonDataLocal = yield obj.toJSON(); let jsonDataLocal = yield obj.toJSON();
@ -333,42 +426,51 @@ Zotero.Sync.Data.Local = {
['dateAdded', 'dateModified'] ['dateAdded', 'dateModified']
); );
// If no changes, update local version and keep as unsynced // If no changes, update local version number and mark as synced
if (!result.changes.length && !result.conflicts.length) { if (!result.changes.length && !result.conflicts.length) {
Zotero.debug("No remote changes to apply to local " + objectType Zotero.debug("No remote changes to apply to local "
+ " " + obj.libraryKey); + objectType + " " + obj.libraryKey);
yield obj.updateVersion(json.version); obj.version = json.version;
obj.synced = true;
yield obj.save();
continue;
}
if (result.conflicts.length) {
if (objectType != 'item') {
throw new Error(`Unexpected conflict on ${objectType} object`);
}
Zotero.debug("Conflict!");
conflicts.push({
left: jsonDataLocal,
right: jsonData,
changes: result.changes,
conflicts: result.conflicts
});
continue; continue;
} }
// If no conflicts, apply remote changes automatically // If no conflicts, apply remote changes automatically
if (!result.conflicts.length) { Zotero.debug(`Applying remote changes to ${objectType} `
Zotero.DataObjectUtilities.applyChanges( + obj.libraryKey);
jsonData, result.changes Zotero.debug(result.changes);
); Zotero.DataObjectUtilities.applyChanges(
let saved = yield this._saveObjectFromJSON(obj, jsonData, options); jsonDataLocal, result.changes
if (saved) numSaved++; );
continue; // Transfer properties that aren't in the changeset
} ['version', 'dateAdded', 'dateModified'].forEach(x => {
if (jsonDataLocal[x] !== jsonData[x]) {
if (objectType != 'item') { Zotero.debug(`Applying remote '${x}' value`);
throw new Error(`Unexpected conflict on ${objectType} object`); }
} jsonDataLocal[x] = jsonData[x];
})
conflicts.push({ jsonData = jsonDataLocal;
left: jsonDataLocal,
right: jsonData,
changes: result.changes,
conflicts: result.conflicts
});
continue;
} }
let saved = yield this._saveObjectFromJSON(obj, jsonData, options);
if (saved) numSaved++;
} }
// Object doesn't exist locally // Object doesn't exist locally
else { else {
Zotero.debug(ObjectType + " doesn't exist locally");
isNewObject = true; isNewObject = true;
// Check if object has been deleted locally // Check if object has been deleted locally
@ -376,6 +478,8 @@ Zotero.Sync.Data.Local = {
objectType, libraryID, objectKey objectType, libraryID, objectKey
); );
if (dateDeleted) { if (dateDeleted) {
Zotero.debug(ObjectType + " was deleted locally");
switch (objectType) { switch (objectType) {
case 'item': case 'item':
conflicts.push({ conflicts.push({
@ -410,24 +514,30 @@ Zotero.Sync.Data.Local = {
obj.key = objectKey; obj.key = objectKey;
yield obj.loadPrimaryData(); yield obj.loadPrimaryData();
let saved = yield this._saveObjectFromJSON(obj, jsonData, options, { // Don't cache new items immediately, which skips reloading after save
// Don't cache new items immediately, which skips reloading after save skipCache = true;
skipCache: true }
});
if (saved) numSaved++; let saved = yield this._saveObjectFromJSON(
obj, jsonData, options, { skipCache }
);
// Mark updated attachments for download
if (saved && objectType == 'item' && obj.isImportedAttachment()) {
yield this._checkAttachmentForDownload(
obj, jsonData.mtime, isNewObject
);
}
if (saved) {
numSaved++;
} }
} }
}.bind(this)); }.bind(this));
}.bind(this) }.bind(this)
); );
// Keep retrying if we skipped any, as long as we're still making progress
if (numSkipped && numSaved != 0) {
Zotero.debug("More " + objectTypePlural + " in cache -- continuing");
yield this.processSyncCacheForObjectType(libraryID, objectType, options);
}
if (conflicts.length) { if (conflicts.length) {
// Sort conflicts by local Date Modified/Deleted
conflicts.sort(function (a, b) { conflicts.sort(function (a, b) {
var d1 = a.left.dateDeleted || a.left.dateModified; var d1 = a.left.dateDeleted || a.left.dateModified;
var d2 = b.left.dateDeleted || b.left.dateModified; var d2 = b.left.dateDeleted || b.left.dateModified;
@ -442,6 +552,7 @@ Zotero.Sync.Data.Local = {
var mergeData = this.resolveConflicts(conflicts); var mergeData = this.resolveConflicts(conflicts);
if (mergeData) { if (mergeData) {
Zotero.debug("Processing resolved conflicts");
let mergeOptions = {}; let mergeOptions = {};
Object.assign(mergeOptions, options); Object.assign(mergeOptions, options);
// Tell _saveObjectFromJSON not to save with 'synced' set to true // Tell _saveObjectFromJSON not to save with 'synced' set to true
@ -484,11 +595,55 @@ Zotero.Sync.Data.Local = {
} }
} }
// Keep retrying if we skipped any, as long as we're still making progress
if (numSkipped && numSaved != 0) {
Zotero.debug("More " + objectTypePlural + " in cache -- continuing");
return this.processSyncCacheForObjectType(libraryID, objectType, options);
}
data = yield this._getUnwrittenData(libraryID, objectType); data = yield this._getUnwrittenData(libraryID, objectType);
Zotero.debug("Skipping " + data.length + " " if (data.length) {
+ (data.length == 1 ? objectType : objectTypePlural) Zotero.debug(`Skipping ${data.length} `
+ " in sync cache"); + (data.length == 1 ? objectType : objectTypePlural)
return data; + " in sync cache");
}
}),
_checkAttachmentForDownload: Zotero.Promise.coroutine(function* (item, mtime, isNewObject) {
var markToDownload = false;
if (!isNewObject) {
// Convert previously used Unix timestamps to ms-based timestamps
if (mtime < 10000000000) {
Zotero.debug("Converting Unix timestamp '" + mtime + "' to ms");
mtime = mtime * 1000;
}
var fmtime = null;
try {
fmtime = yield item.attachmentModificationTime;
}
catch (e) {
// This will probably fail later too, but ignore it for now
Zotero.logError(e);
}
if (fmtime) {
let state = Zotero.Sync.Storage.Local.checkFileModTime(item, fmtime, mtime);
if (state !== false) {
markToDownload = true;
}
}
else {
markToDownload = true;
}
}
else {
markToDownload = true;
}
if (markToDownload) {
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
}
}), }),
@ -501,6 +656,8 @@ Zotero.Sync.Data.Local = {
resolveConflicts: function (conflicts) { resolveConflicts: function (conflicts) {
Zotero.debug("Showing conflict resolution window");
var io = { var io = {
dataIn: { dataIn: {
captions: [ captions: [
@ -511,9 +668,7 @@ Zotero.Sync.Data.Local = {
conflicts conflicts
} }
}; };
var url = 'chrome://zotero/content/merge.xul'; var url = 'chrome://zotero/content/merge.xul';
var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"] var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator); .getService(Components.interfaces.nsIWindowMediator);
var lastWin = wm.getMostRecentWindow("navigator:browser"); var lastWin = wm.getMostRecentWindow("navigator:browser");
@ -553,7 +708,8 @@ Zotero.Sync.Data.Local = {
_saveObjectFromJSON: Zotero.Promise.coroutine(function* (obj, json, options) { _saveObjectFromJSON: Zotero.Promise.coroutine(function* (obj, json, options) {
try { try {
yield obj.fromJSON(json); yield obj.loadAllData();
obj.fromJSON(json);
if (!options.saveAsChanged) { if (!options.saveAsChanged) {
obj.version = json.version; obj.version = json.version;
obj.synced = true; obj.synced = true;
@ -611,6 +767,11 @@ Zotero.Sync.Data.Local = {
var changeset1 = Zotero.DataObjectUtilities.diff(originalJSON, currentJSON, ignoreFields); var changeset1 = Zotero.DataObjectUtilities.diff(originalJSON, currentJSON, ignoreFields);
var changeset2 = Zotero.DataObjectUtilities.diff(originalJSON, newJSON, ignoreFields); var changeset2 = Zotero.DataObjectUtilities.diff(originalJSON, newJSON, ignoreFields);
Zotero.debug("CHANGESET1");
Zotero.debug(changeset1);
Zotero.debug("CHANGESET2");
Zotero.debug(changeset2);
var conflicts = []; var conflicts = [];
for (let i = 0; i < changeset1.length; i++) { for (let i = 0; i < changeset1.length; i++) {
@ -725,27 +886,43 @@ Zotero.Sync.Data.Local = {
var conflicts = []; var conflicts = [];
for (let i = 0; i < changeset.length; i++) { for (let i = 0; i < changeset.length; i++) {
let c = changeset[i]; let c2 = changeset[i];
// Member changes are additive only, so ignore removals // Member changes are additive only, so ignore removals
if (c.op.endsWith('-remove')) { if (c2.op.endsWith('-remove')) {
continue; continue;
} }
// Record member changes // Record member changes
if (c.op.startsWith('member-') || c.op.startsWith('property-member-')) { if (c2.op.startsWith('member-') || c2.op.startsWith('property-member-')) {
changes.push(c); changes.push(c2);
continue; continue;
} }
// Automatically apply remote changes for non-items, even if in conflict // Automatically apply remote changes for non-items, even if in conflict
if (objectType != 'item') { if (objectType != 'item') {
changes.push(c); changes.push(c2);
continue; continue;
} }
// Field changes are conflicts // Field changes are conflicts
conflicts.push(c); //
// Since we don't know what changed, use only 'add' and 'delete'
if (c2.op == 'modify') {
c2.op = 'add';
}
let val = currentJSON[c2.field];
let c1 = {
field: c2.field,
op: val !== undefined ? 'add' : 'delete'
};
if (val !== undefined) {
c1.value = val;
}
if (c2.op == 'modify') {
c2.op = 'add';
}
conflicts.push([c1, c2]);
} }
return { changes, conflicts }; return { changes, conflicts };

351
chrome/content/zotero/xpcom/sync/syncRunner.js
View File

@ -29,34 +29,62 @@ if (!Zotero.Sync) {
Zotero.Sync = {}; Zotero.Sync = {};
} }
Zotero.Sync.Runner_Module = function () { // Initialized as Zotero.Sync.Runner in zotero.js
Zotero.Sync.Runner_Module = function (options = {}) {
const stopOnError = true;
Zotero.defineProperty(this, 'background', { get: () => _background }); Zotero.defineProperty(this, 'background', { get: () => _background });
Zotero.defineProperty(this, 'lastSyncStatus', { get: () => _lastSyncStatus }); Zotero.defineProperty(this, 'lastSyncStatus', { get: () => _lastSyncStatus });
const stopOnError = true; this.baseURL = options.baseURL || ZOTERO_CONFIG.API_URL;
this.apiVersion = options.apiVersion || ZOTERO_CONFIG.API_VERSION;
this.apiKey = options.apiKey || Zotero.Sync.Data.Local.getAPIKey();
Components.utils.import("resource://zotero/concurrentCaller.js");
this.caller = new ConcurrentCaller(4);
this.caller.setLogger(msg => Zotero.debug(msg));
this.caller.stopOnError = stopOnError;
this.caller.onError = function (e) {
this.addError(e);
if (e.fatal) {
this.caller.stop();
throw e;
}
}.bind(this);
var _autoSyncTimer; var _autoSyncTimer;
var _background; var _background;
var _firstInSession = true; var _firstInSession = true;
var _syncInProgress = false; var _syncInProgress = false;
var _syncEngines = [];
var _storageEngines = [];
var _lastSyncStatus; var _lastSyncStatus;
var _currentSyncStatusLabel; var _currentSyncStatusLabel;
var _currentLastSyncLabel; var _currentLastSyncLabel;
var _errors = []; var _errors = [];
this.getAPIClient = function () {
return new Zotero.Sync.APIClient({
baseURL: this.baseURL,
apiVersion: this.apiVersion,
apiKey: this.apiKey,
caller: this.caller
});
}
/** /**
* Begin a sync session * Begin a sync session
* *
* @param {Object} [options] * @param {Object} [options]
* @param {String} [apiKey] * @param {Boolean} [options.background=false] Whether this is a background request, which
* @param {Boolean} [background=false] - Whether this is a background request, which prevents * prevents some alerts from being shown
* some alerts from being shown * @param {Integer[]} [options.libraries] IDs of libraries to sync
* @param {String} [baseURL] * @param {Function} [options.onError] Function to pass errors to instead of
* @param {Integer[]} [libraries] - IDs of libraries to sync * handling internally (used for testing)
* @param {Function} [onError] - Function to pass errors to instead of handling internally
* (used for testing)
*/ */
this.sync = Zotero.Promise.coroutine(function* (options = {}) { this.sync = Zotero.Promise.coroutine(function* (options = {}) {
// Clear message list // Clear message list
@ -84,14 +112,13 @@ Zotero.Sync.Runner_Module = function () {
// Purge deleted objects so they don't cause sync errors (e.g., long tags) // Purge deleted objects so they don't cause sync errors (e.g., long tags)
yield Zotero.purgeDataObjects(true); yield Zotero.purgeDataObjects(true);
options.apiKey = options.apiKey || Zotero.Prefs.get('devAPIKey'); if (!this.apiKey) {
if (!options.apiKey) { let msg = "API key not set";
let msg = "API key not provided";
let e = new Zotero.Error(msg, 0, { dialogButtonText: null }) let e = new Zotero.Error(msg, 0, { dialogButtonText: null })
this.updateIcons(e); this.updateIcons(e);
_syncInProgress = false;
return false; return false;
} }
options.baseURL = options.baseURL || ZOTERO_CONFIG.API_URL;
if (_firstInSession) { if (_firstInSession) {
options.firstInSession = true; options.firstInSession = true;
_firstInSession = false; _firstInSession = false;
@ -102,66 +129,45 @@ Zotero.Sync.Runner_Module = function () {
this.updateIcons('animate'); this.updateIcons('animate');
try { try {
Components.utils.import("resource://zotero/concurrent-caller.js"); let client = this.getAPIClient();
var caller = new ConcurrentCaller(4); // TEMP: one for now
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = stopOnError;
caller.onError = function (e) {
this.addError(e);
if (e.fatal) {
caller.stop();
throw e;
}
}.bind(this);
// TODO: Use a single client for all operations? let keyInfo = yield this.checkAccess(client, options);
var client = new Zotero.Sync.APIClient({
baseURL: options.baseURL,
apiVersion: ZOTERO_CONFIG.API_VERSION,
apiKey: options.apiKey,
concurrentCaller: caller,
background: options.background
});
var keyInfo = yield this.checkAccess(client, options);
if (!keyInfo) { if (!keyInfo) {
this.stop(); this.end();
Zotero.debug("Syncing cancelled"); Zotero.debug("Syncing cancelled");
return false; return false;
} }
var libraries = yield this.checkLibraries(client, options, keyInfo, libraries); let engineOptions = {
apiClient: client,
for (let libraryID of libraries) { caller: this.caller,
try { setStatus: this.setSyncStatus.bind(this),
let engine = new Zotero.Sync.Data.Engine({ stopOnError,
libraryID: libraryID, onError: function (e) {
apiClient: client,
setStatus: this.setSyncStatus.bind(this),
stopOnError: stopOnError,
onError: this.addError.bind(this)
});
yield engine.start();
}
catch (e) {
Zotero.debug("Sync failed for library " + libraryID);
Zotero.debug(e, 1);
Components.utils.reportError(e);
this.checkError(e);
if (options.onError) { if (options.onError) {
options.onError(e); options.onError(e);
} }
else { else {
this.addError(e); this.addError.bind(this);
} }
if (stopOnError || e.fatal) { }.bind(this),
caller.stop(); background: _background,
break; firstInSession: _firstInSession
} };
}
}
yield Zotero.Sync.Data.Local.updateLastSyncTime(); let nextLibraries = yield this.checkLibraries(
client, options, keyInfo, options.libraries
);
// Sync data, files, and then any data that needs to be uploaded
let attempt = 1;
while (nextLibraries.length) {
if (attempt > 3) {
throw new Error("Too many sync attempts -- stopping");
}
nextLibraries = yield _doDataSync(nextLibraries, engineOptions);
nextLibraries = yield _doFileSync(nextLibraries, engineOptions);
attempt++;
}
} }
catch (e) { catch (e) {
if (options.onError) { if (options.onError) {
@ -171,62 +177,19 @@ Zotero.Sync.Runner_Module = function () {
this.addError(e); this.addError(e);
} }
} }
finally {
this.stop(); this.end();
}
Zotero.debug("Done syncing"); Zotero.debug("Done syncing");
/*if (results.changesMade) {
Zotero.debug("Changes made during file sync "
+ "-- performing additional data sync");
this.sync(options);
}*/
return; return;
var storageSync = function () {
Zotero.Sync.Runner.setSyncStatus(Zotero.getString('sync.status.syncingFiles'));
Zotero.Sync.Storage.sync(options)
.then(function (results) {
Zotero.debug("File sync is finished");
if (results.errors.length) {
Zotero.debug(results.errors, 1);
for each(var e in results.errors) {
Components.utils.reportError(e);
}
Zotero.Sync.Runner.setErrors(results.errors);
return;
}
if (results.changesMade) {
Zotero.debug("Changes made during file sync "
+ "-- performing additional data sync");
Zotero.Sync.Server.sync(finalCallbacks);
}
else {
Zotero.Sync.Runner.stop();
}
})
.catch(function (e) {
Zotero.debug("File sync failed", 1);
Zotero.Sync.Runner.error(e);
})
.done();
};
Zotero.Sync.Server.sync({
// Sync 1 success
onSuccess: storageSync,
// Sync 1 skip
onSkip: storageSync,
// Sync 1 stop
onStop: function () {
Zotero.Sync.Runner.stop();
},
// Sync 1 error
onError: function (e) {
Zotero.Sync.Runner.error(e);
}
});
}); });
@ -242,8 +205,9 @@ Zotero.Sync.Runner_Module = function () {
} }
// Sanity check // Sanity check
if (!json.userID) throw new Error("userID not found in response"); if (!json.userID) throw new Error("userID not found in key response");
if (!json.username) throw new Error("username not found in response"); if (!json.username) throw new Error("username not found in key response");
if (!json.access) throw new Error("'access' not found in key response");
// Make sure user hasn't changed, and prompt to update database if so // Make sure user hasn't changed, and prompt to update database if so
if (!(yield this.checkUser(json.userID, json.username))) { if (!(yield this.checkUser(json.userID, json.username))) {
@ -446,8 +410,6 @@ Zotero.Sync.Runner_Module = function () {
* *
* @param {Integer} userID New userID * @param {Integer} userID New userID
* @param {Integer} libraryID New libraryID * @param {Integer} libraryID New libraryID
* @param {Integer} noServerData The server account is empty — this is
* the account after a server clear
* @return {Boolean} - True to continue, false to cancel * @return {Boolean} - True to continue, false to cancel
*/ */
this.checkUser = Zotero.Promise.coroutine(function* (userID, username) { this.checkUser = Zotero.Promise.coroutine(function* (userID, username) {
@ -544,7 +506,154 @@ Zotero.Sync.Runner_Module = function () {
}); });
var _doDataSync = Zotero.Promise.coroutine(function* (libraries, options, skipUpdateLastSyncTime) {
var successfulLibraries = [];
for (let libraryID of libraries) {
try {
let opts = {};
Object.assign(opts, options);
opts.libraryID = libraryID;
let engine = new Zotero.Sync.Data.Engine(opts);
yield engine.start();
successfulLibraries.push(libraryID);
}
catch (e) {
Zotero.debug("Sync failed for library " + libraryID);
Zotero.logError(e);
this.checkError(e);
if (options.onError) {
options.onError(e);
}
else {
this.addError(e);
}
if (stopOnError || e.fatal) {
Zotero.debug("Stopping on error", 1);
options.caller.stop();
break;
}
}
}
// Update last-sync time if any libraries synced
// TEMP: Do we want to show updated time if some libraries haven't synced?
if (!libraries.length || successfulLibraries.length) {
yield Zotero.Sync.Data.Local.updateLastSyncTime();
}
return successfulLibraries;
}.bind(this));
var _doFileSync = Zotero.Promise.coroutine(function* (libraries, options) {
Zotero.debug("Starting file syncing");
this.setSyncStatus(Zotero.getString('sync.status.syncingFiles'));
let librariesToSync = [];
for (let libraryID of libraries) {
try {
let opts = {};
Object.assign(opts, options);
opts.libraryID = libraryID;
let tries = 3;
while (true) {
if (tries == 0) {
throw new Error("Too many file sync attempts for library " + libraryID);
}
tries--;
let engine = new Zotero.Sync.Storage.Engine(opts);
let results = yield engine.start();
if (results.syncRequired) {
librariesToSync.push(libraryID);
}
else if (results.fileSyncRequired) {
Zotero.debug("Another file sync required -- restarting");
continue;
}
break;
}
}
catch (e) {
Zotero.debug("File sync failed for library " + libraryID);
Zotero.debug(e, 1);
Components.utils.reportError(e);
this.checkError(e);
if (options.onError) {
options.onError(e);
}
else {
this.addError(e);
}
if (stopOnError || e.fatal) {
options.caller.stop();
break;
}
}
}
Zotero.debug("Done with file syncing");
return librariesToSync;
}.bind(this));
/**
* Download a single file on demand (not within a sync process)
*/
this.downloadFile = Zotero.Promise.coroutine(function* (item, requestCallbacks) {
if (Zotero.HTTP.browserIsOffline()) {
Zotero.debug("Browser is offline", 2);
return false;
}
// TEMP
var options = {};
var itemID = item.id;
var modeClass = Zotero.Sync.Storage.Local.getClassForLibrary(item.libraryID);
var controller = new modeClass({
apiClient: this.getAPIClient()
});
// TODO: verify WebDAV on-demand?
if (!controller.verified) {
Zotero.debug("File syncing is not active for item's library -- skipping download");
return false;
}
if (!item.isImportedAttachment()) {
throw new Error("Not an imported attachment");
}
if (yield item.getFilePathAsync()) {
Zotero.debug("File already exists -- replacing");
}
// TODO: start sync icon?
// TODO: create queue for cancelling
if (!requestCallbacks) {
requestCallbacks = {};
}
var onStart = function (request) {
return controller.downloadFile(request);
};
var request = new Zotero.Sync.Storage.Request({
type: 'download',
libraryID: item.libraryID,
name: item.libraryKey,
onStart: requestCallbacks.onStart
? [onStart, requestCallbacks.onStart]
: [onStart]
});
return request.start();
});
this.stop = function () { this.stop = function () {
_syncEngines.forEach(engine => engine.stop());
_storageEngines.forEach(engine => engine.stop());
}
this.end = function () {
this.updateIcons(_errors); this.updateIcons(_errors);
_errors = []; _errors = [];
_syncInProgress = false; _syncInProgress = false;
@ -669,7 +778,6 @@ Zotero.Sync.Runner_Module = function () {
if (libraryID) { if (libraryID) {
e.libraryID = libraryID; e.libraryID = libraryID;
} }
Zotero.logError(e);
_errors.push(this.parseError(e)); _errors.push(this.parseError(e));
} }
@ -1027,7 +1135,8 @@ Zotero.Sync.Runner_Module = function () {
var lastSyncTime = Zotero.Sync.Data.Local.getLastSyncTime(); var lastSyncTime = Zotero.Sync.Data.Local.getLastSyncTime();
if (!lastSyncTime) { if (!lastSyncTime) {
try { try {
lastSyncTime = Zotero.Sync.Data.Local.getLastClassicSyncTime() lastSyncTime = Zotero.Sync.Data.Local.getLastClassicSyncTime();
Zotero.debug(lastSyncTime);
} }
catch (e) { catch (e) {
Zotero.debug(e, 2); Zotero.debug(e, 2);
@ -1052,5 +1161,3 @@ Zotero.Sync.Runner_Module = function () {
_currentLastSyncLabel.hidden = false; _currentLastSyncLabel.hidden = false;
} }
} }
Zotero.Sync.Runner = new Zotero.Sync.Runner_Module;

4
chrome/content/zotero/xpcom/zotero.js
View File

@ -607,6 +607,7 @@ Components.utils.import("resource://gre/modules/osfile.jsm");
yield Zotero.Sync.Data.Local.init(); yield Zotero.Sync.Data.Local.init();
yield Zotero.Sync.Data.Utilities.init(); yield Zotero.Sync.Data.Utilities.init();
Zotero.Sync.EventListeners.init(); Zotero.Sync.EventListeners.init();
Zotero.Sync.Runner = new Zotero.Sync.Runner_Module;
Zotero.MIMETypeHandler.init(); Zotero.MIMETypeHandler.init();
yield Zotero.Proxies.init(); yield Zotero.Proxies.init();
@ -2706,6 +2707,9 @@ Zotero.Browser = new function() {
if(!win) { if(!win) {
var win = Services.ww.activeWindow; var win = Services.ww.activeWindow;
} }
if (!win) {
throw new Error("Parent window not available for hidden browser");
}
} }
// Create a hidden browser // Create a hidden browser

54
chrome/content/zotero/zoteroPane.js
View File

@ -1325,6 +1325,7 @@ var ZoteroPane = new function()
else if (item.isAttachment()) { else if (item.isAttachment()) {
var attachmentBox = document.getElementById('zotero-attachment-box'); var attachmentBox = document.getElementById('zotero-attachment-box');
attachmentBox.mode = this.collectionsView.editable ? 'edit' : 'view'; attachmentBox.mode = this.collectionsView.editable ? 'edit' : 'view';
yield item.loadNote();
attachmentBox.item = item; attachmentBox.item = item;
document.getElementById('zotero-item-pane-content').selectedIndex = 3; document.getElementById('zotero-item-pane-content').selectedIndex = 3;
@ -3692,38 +3693,41 @@ var ZoteroPane = new function()
} }
} }
else { else {
if (!item.isImportedAttachment() || !Zotero.Sync.Storage.downloadAsNeeded(item.libraryID)) { if (!item.isImportedAttachment()
|| !Zotero.Sync.Storage.Local.downloadAsNeeded(item.libraryID)) {
this.showAttachmentNotFoundDialog(itemID, noLocateOnMissing); this.showAttachmentNotFoundDialog(itemID, noLocateOnMissing);
return; return;
} }
let downloadedItem = item; let downloadedItem = item;
yield Zotero.Sync.Storage.downloadFile( try {
downloadedItem, yield Zotero.Sync.Runner.downloadFile(
{ downloadedItem,
onProgress: function (progress, progressMax) {} {
} onProgress: function (progress, progressMax) {}
) }
.then(function () { );
if (!downloadedItem.getFile()) { }
ZoteroPane_Local.showAttachmentNotFoundDialog(downloadedItem.id, noLocateOnMissing); catch (e) {
return;
}
// check if unchanged?
// maybe not necessary, since we'll get an error if there's an error
Zotero.Notifier.trigger('redraw', 'item', []);
Zotero.debug('downloaded');
Zotero.debug(downloadedItem.id);
return ZoteroPane_Local.viewAttachment(downloadedItem.id, event, false, forceExternalViewer);
})
.catch(function (e) {
// TODO: show error somewhere else // TODO: show error somewhere else
Zotero.debug(e, 1); Zotero.debug(e, 1);
ZoteroPane_Local.syncAlert(e); ZoteroPane_Local.syncAlert(e);
}); return;
}
if (!(yield downloadedItem.getFilePathAsync())) {
ZoteroPane_Local.showAttachmentNotFoundDialog(downloadedItem.id, noLocateOnMissing);
return;
}
// check if unchanged?
// maybe not necessary, since we'll get an error if there's an error
Zotero.Notifier.trigger('redraw', 'item', []);
Zotero.debug('downloaded');
Zotero.debug(downloadedItem.id);
return ZoteroPane_Local.viewAttachment(downloadedItem.id, event, false, forceExternalViewer);
} }
} }
}); });
@ -3962,7 +3966,7 @@ var ZoteroPane = new function()
this.syncAlert = function (e) { this.syncAlert = function (e) {
e = Zotero.Sync.Runner.parseSyncError(e); e = Zotero.Sync.Runner.parseError(e);
var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"] var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"]
.getService(Components.interfaces.nsIPromptService); .getService(Components.interfaces.nsIPromptService);

5
chrome/content/zotero/zoteroPane.xul
View File

@ -195,9 +195,10 @@
</hbox> </hbox>
<hbox align="center" pack="end"> <hbox align="center" pack="end">
<hbox id="zotero-tb-sync-progress-box" hidden="true" align="center"> <hbox id="zotero-tb-sync-progress-box" hidden="true" align="center">
<!-- TODO: localize -->
<toolbarbutton id="zotero-tb-sync-storage-cancel" <toolbarbutton id="zotero-tb-sync-storage-cancel"
tooltiptext="Cancel Storage Sync" tooltiptext="Stop sync"
oncommand="Zotero.Sync.Storage.QueueManager.cancel()"/> oncommand="Zotero.Sync.Runner.stop()"/>
<progressmeter id="zotero-tb-sync-progress" mode="determined" <progressmeter id="zotero-tb-sync-progress" mode="determined"
value="0" tooltip="zotero-tb-sync-progress-tooltip"> value="0" tooltip="zotero-tb-sync-progress-tooltip">
</progressmeter> </progressmeter>

12
chrome/locale/en-US/zotero/zotero.properties
View File

@ -961,12 +961,12 @@ rtfScan.saveTitle = Select a location in which to save the formatted file
rtfScan.scannedFileSuffix = (Scanned) rtfScan.scannedFileSuffix = (Scanned)
file.accessError.theFile = The file '%S' file.accessError.theFileCannotBeCreated = The file '%S' cannot be created.
file.accessError.aFile = A file file.accessError.theFileCannotBeUpdated = The file '%S' cannot be updated.
file.accessError.cannotBe = cannot be file.accessError.theFileCannotBeDeleted = The file '%S' cannot be deleted.
file.accessError.created = created file.accessError.aFileCannotBeCreated = A file cannot be created.
file.accessError.updated = updated file.accessError.aFileCannotBeUpdated = A file cannot be updated.
file.accessError.deleted = deleted file.accessError.aFileCannotBeDeleted = A file cannot be deleted.
file.accessError.message.windows = Check that the file is not currently in use, that its permissions allow write access, and that it has a valid filename. file.accessError.message.windows = Check that the file is not currently in use, that its permissions allow write access, and that it has a valid filename.
file.accessError.message.other = Check that the file is not currently in use and that its permissions allow write access. file.accessError.message.other = Check that the file is not currently in use and that its permissions allow write access.
file.accessError.restart = Restarting your computer or disabling security software may also help. file.accessError.restart = Restarting your computer or disabling security software may also help.

9
components/zotero-service.js
View File

@ -107,11 +107,12 @@ const xpcomFilesLocal = [
'sync/syncRunner', 'sync/syncRunner',
'sync/syncUtilities', 'sync/syncUtilities',
'storage', 'storage',
'storage/storageEngine',
'storage/storageLocal',
'storage/storageRequest',
'storage/storageResult',
'storage/storageUtilities',
'storage/streamListener', 'storage/streamListener',
'storage/queueManager',
'storage/queue',
'storage/request',
'storage/mode',
'storage/zfs', 'storage/zfs',
'storage/webdav', 'storage/webdav',
'syncedSettings', 'syncedSettings',

5356
test/resource/httpd.js Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
test/tests/data/snapshot/img.gif Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 42 B

8
test/tests/data/test.html Normal file
View File

@ -0,0 +1,8 @@
<html>
<head>
<meta charset="utf-8"/>
</head>
<body>
<p>This is a test.</p>
</body>
</html>

1
test/tests/data/test.txt Normal file
View File

@ -0,0 +1 @@
This is a test file.

10
test/tests/itemTest.js
View File

@ -537,6 +537,10 @@ describe("Zotero.Item", function () {
file.append(filename); file.append(filename);
assert.equal(item.getFilePath(), file.path); assert.equal(item.getFilePath(), file.path);
}); });
it.skip("should get and set a filename for a base-dir-relative file", function* () {
})
}) })
describe("#attachmentPath", function () { describe("#attachmentPath", function () {
@ -608,11 +612,13 @@ describe("Zotero.Item", function () {
assert.equal(OS.Path.basename(path), newName) assert.equal(OS.Path.basename(path), newName)
yield OS.File.exists(path); yield OS.File.exists(path);
// File should be flagged for upload
// DEBUG: Is this necessary?
assert.equal( assert.equal(
(yield Zotero.Sync.Storage.getSyncState(item.id)), (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
); );
assert.isNull(yield Zotero.Sync.Storage.getSyncedHash(item.id)); assert.isNull(yield Zotero.Sync.Storage.Local.getSyncedHash(item.id));
}) })
}) })

822
test/tests/storageEngineTest.js Normal file
View File

@ -0,0 +1,822 @@
"use strict";
describe("Zotero.Sync.Storage.Engine", function () {
Components.utils.import("resource://zotero-unit/httpd.js");
var win;
var apiKey = Zotero.Utilities.randomString(24);
var port = 16213;
var baseURL = `http://localhost:${port}/`;
var server;
var responses = {};
var setup = Zotero.Promise.coroutine(function* (options = {}) {
server = sinon.fakeServer.create();
server.autoRespond = true;
Components.utils.import("resource://zotero/concurrentCaller.js");
var caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true;
Components.utils.import("resource://zotero/config.js");
var client = new Zotero.Sync.APIClient({
baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
apiKey,
caller,
background: options.background || true
});
var engine = new Zotero.Sync.Storage.Engine({
apiClient: client,
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
stopOnError: true
});
return { engine, client, caller };
});
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
function parseQueryString(str) {
var queryStringParams = str.split('&');
var params = {};
for (let param of queryStringParams) {
let [ key, val ] = param.split('=');
params[key] = decodeURIComponent(val);
}
return params;
}
function assertAPIKey(request) {
assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
}
//
// Tests
//
before(function* () {
})
beforeEach(function* () {
Zotero.debug("BEFORE HERE");
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
Zotero.debug("DONE RESET");
win = yield loadZoteroPane();
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
this.httpd = new HttpServer();
this.httpd.start(port);
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser");
// Set download-on-sync by default
Zotero.Sync.Storage.Local.downloadOnSync(
Zotero.Libraries.userLibraryID, true
);
Zotero.debug("DONE BEFORE");
})
afterEach(function* () {
var defer = new Zotero.Promise.defer();
this.httpd.stop(() => defer.resolve());
yield defer.promise;
win.close();
})
after(function* () {
this.timeout(60000);
//yield resetDB();
win.close();
})
describe("ZFS", function () {
describe("Syncing", function () {
it("should skip downloads if no last storage sync time", function* () {
var { engine, client, caller } = yield setup();
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 404
});
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.isFalse(Zotero.Libraries.userLibrary.lastStorageSync);
})
it("should skip downloads if unchanged last storage sync time", function* () {
var { engine, client, caller } = yield setup();
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
var library = Zotero.Libraries.userLibrary;
library.lastStorageSync = newStorageSyncTime;
yield library.saveTx();
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.equal(library.lastStorageSync, newStorageSyncTime);
})
it("should ignore a remotely missing file", function* () {
var { engine, client, caller } = yield setup();
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 404, null);
}
}
);
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
})
it("should handle a remotely failing file", function* () {
var { engine, client, caller } = yield setup();
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 500, null);
}
}
);
// TODO: In stopOnError mode, this the promise is rejected.
// This should probably test with stopOnError mode turned off instead.
var e = yield getPromiseError(engine.start());
assert.equal(e.message, Zotero.Sync.Storage.defaultError);
})
it("should download a missing file", function* () {
var { engine, client, caller } = yield setup();
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
// TODO: Test binary data
var text = Zotero.Utilities.randomString();
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var mtime = "1441252524905";
var md5 = Zotero.Utilities.Internal.md5(text)
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
var s3Path = `pretend-s3/${item.key}`;
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
if (!request.hasHeader('Zotero-API-Key')) {
response.setStatusLine(null, 403, "Forbidden");
return;
}
var key = request.getHeader('Zotero-API-Key');
if (key != apiKey) {
response.setStatusLine(null, 403, "Invalid key");
return;
}
response.setStatusLine(null, 302, "Found");
response.setHeader("Zotero-File-Modification-Time", mtime, false);
response.setHeader("Zotero-File-MD5", md5, false);
response.setHeader("Zotero-File-Compressed", "No", false);
response.setHeader("Location", baseURL + s3Path, false);
}
}
);
this.httpd.registerPathHandler(
"/" + s3Path,
{
handle: function (request, response) {
response.setStatusLine(null, 200, "OK");
response.write(text);
}
}
);
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync());
assert.equal(contents, text);
})
it("should upload new files", function* () {
var { engine, client, caller } = yield setup();
// Single file
var file1 = getTestDataDirectory();
file1.append('test.png');
var item1 = yield Zotero.Attachments.importFromFile({ file: file1 });
var mtime1 = yield item1.attachmentModificationTime;
var hash1 = yield item1.attachmentHash;
var path1 = item1.getFilePath();
var filename1 = 'test.png';
var size1 = (yield OS.File.stat(path1)).size;
var contentType1 = 'image/png';
var prefix1 = Zotero.Utilities.randomString();
var suffix1 = Zotero.Utilities.randomString();
var uploadKey1 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
// HTML file with auxiliary image
var file2 = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
var parentItem = yield createDataObject('item');
var item2 = yield Zotero.Attachments.importSnapshotFromFile({
file: file2,
url: 'http://example.com/',
parentItemID: parentItem.id,
title: 'Test',
contentType: 'text/html',
charset: 'utf-8'
});
var mtime2 = yield item2.attachmentModificationTime;
var hash2 = yield item2.attachmentHash;
var path2 = item2.getFilePath();
var filename2 = 'index.html';
var size2 = (yield OS.File.stat(path2)).size;
var contentType2 = 'text/html';
var charset2 = 'utf-8';
var prefix2 = Zotero.Utilities.randomString();
var suffix2 = Zotero.Utilities.randomString();
var uploadKey2 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
var deferreds = [];
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 404
});
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.md5, hash1);
assert.equal(params.mtime, mtime1);
assert.equal(params.filename, filename1);
assert.equal(params.filesize, size1);
assert.equal(params.contentType, contentType1);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/1",
contentType: contentType1,
prefix: prefix1,
suffix: suffix1,
uploadKey: uploadKey1
})
);
}
// Get upload authorization for multi-file zip
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
// Verify ZIP hash
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
item2.key + '.zip'
);
deferreds.push({
promise: Zotero.Utilities.Internal.md5Async(tmpZipPath)
.then(function (md5) {
assert.equal(params.zipMD5, md5);
})
});
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
Zotero.debug(params);
assert.equal(params.md5, hash2);
assert.notEqual(params.zipMD5, hash2);
assert.equal(params.mtime, mtime2);
assert.equal(params.filename, filename2);
assert.equal(params.zipFilename, item2.key + ".zip");
assert.isTrue(parseInt(params.filesize) == params.filesize);
assert.equal(params.contentType, contentType2);
assert.equal(params.charset, charset2);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/2",
contentType: 'application/zip',
prefix: prefix2,
suffix: suffix2,
uploadKey: uploadKey2
})
);
}
// Upload single file to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
assert.equal(req.requestHeaders["Content-Type"], contentType1 + fixSinonBug);
assert.equal(req.requestBody.size, (new Blob([prefix1, File(file1), suffix1]).size));
req.respond(201, {}, "");
}
// Upload multi-file ZIP to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/2") {
assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
// Verify uploaded ZIP file
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
Zotero.Utilities.randomString() + '.zip'
);
let deferred = Zotero.Promise.defer();
deferreds.push(deferred);
var reader = new FileReader();
reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () {
try {
let file = yield OS.File.open(tmpZipPath, {
create: true
});
var contents = new Uint8Array(reader.result);
contents = contents.slice(prefix2.length, suffix2.length * -1);
yield file.write(contents);
yield file.close();
var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
.createInstance(Components.interfaces.nsIZipReader);
zr.open(Zotero.File.pathToFile(tmpZipPath));
zr.test(null);
var entries = zr.findEntries('*');
var entryNames = [];
while (entries.hasMore()) {
entryNames.push(entries.getNext());
}
assert.equal(entryNames.length, 2);
assert.sameMembers(entryNames, ['index.html', 'img.gif']);
assert.equal(zr.getEntry('index.html').realSize, size2);
assert.equal(zr.getEntry('img.gif').realSize, 42);
deferred.resolve();
}
catch (e) {
deferred.reject(e);
}
}));
reader.readAsArrayBuffer(req.requestBody);
req.respond(201, {}, "");
}
// Register single-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey1);
req.respond(
204,
{
"Last-Modified-Version": 10
},
""
);
}
// Register multi-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey2);
req.respond(
204,
{
"Last-Modified-Version": 15
},
""
);
}
})
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "POST",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
// TODO: One-step uploads
/*// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file`) {
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/json" + fixSinonBug
);
let params = JSON.parse(req.requestBody);
assert.equal(params.md5, hash);
assert.equal(params.mtime, mtime);
assert.equal(params.filename, filename);
assert.equal(params.size, size);
assert.equal(params.contentType, contentType);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3",
headers: {
"Content-Type": contentType,
"Content-MD5": hash,
//"Content-Length": params.size, process but don't return
//"x-amz-meta-"
},
uploadKey
})
);
}
else if (req.method == "PUT" && req.url == baseURL + "pretend-s3") {
assert.equal(req.requestHeaders["Content-Type"], contentType + fixSinonBug);
assert.instanceOf(req.requestBody, File);
req.respond(201, {}, "");
}
})*/
var result = yield engine.start();
yield Zotero.Promise.all(deferreds.map(d => d.promise));
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item1.id)), mtime1);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item1.id)), hash1);
assert.equal(item1.version, 10);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item2.id)), mtime2);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item2.id)), hash2);
assert.equal(item2.version, 15);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
})
it("should update local info for file that already exists on the server", function* () {
var { engine, client, caller } = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file: file });
item.version = 5;
yield item.saveTx();
var json = yield item.toJSON();
yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var newVersion = 10;
setResponse({
method: "POST",
url: "users/1/laststoragesync",
status: 200,
text: "" + (Math.round(new Date().getTime() / 1000) - 50000)
});
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
req.respond(
200,
{
"Content-Type": "application/json",
"Last-Modified-Version": newVersion
},
JSON.stringify({
exists: 1,
})
);
}
})
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "POST",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
// TODO: One-step uploads
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.equal(item.version, newVersion);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
})
})
describe("#_processUploadFile()", function () {
it("should handle 412 with matching version and hash matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.ZFS_Module({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var itemJSON = yield item.toResponseJSON();
// Set saved hash to a different value, which should be overwritten
//
// We're also testing cases where a hash isn't set for a file (e.g., if the
// storage directory was transferred, the mtime doesn't match, but the file was
// never downloaded), but there's no difference in behavior
var dbHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, dbHash)
});
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-Match"] == dbHash) {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"ETag does not match current version of file"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncedHash(item.id), itemJSON.data.md5
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isFalse(result.fileSyncRequired);
})
it("should handle 412 with matching version and hash not matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.ZFS_Module({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var fileHash = yield item.attachmentHash;
var itemJSON = yield item.toResponseJSON();
itemJSON.data.md5 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"If-None-Match: * set but file exists"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.isNull(Zotero.Sync.Storage.Local.getSyncedHash(item.id));
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item.id),
Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isTrue(result.fileSyncRequired);
})
it("should handle 412 with greater version", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.ZFS_Module({
apiClient: client
})
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.version = 5;
item.synced = true;
yield item.saveTx();
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 10
},
"If-None-Match: * set but file exists"
);
}
})
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
assert.equal(item.version, 5);
assert.equal(item.synced, true);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isTrue(result.syncRequired);
})
})
})
})

329
test/tests/storageLocalTest.js Normal file
View File

@ -0,0 +1,329 @@
"use strict";
describe("Zotero.Sync.Storage.Local", function () {
var win;
before(function* () {
win = yield loadBrowserWindow();
});
beforeEach(function* () {
yield resetDB({
thisArg: this
})
})
after(function () {
if (win) {
win.close();
}
});
describe("#checkForUpdatedFiles()", function () {
it("should flag modified file for upload and return it", function* () {
// Create attachment
let item = yield importFileAttachment('test.txt')
var hash = yield item.attachmentHash;
// Set file mtime to the past (without milliseconds, which aren't used on OS X)
var mtime = (Math.floor(new Date().getTime() / 1000) * 1000) - 1000;
yield OS.File.setDates((yield item.getFilePathAsync()), null, mtime);
// Mark as synced, so it will be checked
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
});
// Update mtime and contents
var path = yield item.getFilePathAsync();
yield OS.File.setDates(path);
yield Zotero.File.putContentsAsync(path, Zotero.Utilities.randomString());
// File should be returned
var libraryID = Zotero.Libraries.userLibraryID;
var changed = yield Zotero.Sync.Storage.Local.checkForUpdatedFiles(libraryID);
yield item.eraseTx();
assert.equal(changed, true);
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
);
})
it("should skip a file if mod time hasn't changed", function* () {
// Create attachment
let item = yield importFileAttachment('test.txt')
var hash = yield item.attachmentHash;
var mtime = yield item.attachmentModificationTime;
// Mark as synced, so it will be checked
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
});
var libraryID = Zotero.Libraries.userLibraryID;
var changed = yield Zotero.Sync.Storage.Local.checkForUpdatedFiles(libraryID);
var syncState = yield Zotero.Sync.Storage.Local.getSyncState(item.id);
yield item.eraseTx();
assert.isFalse(changed);
assert.equal(syncState, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
})
it("should skip a file if mod time has changed but contents haven't", function* () {
// Create attachment
let item = yield importFileAttachment('test.txt')
var hash = yield item.attachmentHash;
// Set file mtime to the past (without milliseconds, which aren't used on OS X)
var mtime = (Math.floor(new Date().getTime() / 1000) * 1000) - 1000;
yield OS.File.setDates((yield item.getFilePathAsync()), null, mtime);
// Mark as synced, so it will be checked
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
});
// Update mtime, but not contents
var path = yield item.getFilePathAsync();
yield OS.File.setDates(path);
var libraryID = Zotero.Libraries.userLibraryID;
var changed = yield Zotero.Sync.Storage.Local.checkForUpdatedFiles(libraryID);
var syncState = yield Zotero.Sync.Storage.Local.getSyncState(item.id);
var syncedModTime = yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id);
var newModTime = yield item.attachmentModificationTime;
yield item.eraseTx();
assert.isFalse(changed);
assert.equal(syncState, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
assert.equal(syncedModTime, newModTime);
})
})
describe("#processDownload()", function () {
var file1Name = 'index.html';
var file1Contents = '<html><body>Test</body></html>';
var file2Name = 'test.txt';
var file2Contents = 'Test';
var createZIP = Zotero.Promise.coroutine(function* (zipFile) {
var tmpDir = Zotero.getTempDirectory().path;
var zipDir = OS.Path.join(tmpDir, Zotero.Utilities.randomString());
yield OS.File.makeDir(zipDir);
yield Zotero.File.putContentsAsync(OS.Path.join(zipDir, file1Name), file1Contents);
yield Zotero.File.putContentsAsync(OS.Path.join(zipDir, file2Name), file2Contents);
yield Zotero.File.zipDirectory(zipDir, zipFile);
yield OS.File.removeDir(zipDir);
});
it("should download and extract a ZIP file into the attachment directory", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var parentItem = yield createDataObject('item');
var key = Zotero.DataObjectUtilities.generateKey();
var tmpDir = Zotero.getTempDirectory().path;
var zipFile = OS.Path.join(tmpDir, key + '.tmp');
yield createZIP(zipFile);
var md5 = Zotero.Utilities.Internal.md5(Zotero.File.pathToFile(zipFile));
var mtime = 1445667239000;
var json = {
key,
version: 10,
itemType: 'attachment',
linkMode: 'imported_url',
url: 'https://example.com',
filename: file1Name,
contentType: 'text/html',
charset: 'utf-8',
md5,
mtime
};
yield Zotero.Sync.Data.Local.saveCacheObjects(
'item', libraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, 'item', { stopOnError: true }
);
var item = yield Zotero.Items.getByLibraryAndKeyAsync(libraryID, key);
yield Zotero.Sync.Storage.Local.processDownload({
item,
md5,
mtime,
compressed: true
});
yield OS.File.remove(zipFile);
yield assert.eventually.equal(
item.attachmentHash, Zotero.Utilities.Internal.md5(file1Contents)
);
yield assert.eventually.equal(item.attachmentModificationTime, mtime);
})
})
describe("#_deleteExistingAttachmentFiles()", function () {
it("should delete all files", function* () {
var item = yield importFileAttachment('test.html');
var path = OS.Path.dirname(item.getFilePath());
var files = ['a', 'b', 'c', 'd'];
for (let file of files) {
yield Zotero.File.putContentsAsync(OS.Path.join(path, file), file);
}
yield Zotero.Sync.Storage.Local._deleteExistingAttachmentFiles(item);
for (let file of files) {
assert.isFalse(
(yield OS.File.exists(OS.Path.join(path, file))),
`File '${file}' doesn't exist`
);
}
})
})
describe("#getConflicts()", function () {
it("should return an array of objects for attachments in conflict", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var item1 = yield importFileAttachment('test.png');
item1.version = 10;
yield item1.saveTx();
var item2 = yield importFileAttachment('test.txt');
var item3 = yield importFileAttachment('test.html');
item3.version = 11;
yield item3.saveTx();
var json1 = yield item1.toJSON();
var json3 = yield item3.toJSON();
// Change remote mtimes
// Round to nearest second because OS X doesn't support ms resolution
var now = Math.round(new Date().getTime() / 1000) * 1000;
json1.mtime = now - 10000;
json3.mtime = now - 20000;
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]);
yield Zotero.Sync.Storage.Local.setSyncState(
item1.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
yield Zotero.Sync.Storage.Local.setSyncState(
item3.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
var conflicts = yield Zotero.Sync.Storage.Local.getConflicts(libraryID);
assert.lengthOf(conflicts, 2);
var item1Conflict = conflicts.find(x => x.left.key == item1.key);
assert.equal(
item1Conflict.left.dateModified,
Zotero.Date.dateToISO(new Date(yield item1.attachmentModificationTime))
);
assert.equal(
item1Conflict.right.dateModified,
Zotero.Date.dateToISO(new Date(json1.mtime))
);
var item3Conflict = conflicts.find(x => x.left.key == item3.key);
assert.equal(
item3Conflict.left.dateModified,
Zotero.Date.dateToISO(new Date(yield item3.attachmentModificationTime))
);
assert.equal(
item3Conflict.right.dateModified,
Zotero.Date.dateToISO(new Date(json3.mtime))
);
})
})
describe("#resolveConflicts()", function () {
it("should show the conflict resolution window on attachment conflicts", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var item1 = yield importFileAttachment('test.png');
item1.version = 10;
yield item1.saveTx();
var item2 = yield importFileAttachment('test.txt');
var item3 = yield importFileAttachment('test.html');
item3.version = 11;
yield item3.saveTx();
var json1 = yield item1.toJSON();
var json3 = yield item3.toJSON();
// Change remote mtimes
json1.mtime = new Date().getTime() + 10000;
json3.mtime = new Date().getTime() - 10000;
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]);
yield Zotero.Sync.Storage.Local.setSyncState(
item1.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
yield Zotero.Sync.Storage.Local.setSyncState(
item3.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
var promise = waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
var doc = dialog.document;
var wizard = doc.documentElement;
var mergeGroup = wizard.getElementsByTagName('zoteromergegroup')[0];
// 1 (remote)
// Later remote version should be selected
assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
// Check checkbox text
assert.equal(
doc.getElementById('resolve-all').label,
Zotero.getString('sync.conflict.resolveAllRemote')
);
// Select local object
mergeGroup.leftpane.click();
assert.equal(mergeGroup.leftpane.getAttribute('selected'), 'true');
wizard.getButton('next').click();
// 2 (local)
// Later local version should be selected
assert.equal(mergeGroup.leftpane.getAttribute('selected'), 'true');
// Select remote object
mergeGroup.rightpane.click();
assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
if (Zotero.isMac) {
assert.isTrue(wizard.getButton('next').hidden);
assert.isFalse(wizard.getButton('finish').hidden);
}
else {
// TODO
}
wizard.getButton('finish').click();
})
yield Zotero.Sync.Storage.Local.resolveConflicts(libraryID);
yield promise;
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item1.id),
Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD
);
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item3.id),
Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD
);
})
})
})

22
test/tests/storageRequestTest.js Normal file
View File

@ -0,0 +1,22 @@
"use strict";
describe("Zotero.Sync.Storage.Request", function () {
describe("#run()", function () {
it("should run a request and wait for it to complete", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var count = 0;
var request = new Zotero.Sync.Storage.Request({
type: 'download',
libraryID,
name: "1/AAAAAAAA",
onStart: Zotero.Promise.coroutine(function* () {
yield Zotero.Promise.delay(25);
count++;
return new Zotero.Sync.Storage.Result;
})
});
var results = yield request.start();
assert.equal(count, 1);
})
})
})

20
test/tests/syncEngineTest.js
View File

@ -19,28 +19,20 @@ describe("Zotero.Sync.Data.Engine", function () {
var caller = new ConcurrentCaller(1); var caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg)); caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true; caller.stopOnError = true;
caller.onError = function (e) {
Zotero.logError(e);
if (options.onError) {
options.onError(e);
}
if (e.fatal) {
caller.stop();
throw e;
}
};
Components.utils.import("resource://zotero/config.js");
var client = new Zotero.Sync.APIClient({ var client = new Zotero.Sync.APIClient({
baseURL: baseURL, baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION, apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
apiKey: apiKey, apiKey,
concurrentCaller: caller, caller,
background: options.background || true background: options.background || true
}); });
var engine = new Zotero.Sync.Data.Engine({ var engine = new Zotero.Sync.Data.Engine({
apiClient: client, apiClient: client,
libraryID: options.libraryID || Zotero.Libraries.userLibraryID libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
stopOnError: true
}); });
return { engine, client, caller }; return { engine, client, caller };

264
test/tests/syncLocalTest.js
View File

@ -4,7 +4,7 @@ describe("Zotero.Sync.Data.Local", function() {
describe("#processSyncCacheForObjectType()", function () { describe("#processSyncCacheForObjectType()", function () {
var types = Zotero.DataObjectUtilities.getTypes(); var types = Zotero.DataObjectUtilities.getTypes();
it("should update local version number if remote version is identical", function* () { it("should update local version number and mark as synced if remote version is identical", function* () {
var libraryID = Zotero.Libraries.userLibraryID; var libraryID = Zotero.Libraries.userLibraryID;
for (let type of types) { for (let type of types) {
@ -24,11 +24,167 @@ describe("Zotero.Sync.Data.Local", function() {
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType( yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true } libraryID, type, { stopOnError: true }
); );
assert.equal( let localObj = objectsClass.getByLibraryAndKey(libraryID, obj.key);
objectsClass.getByLibraryAndKey(libraryID, obj.key).version, 10 assert.equal(localObj.version, 10);
); assert.isTrue(localObj.synced);
} }
}) })
it("should keep local item changes while applying non-conflicting remote changes", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var type = 'item';
let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(type);
let obj = yield createDataObject(type, { version: 5 });
let data = yield obj.toJSON();
yield Zotero.Sync.Data.Local.saveCacheObjects(
type, libraryID, [data]
);
// Change local title
yield modifyDataObject(obj)
var changedTitle = obj.getField('title');
// Save remote version to cache without title but with changed place
data.key = obj.key;
data.version = 10;
var changedPlace = data.place = 'New York';
let json = {
key: obj.key,
version: 10,
data: data
};
yield Zotero.Sync.Data.Local.saveCacheObjects(
type, libraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
);
assert.equal(obj.version, 10);
assert.equal(obj.getField('title'), changedTitle);
assert.equal(obj.getField('place'), changedPlace);
})
it("should mark new attachment items for download", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
Zotero.Sync.Storage.Local.setModeForLibrary(libraryID, 'zfs');
var key = Zotero.DataObjectUtilities.generateKey();
var version = 10;
var json = {
key,
version,
data: {
key,
version,
itemType: 'attachment',
linkMode: 'imported_file',
md5: '57f8a4fda823187b91e1191487b87fe6',
mtime: 1442261130615
}
};
yield Zotero.Sync.Data.Local.saveCacheObjects(
'item', Zotero.Libraries.userLibraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, 'item', { stopOnError: true }
);
var id = Zotero.Items.getIDFromLibraryAndKey(libraryID, key);
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(id)),
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
})
it("should mark updated attachment items for download", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
Zotero.Sync.Storage.Local.setModeForLibrary(libraryID, 'zfs');
var item = yield importFileAttachment('test.png');
item.version = 5;
item.synced = true;
yield item.saveTx();
// Set file as synced
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
item.id, (yield item.attachmentModificationTime)
);
yield Zotero.Sync.Storage.Local.setSyncedHash(
item.id, (yield item.attachmentHash)
);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
});
// Simulate download of version with updated attachment
var json = yield item.toResponseJSON();
json.version = 10;
json.data.version = 10;
json.data.md5 = '57f8a4fda823187b91e1191487b87fe6';
json.data.mtime = new Date().getTime() + 10000;
yield Zotero.Sync.Data.Local.saveCacheObjects(
'item', Zotero.Libraries.userLibraryID, [json]
);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, 'item', { stopOnError: true }
);
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
})
it("should ignore attachment metadata when resolving metadata conflict", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
Zotero.Sync.Storage.Local.setModeForLibrary(libraryID, 'zfs');
var item = yield importFileAttachment('test.png');
item.version = 5;
yield item.saveTx();
var json = yield item.toResponseJSON();
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json]);
// Set file as synced
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
item.id, (yield item.attachmentModificationTime)
);
yield Zotero.Sync.Storage.Local.setSyncedHash(
item.id, (yield item.attachmentHash)
);
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
});
// Modify title locally, leaving item unsynced
var newTitle = Zotero.Utilities.randomString();
item.setField('title', newTitle);
yield item.saveTx();
// Simulate download of version with original title but updated attachment
json.version = 10;
json.data.version = 10;
json.data.md5 = '57f8a4fda823187b91e1191487b87fe6';
json.data.mtime = new Date().getTime() + 10000;
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json]);
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, 'item', { stopOnError: true }
);
assert.equal(item.getField('title'), newTitle);
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
})
}) })
describe("Conflict Resolution", function () { describe("Conflict Resolution", function () {
@ -232,7 +388,10 @@ describe("Zotero.Sync.Data.Local", function() {
jsonData.title = Zotero.Utilities.randomString(); jsonData.title = Zotero.Utilities.randomString();
yield Zotero.Sync.Data.Local.saveCacheObjects(type, libraryID, [json]); yield Zotero.Sync.Data.Local.saveCacheObjects(type, libraryID, [json]);
var windowOpened = false;
waitForWindow('chrome://zotero/content/merge.xul', function (dialog) { waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
windowOpened = true;
var doc = dialog.document; var doc = dialog.document;
var wizard = doc.documentElement; var wizard = doc.documentElement;
var mergeGroup = wizard.getElementsByTagName('zoteromergegroup')[0]; var mergeGroup = wizard.getElementsByTagName('zoteromergegroup')[0];
@ -240,12 +399,14 @@ describe("Zotero.Sync.Data.Local", function() {
// Remote version should be selected by default // Remote version should be selected by default
assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true'); assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
assert.ok(mergeGroup.leftpane.pane.onclick); assert.ok(mergeGroup.leftpane.pane.onclick);
// Select local deleted version
mergeGroup.leftpane.pane.click(); mergeGroup.leftpane.pane.click();
wizard.getButton('finish').click(); wizard.getButton('finish').click();
}) })
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType( yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true } libraryID, type, { stopOnError: true }
); );
assert.isTrue(windowOpened);
obj = objectsClass.getByLibraryAndKey(libraryID, key); obj = objectsClass.getByLibraryAndKey(libraryID, key);
assert.isFalse(obj); assert.isFalse(obj);
@ -825,15 +986,28 @@ describe("Zotero.Sync.Data.Local", function() {
assert.sameDeepMembers( assert.sameDeepMembers(
result.conflicts, result.conflicts,
[ [
{ [
field: "place", {
op: "delete" field: "place",
}, op: "add",
{ value: "Place"
field: "date", },
op: "add", {
value: "2015-05-15" field: "place",
} op: "delete"
}
],
[
{
field: "date",
op: "delete"
},
{
field: "date",
op: "add",
value: "2015-05-15"
}
]
] ]
); );
}) })
@ -1296,4 +1470,68 @@ describe("Zotero.Sync.Data.Local", function() {
}) })
}) })
}) })
describe("#reconcileChangesWithoutCache()", function () {
it("should return conflict for conflicting fields", function () {
var json1 = {
key: "AAAAAAAA",
version: 1234,
title: "Title 1",
pages: 10,
dateModified: "2015-05-14 14:12:34"
};
var json2 = {
key: "AAAAAAAA",
version: 1235,
title: "Title 2",
place: "New York",
dateModified: "2015-05-14 13:45:12"
};
var ignoreFields = ['dateAdded', 'dateModified'];
var result = Zotero.Sync.Data.Local._reconcileChangesWithoutCache(
'item', json1, json2, ignoreFields
);
assert.lengthOf(result.changes, 0);
assert.sameDeepMembers(
result.conflicts,
[
[
{
field: "title",
op: "add",
value: "Title 1"
},
{
field: "title",
op: "add",
value: "Title 2"
}
],
[
{
field: "pages",
op: "add",
value: 10
},
{
field: "pages",
op: "delete"
}
],
[
{
field: "place",
op: "delete"
},
{
field: "place",
op: "add",
value: "New York"
}
]
]
);
})
})
}) })

47
test/tests/syncRunnerTest.js
View File

@ -5,7 +5,7 @@ describe("Zotero.Sync.Runner", function () {
var apiKey = Zotero.Utilities.randomString(24); var apiKey = Zotero.Utilities.randomString(24);
var baseURL = "http://local.zotero/"; var baseURL = "http://local.zotero/";
var userLibraryID, publicationsLibraryID, runner, caller, server, client, stub, spy; var userLibraryID, publicationsLibraryID, runner, caller, server, stub, spy;
var responses = { var responses = {
keyInfo: { keyInfo: {
@ -129,15 +129,7 @@ describe("Zotero.Sync.Runner", function () {
} }
}; };
var client = new Zotero.Sync.APIClient({ return { runner, caller };
baseURL: baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
apiKey: apiKey,
concurrentCaller: caller,
background: options.background || true
});
return { runner, caller, client };
}) })
function setResponse(response) { function setResponse(response) {
@ -160,7 +152,7 @@ describe("Zotero.Sync.Runner", function () {
server = sinon.fakeServer.create(); server = sinon.fakeServer.create();
server.autoRespond = true; server.autoRespond = true;
({ runner, caller, client } = yield setup()); ({ runner, caller } = yield setup());
yield Zotero.Users.setCurrentUserID(1); yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("A"); yield Zotero.Users.setCurrentUsername("A");
@ -180,7 +172,7 @@ describe("Zotero.Sync.Runner", function () {
it("should check key access", function* () { it("should check key access", function* () {
spy = sinon.spy(runner, "checkUser"); spy = sinon.spy(runner, "checkUser");
setResponse('keyInfo.fullAccess'); setResponse('keyInfo.fullAccess');
var json = yield runner.checkAccess(client); var json = yield runner.checkAccess(runner.getAPIClient());
sinon.assert.calledWith(spy, 1, "Username"); sinon.assert.calledWith(spy, 1, "Username");
var compare = {}; var compare = {};
Object.assign(compare, responses.keyInfo.fullAccess.json); Object.assign(compare, responses.keyInfo.fullAccess.json);
@ -216,7 +208,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('userGroups.groupVersions'); setResponse('userGroups.groupVersions');
var libraries = yield runner.checkLibraries( var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
); );
assert.lengthOf(libraries, 4); assert.lengthOf(libraries, 4);
assert.sameMembers( assert.sameMembers(
@ -240,19 +232,25 @@ describe("Zotero.Sync.Runner", function () {
setResponse('userGroups.groupVersions'); setResponse('userGroups.groupVersions');
var libraries = yield runner.checkLibraries( var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json, [userLibraryID] runner.getAPIClient(), false, responses.keyInfo.fullAccess.json, [userLibraryID]
); );
assert.lengthOf(libraries, 1); assert.lengthOf(libraries, 1);
assert.sameMembers(libraries, [userLibraryID]); assert.sameMembers(libraries, [userLibraryID]);
var libraries = yield runner.checkLibraries( var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json, [userLibraryID, publicationsLibraryID] runner.getAPIClient(),
false,
responses.keyInfo.fullAccess.json,
[userLibraryID, publicationsLibraryID]
); );
assert.lengthOf(libraries, 2); assert.lengthOf(libraries, 2);
assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID]); assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID]);
var libraries = yield runner.checkLibraries( var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json, [group1.libraryID] runner.getAPIClient(),
false,
responses.keyInfo.fullAccess.json,
[group1.libraryID]
); );
assert.lengthOf(libraries, 1); assert.lengthOf(libraries, 1);
assert.sameMembers(libraries, [group1.libraryID]); assert.sameMembers(libraries, [group1.libraryID]);
@ -277,7 +275,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('groups.ownerGroup'); setResponse('groups.ownerGroup');
setResponse('groups.memberGroup'); setResponse('groups.memberGroup');
var libraries = yield runner.checkLibraries( var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
); );
assert.lengthOf(libraries, 4); assert.lengthOf(libraries, 4);
assert.sameMembers( assert.sameMembers(
@ -318,7 +316,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('groups.ownerGroup'); setResponse('groups.ownerGroup');
setResponse('groups.memberGroup'); setResponse('groups.memberGroup');
var libraries = yield runner.checkLibraries( var libraries = yield runner.checkLibraries(
client, runner.getAPIClient(),
false, false,
responses.keyInfo.fullAccess.json, responses.keyInfo.fullAccess.json,
[group1.libraryID, group2.libraryID] [group1.libraryID, group2.libraryID]
@ -339,7 +337,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('groups.ownerGroup'); setResponse('groups.ownerGroup');
setResponse('groups.memberGroup'); setResponse('groups.memberGroup');
var libraries = yield runner.checkLibraries( var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
); );
assert.lengthOf(libraries, 4); assert.lengthOf(libraries, 4);
var groupData1 = responses.groups.ownerGroup; var groupData1 = responses.groups.ownerGroup;
@ -370,7 +368,7 @@ describe("Zotero.Sync.Runner", function () {
assert.include(text, group1.name); assert.include(text, group1.name);
}); });
var libraries = yield runner.checkLibraries( var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
); );
assert.lengthOf(libraries, 3); assert.lengthOf(libraries, 3);
assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID, group2.libraryID]); assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID, group2.libraryID]);
@ -388,7 +386,7 @@ describe("Zotero.Sync.Runner", function () {
assert.include(text, group.name); assert.include(text, group.name);
}, "extra1"); }, "extra1");
var libraries = yield runner.checkLibraries( var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
); );
assert.lengthOf(libraries, 3); assert.lengthOf(libraries, 3);
assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID, group.libraryID]); assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID, group.libraryID]);
@ -405,7 +403,7 @@ describe("Zotero.Sync.Runner", function () {
assert.include(text, group.name); assert.include(text, group.name);
}, "cancel"); }, "cancel");
var libraries = yield runner.checkLibraries( var libraries = yield runner.checkLibraries(
client, false, responses.keyInfo.fullAccess.json runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
); );
assert.lengthOf(libraries, 0); assert.lengthOf(libraries, 0);
assert.isTrue(Zotero.Groups.exists(groupData.json.id)); assert.isTrue(Zotero.Groups.exists(groupData.json.id));
@ -656,6 +654,11 @@ describe("Zotero.Sync.Runner", function () {
Zotero.Libraries.getVersion(Zotero.Groups.getLibraryIDFromGroupID(2694172)), Zotero.Libraries.getVersion(Zotero.Groups.getLibraryIDFromGroupID(2694172)),
20 20
); );
// Last sync time should be within the last second
var lastSyncTime = Zotero.Sync.Data.Local.getLastSyncTime();
assert.isAbove(lastSyncTime, new Date().getTime() - 1000);
assert.isBelow(lastSyncTime, new Date().getTime());
}) })
}) })
}) })

94
test/tests/zoteroPaneTest.js
View File

@ -1,3 +1,5 @@
"use strict";
describe("ZoteroPane", function() { describe("ZoteroPane", function() {
var win, doc, zp; var win, doc, zp;
@ -90,4 +92,96 @@ describe("ZoteroPane", function() {
); );
}) })
}) })
describe("#viewAttachment", function () {
Components.utils.import("resource://zotero-unit/httpd.js");
var apiKey = Zotero.Utilities.randomString(24);
var port = 16213;
var baseURL = `http://localhost:${port}/`;
var server;
var responses = {};
var setup = Zotero.Promise.coroutine(function* (options = {}) {
server = sinon.fakeServer.create();
server.autoRespond = true;
});
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
before(function () {
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
Zotero.Sync.Runner.apiKey = apiKey;
Zotero.Sync.Runner.baseURL = baseURL;
})
beforeEach(function* () {
this.httpd = new HttpServer();
this.httpd.start(port);
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser");
})
afterEach(function* () {
var defer = new Zotero.Promise.defer();
this.httpd.stop(() => defer.resolve());
yield defer.promise;
})
it("should download an attachment on-demand", function* () {
yield setup();
Zotero.Sync.Storage.Local.downloadAsNeeded(Zotero.Libraries.userLibraryID, true);
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
// TODO: Test binary data
var text = Zotero.Utilities.randomString();
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var mtime = "1441252524000";
var md5 = Zotero.Utilities.Internal.md5(text)
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
var s3Path = `pretend-s3/${item.key}`;
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 302, "Found");
response.setHeader("Zotero-File-Modification-Time", mtime, false);
response.setHeader("Zotero-File-MD5", md5, false);
response.setHeader("Zotero-File-Compressed", "No", false);
response.setHeader("Location", baseURL + s3Path, false);
}
}
);
this.httpd.registerPathHandler(
"/" + s3Path,
{
handle: function (request, response) {
response.setStatusLine(null, 200, "OK");
response.write(text);
}
}
);
yield zp.viewAttachment(item.id);
assert.equal((yield item.attachmentHash), md5);
assert.equal((yield item.attachmentModificationTime), mtime);
var path = yield item.getFilePathAsync();
assert.equal((yield Zotero.File.getContentsAsync(path)), text);
})
})
}) })