suzanne.soy/zotero
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WebDAV file sync overhaul for 5.0

Browse Source 
Also:

- Remove last-sync-time mechanism for both WebDAV and ZFS, since it can
  be determined by storage properties (mtime/md5) in data sync
- Add option to include synced storage properties in item toJSON()
  instead of local file properties
- Set "Fake-Server-Match" header in setHTTPResponse() test support
  function, which can be used for request count assertions -- see
  resetRequestCount() and assertRequestCount() in webdavTest.js
- Allow string (e.g., 'to_download') instead of constant in
  Zotero.Sync.Data.Local.setSyncState()
- Misc storage tweaks
This commit is contained in:
Dan Stillman 2015-12-23 04:52:09 -05:00
parent 6844deba60
commit c5a9987f37
32 changed files with 3182 additions and 2789 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

296
chrome/content/zotero/preferences/preferences_sync.js
View File

@ -28,11 +28,17 @@ Components.utils.import("resource://gre/modules/Services.jsm");
Zotero_Preferences.Sync = { Zotero_Preferences.Sync = {
init: Zotero.Promise.coroutine(function* () { init: Zotero.Promise.coroutine(function* () {
this.updateStorageSettings(null, null, true); this.updateStorageSettingsUI();
var username = Zotero.Users.getCurrentUsername() || ""; var username = Zotero.Users.getCurrentUsername() || "";
var apiKey = Zotero.Sync.Data.Local.getAPIKey(); var apiKey = Zotero.Sync.Data.Local.getAPIKey();
this.displayFields(apiKey ? username : ""); this.displayFields(apiKey ? username : "");
var pass = Zotero.Sync.Runner.getStorageController('webdav').password;
if (pass) {
document.getElementById('storage-password').value = pass;
}
if (apiKey) { if (apiKey) {
try { try {
var keyInfo = yield Zotero.Sync.Runner.checkAccess( var keyInfo = yield Zotero.Sync.Runner.checkAccess(
@ -57,13 +63,6 @@ Zotero_Preferences.Sync = {
} }
} }
} }
// TEMP: Disabled
//var pass = Zotero.Sync.Storage.WebDAV.password;
//if (pass) {
// document.getElementById('storage-password').value = pass;
//}
}), }),
displayFields: function (username) { displayFields: function (username) {
@ -249,17 +248,13 @@ Zotero_Preferences.Sync = {
return true; return true;
}), }),
updateStorageSettings: function (enabled, protocol, skipWarnings) {
if (enabled === null) { updateStorageSettingsUI: Zotero.Promise.coroutine(function* () {
enabled = document.getElementById('pref-storage-enabled').value; this.unverifyStorageServer();
}
var oldProtocol = document.getElementById('pref-storage-protocol').value; var protocol = document.getElementById('pref-storage-protocol').value;
if (protocol === null) { var enabled = document.getElementById('pref-storage-enabled').value;
protocol = oldProtocol;
}
var storageSettings = document.getElementById('storage-settings'); var storageSettings = document.getElementById('storage-settings');
var protocolMenu = document.getElementById('storage-protocol'); var protocolMenu = document.getElementById('storage-protocol');
@ -275,66 +270,13 @@ Zotero_Preferences.Sync = {
sep.hidden = true; sep.hidden = true;
} }
var menulists = storageSettings.getElementsByTagName('menulist'); var menulists = document.querySelectorAll('#storage-settings menulist.storage-personal');
for each(var menulist in menulists) { for (let menulist of menulists) {
if (menulist.className == 'storage-personal') { menulist.disabled = !enabled;
menulist.disabled = !enabled;
}
} }
if (!skipWarnings) { this.updateStorageTerms();
// WARN if going between }),
}
if (oldProtocol == 'zotero' && protocol == 'webdav') {
var sql = "SELECT COUNT(*) FROM version WHERE schema LIKE 'storage_zfs%'";
if (Zotero.DB.valueQuery(sql)) {
var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"]
.getService(Components.interfaces.nsIPromptService);
var buttonFlags = (ps.BUTTON_POS_0) * (ps.BUTTON_TITLE_IS_STRING)
+ (ps.BUTTON_POS_1) * (ps.BUTTON_TITLE_IS_STRING)
+ ps.BUTTON_DELAY_ENABLE;
var account = Zotero.Sync.Server.username;
var index = ps.confirmEx(
null,
Zotero.getString('zotero.preferences.sync.purgeStorage.title'),
Zotero.getString('zotero.preferences.sync.purgeStorage.desc'),
buttonFlags,
Zotero.getString('zotero.preferences.sync.purgeStorage.confirmButton'),
Zotero.getString('zotero.preferences.sync.purgeStorage.cancelButton'), null, null, {}
);
if (index == 0) {
var sql = "INSERT OR IGNORE INTO settings VALUES (?,?,?)";
Zotero.DB.query(sql, ['storage', 'zfsPurge', 'user']);
Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles()
.then(function () {
ps.alert(
null,
Zotero.getString("general.success"),
"Attachment files from your personal library have been removed from the Zotero servers."
);
})
.catch(function (e) {
Zotero.debug(e, 1);
Components.utils.reportError(e);
ps.alert(
null,
Zotero.getString("general.error"),
"An error occurred. Please try again later."
);
});
}
}
}
var self = this;
setTimeout(function () {
self.updateStorageTerms();
}, 1)
},
updateStorageSettingsGroups: function (enabled) { updateStorageSettingsGroups: function (enabled) {
@ -364,14 +306,90 @@ Zotero_Preferences.Sync = {
}, },
unverifyStorageServer: function () { onStorageSettingsKeyPress: Zotero.Promise.coroutine(function* (event) {
Zotero.Prefs.set('sync.storage.verified', false); if (event.keyCode == 13) {
Zotero.Sync.Storage.WebDAV.clearCachedCredentials(); yield this.onStorageSettingsChange();
Zotero.Sync.Storage.resetAllSyncStates(null, true, false); yield this.verifyStorageServer();
}, }
}),
verifyStorageServer: function () { onStorageSettingsChange: Zotero.Promise.coroutine(function* () {
// Clean URL
var urlPref = document.getElementById('pref-storage-url');
urlPref.value = urlPref.value.replace(/(^https?:\/\/|\/zotero\/?$|\/$)/g, '');
var oldProtocol = document.getElementById('pref-storage-protocol').value;
var oldEnabled = document.getElementById('pref-storage-enabled').value;
yield Zotero.Promise.delay(1);
var newProtocol = document.getElementById('pref-storage-protocol').value;
var newEnabled = document.getElementById('pref-storage-enabled').value;
if (oldProtocol != newProtocol) {
yield Zotero.Sync.Storage.Local.resetModeSyncStates(oldProtocol);
}
if (oldProtocol == 'webdav') {
this.unverifyStorageServer();
Zotero.Sync.Runner.resetStorageController(oldProtocol);
var username = document.getElementById('storage-username').value;
var password = document.getElementById('storage-password').value;
if (username) {
Zotero.Sync.Runner.getStorageController('webdav').password = password;
}
}
if (oldProtocol == 'zotero' && newProtocol == 'webdav') {
var sql = "SELECT COUNT(*) FROM settings "
+ "WHERE schema='storage' AND key='zfsPurge' AND value='user'";
if (!Zotero.DB.valueQueryAsync(sql)) {
var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"]
.getService(Components.interfaces.nsIPromptService);
var buttonFlags = (ps.BUTTON_POS_0) * (ps.BUTTON_TITLE_IS_STRING)
+ (ps.BUTTON_POS_1) * (ps.BUTTON_TITLE_IS_STRING)
+ ps.BUTTON_DELAY_ENABLE;
var account = Zotero.Sync.Server.username;
var index = ps.confirmEx(
null,
Zotero.getString('zotero.preferences.sync.purgeStorage.title'),
Zotero.getString('zotero.preferences.sync.purgeStorage.desc'),
buttonFlags,
Zotero.getString('zotero.preferences.sync.purgeStorage.confirmButton'),
Zotero.getString('zotero.preferences.sync.purgeStorage.cancelButton'), null, null, {}
);
if (index == 0) {
var sql = "INSERT OR IGNORE INTO settings VALUES (?,?,?)";
yield Zotero.DB.queryAsync(sql, ['storage', 'zfsPurge', 'user']);
try {
yield Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles();
ps.alert(
null,
Zotero.getString("general.success"),
"Attachment files from your personal library have been removed from the Zotero servers."
);
}
catch (e) {
Zotero.logError(e);
ps.alert(
null,
Zotero.getString("general.error"),
"An error occurred. Please try again later."
);
}
}
}
}
this.updateStorageSettingsUI();
}),
verifyStorageServer: Zotero.Promise.coroutine(function* () {
Zotero.debug("Verifying storage"); Zotero.debug("Verifying storage");
var verifyButton = document.getElementById("storage-verify"); var verifyButton = document.getElementById("storage-verify");
@ -385,78 +403,56 @@ Zotero_Preferences.Sync = {
abortButton.hidden = false; abortButton.hidden = false;
progressMeter.hidden = false; progressMeter.hidden = false;
var success = false;
var request = null; var request = null;
var onDone = false;
Zotero.Sync.Storage.WebDAV.checkServer() var controller = Zotero.Sync.Runner.getStorageController('webdav');
// Get the XMLHttpRequest for possible cancelling
.progress(function (obj) { try {
request = obj.xmlhttp; yield controller.checkServer({
}) // Get the XMLHttpRequest for possible cancelling
.finally(function () { onRequest: r => request = r
})
success = true;
}
catch (e) {
if (e instanceof controller.VerificationError) {
switch (e.error) {
case "NO_URL":
urlField.focus();
break;
case "NO_USERNAME":
usernameField.focus();
break;
case "NO_PASSWORD":
case "AUTH_FAILED":
passwordField.focus();
break;
}
}
success = yield controller.handleVerificationError(e);
}
finally {
verifyButton.hidden = false; verifyButton.hidden = false;
abortButton.hidden = true; abortButton.hidden = true;
progressMeter.hidden = true; progressMeter.hidden = true;
}) }
.spread(function (uri, status) {
switch (status) { if (success) {
case Zotero.Sync.Storage.ERROR_NO_URL: Zotero.debug("WebDAV verification succeeded");
onDone = function () {
urlField.focus();
};
break;
case Zotero.Sync.Storage.ERROR_NO_USERNAME:
onDone = function () {
usernameField.focus();
};
break;
case Zotero.Sync.Storage.ERROR_NO_PASSWORD:
case Zotero.Sync.Storage.ERROR_AUTH_FAILED:
onDone = function () {
passwordField.focus();
};
break;
}
return Zotero.Sync.Storage.WebDAV.checkServerCallback(uri, status, window); Zotero.alert(
}) window,
.then(function (success) { Zotero.getString('sync.storage.serverConfigurationVerified'),
if (success) { Zotero.getString('sync.storage.fileSyncSetUp')
Zotero.debug("WebDAV verification succeeded"); );
}
var promptService = Components.classes["@mozilla.org/embedcomp/prompt-service;1"] else {
.getService(Components.interfaces.nsIPromptService); Zotero.logError("WebDAV verification failed");
promptService.alert( }
window,
Zotero.getString('sync.storage.serverConfigurationVerified'),
Zotero.getString('sync.storage.fileSyncSetUp')
);
Zotero.Prefs.set("sync.storage.verified", true);
}
else {
Zotero.debug("WebDAV verification failed");
if (onDone) {
setTimeout(function () {
onDone();
}, 1);
}
}
})
.catch(function (e) {
Zotero.debug("WebDAV verification failed");
Zotero.debug(e, 1);
Components.utils.reportError(e);
Zotero.Utilities.Internal.errorPrompt(Zotero.getString('general.error'), e);
if (onDone) {
setTimeout(function () {
onDone();
}, 1);
}
})
.done();
abortButton.onclick = function () { abortButton.onclick = function () {
if (request) { if (request) {
@ -468,6 +464,12 @@ Zotero_Preferences.Sync = {
progressMeter.hidden = true; progressMeter.hidden = true;
} }
} }
}),
unverifyStorageServer: function () {
Zotero.debug("Unverifying storage");
Zotero.Prefs.set('sync.storage.verified', false);
}, },

37
chrome/content/zotero/preferences/preferences_sync.xul
View File

@ -34,8 +34,7 @@
<preference id="pref-sync-username" name="extensions.zotero.sync.server.username" type="unichar" instantApply="true"/> <preference id="pref-sync-username" name="extensions.zotero.sync.server.username" type="unichar" instantApply="true"/>
<preference id="pref-sync-fulltext-enabled" name="extensions.zotero.sync.fulltext.enabled" type="bool"/> <preference id="pref-sync-fulltext-enabled" name="extensions.zotero.sync.fulltext.enabled" type="bool"/>
<preference id="pref-storage-enabled" name="extensions.zotero.sync.storage.enabled" type="bool"/> <preference id="pref-storage-enabled" name="extensions.zotero.sync.storage.enabled" type="bool"/>
<preference id="pref-storage-protocol" name="extensions.zotero.sync.storage.protocol" type="string" <preference id="pref-storage-protocol" name="extensions.zotero.sync.storage.protocol" type="string"/>
onchange="Zotero_Preferences.Sync.unverifyStorageServer()"/>
<preference id="pref-storage-scheme" name="extensions.zotero.sync.storage.scheme" type="string" instantApply="true"/> <preference id="pref-storage-scheme" name="extensions.zotero.sync.storage.scheme" type="string" instantApply="true"/>
<preference id="pref-storage-url" name="extensions.zotero.sync.storage.url" type="string"/> <preference id="pref-storage-url" name="extensions.zotero.sync.storage.url" type="string"/>
<preference id="pref-storage-username" name="extensions.zotero.sync.storage.username" type="string"/> <preference id="pref-storage-username" name="extensions.zotero.sync.storage.username" type="string"/>
@ -152,14 +151,14 @@
<hbox> <hbox>
<checkbox label="&zotero.preferences.sync.fileSyncing.myLibrary;" <checkbox label="&zotero.preferences.sync.fileSyncing.myLibrary;"
preference="pref-storage-enabled" preference="pref-storage-enabled"
oncommand="Zotero_Preferences.Sync.updateStorageSettings(this.checked, null)"/> oncommand="Zotero_Preferences.Sync.onStorageSettingsChange()"/>
<menulist id="storage-protocol" class="storage-personal" <menulist id="storage-protocol" class="storage-personal"
style="margin-left: .5em" style="margin-left: .5em"
preference="pref-storage-protocol" preference="pref-storage-protocol"
oncommand="Zotero_Preferences.Sync.updateStorageSettings(null, this.value)"> oncommand="Zotero_Preferences.Sync.onStorageSettingsChange()">
<menupopup> <menupopup>
<menuitem label="Zotero" value="zotero"/> <menuitem label="Zotero" value="zotero"/>
<menuitem label="WebDAV" value="webdav" disabled="true"/><!-- TEMP --> <menuitem label="WebDAV" value="webdav"/>
</menupopup> </menupopup>
</menulist> </menulist>
</hbox> </hbox>
@ -190,13 +189,8 @@
<label value="://"/> <label value="://"/>
<textbox id="storage-url" flex="1" <textbox id="storage-url" flex="1"
preference="pref-storage-url" preference="pref-storage-url"
onkeypress="if (Zotero.isMac &amp;&amp; event.keyCode == 13) { onkeypress="Zotero_Preferences.Sync.onStorageSettingsKeyPress(event)"
this.blur(); onchange="Zotero_Preferences.Sync.onStorageSettingsChange()"/>
setTimeout(Zotero_Preferences.Sync.verifyStorageServer, 1);
}"
onchange="Zotero_Preferences.Sync.unverifyStorageServer();
this.value = this.value.replace(/(^https?:\/\/|\/zotero\/?$|\/$)/g, '');
Zotero.Prefs.set('sync.storage.url', this.value)"/>
<label value="/zotero/"/> <label value="/zotero/"/>
</hbox> </hbox>
</row> </row>
@ -205,27 +199,16 @@
<hbox> <hbox>
<textbox id="storage-username" <textbox id="storage-username"
preference="pref-storage-username" preference="pref-storage-username"
onkeypress="if (Zotero.isMac &amp;&amp; event.keyCode == 13) { onkeypress="Zotero_Preferences.Sync.onStorageSettingsKeyPress(event)"
this.blur(); onchange="Zotero_Preferences.Sync.onStorageSettingsChange()"/>
setTimeout(Zotero_Preferences.Sync.verifyStorageServer, 1); }"
onchange="Zotero_Preferences.Sync.unverifyStorageServer();
Zotero.Prefs.set('sync.storage.username', this.value);
var pass = document.getElementById('storage-password');
if (pass.value) {
Zotero.Sync.Storage.WebDAV.password = pass.value;
}"/>
</hbox> </hbox>
</row> </row>
<row> <row>
<label value="&zotero.preferences.sync.password;"/> <label value="&zotero.preferences.sync.password;"/>
<hbox> <hbox>
<textbox id="storage-password" flex="0" type="password" <textbox id="storage-password" flex="0" type="password"
onkeypress="if (Zotero.isMac &amp;&amp; event.keyCode == 13) { onkeypress="Zotero_Preferences.Sync.onStorageSettingsKeyPress(event)"
this.blur(); onchange="Zotero_Preferences.Sync.onStorageSettingsChange()"/>
setTimeout(Zotero_Preferences.Sync.verifyStorageServer, 1);
}"
onchange="Zotero_Preferences.Sync.unverifyStorageServer();
Zotero.Sync.Storage.WebDAV.password = this.value;"/>
</hbox> </hbox>
</row> </row>
<row> <row>

28
chrome/content/zotero/xpcom/data/item.js
View File

@ -2294,9 +2294,7 @@ Zotero.Item.prototype.renameAttachmentFile = Zotero.Promise.coroutine(function*
yield Zotero.DB.executeTransaction(function* () { yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(this.id, null, false); yield Zotero.Sync.Storage.Local.setSyncedHash(this.id, null, false);
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(this.id, "to_upload");
this.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
);
}.bind(this)); }.bind(this));
return true; return true;
@ -2692,11 +2690,12 @@ Zotero.defineProperty(Zotero.Item.prototype, 'attachmentSyncState', {
} }
switch (val) { switch (val) {
case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD: case Zotero.Sync.Storage.Local.SYNC_STATE_TO_UPLOAD:
case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD: case Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD:
case Zotero.Sync.Storage.SYNC_STATE_IN_SYNC: case Zotero.Sync.Storage.Local.SYNC_STATE_IN_SYNC:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD: case Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_UPLOAD:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD: case Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_DOWNLOAD:
case Zotero.Sync.Storage.Local.SYNC_STATE_IN_CONFLICT:
break; break;
default: default:
@ -3670,6 +3669,9 @@ Zotero.Item.prototype._eraseData = Zotero.Promise.coroutine(function* (env) {
Components.utils.reportError(e); Components.utils.reportError(e);
} }
} }
// Zotero.Sync.EventListeners.ChangeListener needs to know if this was a storage file
env.notifierData[this.id].storageDeleteLog = this.isImportedAttachment();
} }
// Regular item // Regular item
else { else {
@ -3905,8 +3907,14 @@ Zotero.Item.prototype.toJSON = Zotero.Promise.coroutine(function* (options = {})
} }
if (this.isFileAttachment()) { if (this.isFileAttachment()) {
obj.mtime = (yield this.attachmentModificationTime) || null; if (options.syncedStorageProperties) {
obj.md5 = (yield this.attachmentHash) || null; obj.mtime = yield Zotero.Sync.Storage.Local.getSyncedModificationTime(this.id);
obj.md5 = yield Zotero.Sync.Storage.Local.getSyncedHash(this.id);
}
else {
obj.mtime = (yield this.attachmentModificationTime) || null;
obj.md5 = (yield this.attachmentHash) || null;
}
} }
} }

45
chrome/content/zotero/xpcom/data/library.js
View File

@ -33,6 +33,7 @@ Zotero.Library = function(params = {}) {
this._hasCollections = null; this._hasCollections = null;
this._hasSearches = null; this._hasSearches = null;
this._storageDownloadNeeded = false;
Zotero.Utilities.assignProps( Zotero.Utilities.assignProps(
this, this,
@ -42,8 +43,8 @@ Zotero.Library = function(params = {}) {
'editable', 'editable',
'filesEditable', 'filesEditable',
'libraryVersion', 'libraryVersion',
'storageVersion',
'lastSync', 'lastSync',
'lastStorageSync'
] ]
); );
@ -64,7 +65,7 @@ Zotero.Library = function(params = {}) {
// DB columns // DB columns
Zotero.defineProperty(Zotero.Library, '_dbColumns', { Zotero.defineProperty(Zotero.Library, '_dbColumns', {
value: Object.freeze([ value: Object.freeze([
'type', 'editable', 'filesEditable', 'version', 'lastSync', 'lastStorageSync' 'type', 'editable', 'filesEditable', 'version', 'storageVersion', 'lastSync'
]) ])
}); });
@ -172,7 +173,7 @@ Zotero.defineProperty(Zotero.Library.prototype, 'hasTrash', {
// Create other accessors // Create other accessors
(function() { (function() {
let accessors = ['editable', 'filesEditable', 'lastStorageSync']; let accessors = ['editable', 'filesEditable', 'storageVersion'];
for (let i=0; i<accessors.length; i++) { for (let i=0; i<accessors.length; i++) {
let prop = Zotero.Library._colToProp(accessors[i]); let prop = Zotero.Library._colToProp(accessors[i]);
Zotero.defineProperty(Zotero.Library.prototype, accessors[i], { Zotero.defineProperty(Zotero.Library.prototype, accessors[i], {
@ -182,6 +183,11 @@ Zotero.defineProperty(Zotero.Library.prototype, 'hasTrash', {
} }
})() })()
Zotero.defineProperty(Zotero.Library.prototype, 'storageDownloadNeeded', {
get: function () { return this._storageDownloadNeeded; },
set: function (val) { this._storageDownloadNeeded = !!val; },
})
Zotero.Library.prototype._isValidProp = function(prop) { Zotero.Library.prototype._isValidProp = function(prop) {
let prefix = '_library'; let prefix = '_library';
if (prop.indexOf(prefix) !== 0 || prop.length == prefix.length) { if (prop.indexOf(prefix) !== 0 || prop.length == prefix.length) {
@ -235,8 +241,10 @@ Zotero.Library.prototype._set = function(prop, val) {
val = !!val; val = !!val;
break; break;
case '_libraryVersion': case '_libraryVersion':
let newVal = Number.parseInt(val, 10); var newVal = Number.parseInt(val, 10);
if (newVal != val) throw new Error(prop + ' must be an integer'); if (newVal != val) {
throw new Error(`${prop} must be an integer (${typeof val} '${val}' given)`);
}
val = newVal val = newVal
// Allow -1 to indicate that a full sync is needed // Allow -1 to indicate that a full sync is needed
@ -247,6 +255,17 @@ Zotero.Library.prototype._set = function(prop, val) {
break; break;
case '_libraryStorageVersion':
var newVal = parseInt(val);
if (newVal != val) {
throw new Error(`${prop} must be an integer (${typeof val} '${val}' given)`);
}
val = newVal;
// Ensure that it is never decreasing
if (val < this._libraryStorageVersion) throw new Error(prop + ' cannot decrease');
break;
case '_libraryLastSync': case '_libraryLastSync':
if (!val) { if (!val) {
val = false; val = false;
@ -257,18 +276,6 @@ Zotero.Library.prototype._set = function(prop, val) {
val = new Date(Math.floor(val.getTime()/1000) * 1000); val = new Date(Math.floor(val.getTime()/1000) * 1000);
} }
break; break;
case '_libraryLastStorageSync':
if (parseInt(val) != val) {
Zotero.debug(val);
throw new Error("timestamp must be an integer");
}
if (val > 9999999999) {
Zotero.debug(val);
throw new Error("timestamp must be in seconds");
}
val = parseInt(val);
break;
} }
if (this[prop] == val) return; // Unchanged if (this[prop] == val) return; // Unchanged
@ -293,8 +300,8 @@ Zotero.Library.prototype._loadDataFromRow = function(row) {
this._libraryEditable = !!row._libraryEditable; this._libraryEditable = !!row._libraryEditable;
this._libraryFilesEditable = !!row._libraryFilesEditable; this._libraryFilesEditable = !!row._libraryFilesEditable;
this._libraryVersion = row._libraryVersion; this._libraryVersion = row._libraryVersion;
this._libraryStorageVersion = row._libraryStorageVersion;
this._libraryLastSync = row._libraryLastSync !== 0 ? new Date(row._libraryLastSync * 1000) : false; this._libraryLastSync = row._libraryLastSync !== 0 ? new Date(row._libraryLastSync * 1000) : false;
this._libraryLastStorageSync = row._libraryLastStorageSync || false;
this._hasCollections = !!row.hasCollections; this._hasCollections = !!row.hasCollections;
this._hasSearches = !!row.hasSearches; this._hasSearches = !!row.hasSearches;
@ -394,7 +401,7 @@ Zotero.Library.prototype._initSave = Zotero.Promise.method(function(env) {
Zotero.Libraries._ensureExists(this._libraryID); Zotero.Libraries._ensureExists(this._libraryID);
if (!Object.keys(this._changed).length) { if (!Object.keys(this._changed).length) {
Zotero.debug("No data changed in " + this._objectType + " " + this.id + ". Not saving.", 4); Zotero.debug(`No data changed in ${this._objectType} ${this.id} -- not saving`, 4);
return false; return false;
} }
} }

8
chrome/content/zotero/xpcom/file.js
View File

@ -666,11 +666,11 @@ Zotero.File = new function(){
var zw = Components.classes["@mozilla.org/zipwriter;1"] var zw = Components.classes["@mozilla.org/zipwriter;1"]
.createInstance(Components.interfaces.nsIZipWriter); .createInstance(Components.interfaces.nsIZipWriter);
zw.open(this.pathToFile(zipPath), 0x04 | 0x08 | 0x20); // open rw, create, truncate zw.open(this.pathToFile(zipPath), 0x04 | 0x08 | 0x20); // open rw, create, truncate
var entries = yield _zipDirectory(dirPath, dirPath, zw); var entries = yield _addZipEntries(dirPath, dirPath, zw);
if (entries.length == 0) { if (entries.length == 0) {
Zotero.debug('No files to add -- removing ZIP file'); Zotero.debug('No files to add -- removing ZIP file');
zw.close(); zw.close();
zipPath.remove(null); yield OS.File.remove(zipPath);
return false; return false;
} }
@ -716,7 +716,7 @@ Zotero.File = new function(){
}); });
var _zipDirectory = Zotero.Promise.coroutine(function* (rootPath, path, zipWriter) { var _addZipEntries = Zotero.Promise.coroutine(function* (rootPath, path, zipWriter) {
var entries = []; var entries = [];
let iterator; let iterator;
try { try {
@ -727,7 +727,7 @@ Zotero.File = new function(){
return; return;
} }
if (entry.isDir) { if (entry.isDir) {
entries.concat(yield _zipDirectory(rootPath, path, zipWriter)); entries.concat(yield _addZipEntries(rootPath, path, zipWriter));
return; return;
} }
if (entry.name.startsWith('.')) { if (entry.name.startsWith('.')) {

111
chrome/content/zotero/xpcom/http.js
View File

@ -635,74 +635,6 @@ Zotero.HTTP = new function() {
this.WebDAV = {}; this.WebDAV = {};
/**
* Send a WebDAV PROP* request via XMLHTTPRequest
*
* Returns false if browser is offline
*
* @param {String} method PROPFIND or PROPPATCH
* @param {nsIURI} uri
* @param {String} body XML string
* @param {Function} callback
* @param {Object} requestHeaders e.g. { Depth: 0 }
*/
this.WebDAV.doProp = function (method, uri, body, callback, requestHeaders) {
switch (method) {
case 'PROPFIND':
case 'PROPPATCH':
break;
default:
throw ("Invalid method '" + method
+ "' in Zotero.HTTP.doProp");
}
if (requestHeaders && requestHeaders.depth != undefined) {
var depth = requestHeaders.depth;
}
// Don't display password in console
var disp = Zotero.HTTP.getDisplayURI(uri);
var bodyStart = body.substr(0, 1024);
Zotero.debug("HTTP " + method + " "
+ (depth != undefined ? "(depth " + depth + ") " : "")
+ (body.length > 1024 ?
bodyStart + "... (" + body.length + " chars)" : bodyStart)
+ " to " + disp.spec);
if (Zotero.HTTP.browserIsOffline()) {
Zotero.debug("Browser is offline", 2);
return false;
}
var xmlhttp = Components.classes["@mozilla.org/xmlextras/xmlhttprequest;1"]
.createInstance();
// Prevent certificate/authentication dialogs from popping up
xmlhttp.mozBackgroundRequest = true;
xmlhttp.open(method, uri.spec, true);
if (requestHeaders) {
for (var header in requestHeaders) {
xmlhttp.setRequestHeader(header, requestHeaders[header]);
}
}
xmlhttp.setRequestHeader("Content-Type", 'text/xml; charset="utf-8"');
var useMethodjit = Components.utils.methodjit;
/** @ignore */
xmlhttp.onreadystatechange = function() {
// XXX Remove when we drop support for Fx <24
if(useMethodjit !== undefined) Components.utils.methodjit = useMethodjit;
_stateChange(xmlhttp, callback);
};
xmlhttp.send(body);
return xmlhttp;
}
/** /**
* Send a WebDAV MKCOL request via XMLHTTPRequest * Send a WebDAV MKCOL request via XMLHTTPRequest
* *
@ -736,49 +668,6 @@ Zotero.HTTP = new function() {
} }
/**
* Send a WebDAV PUT request via XMLHTTPRequest
*
* @param {nsIURI} url
* @param {String} body String body to PUT
* @param {Function} onDone
* @return {XMLHTTPRequest}
*/
this.WebDAV.doPut = function (uri, body, callback) {
// Don't display password in console
var disp = Zotero.HTTP.getDisplayURI(uri);
var bodyStart = "'" + body.substr(0, 1024) + "'";
Zotero.debug("HTTP PUT "
+ (body.length > 1024 ?
bodyStart + "... (" + body.length + " chars)" : bodyStart)
+ " to " + disp.spec);
if (Zotero.HTTP.browserIsOffline()) {
return false;
}
var xmlhttp = Components.classes["@mozilla.org/xmlextras/xmlhttprequest;1"]
.createInstance();
// Prevent certificate/authentication dialogs from popping up
xmlhttp.mozBackgroundRequest = true;
xmlhttp.open("PUT", uri.spec, true);
// Some servers (e.g., Jungle Disk DAV) return a 200 response code
// with Content-Length: 0, which triggers a "no element found" error
// in Firefox, so we override to text
xmlhttp.overrideMimeType("text/plain");
var useMethodjit = Components.utils.methodjit;
/** @ignore */
xmlhttp.onreadystatechange = function() {
// XXX Remove when we drop support for Fx <24
if(useMethodjit !== undefined) Components.utils.methodjit = useMethodjit;
_stateChange(xmlhttp, callback);
};
xmlhttp.send(body);
return xmlhttp;
}
/** /**
* Send a WebDAV PUT request via XMLHTTPRequest * Send a WebDAV PUT request via XMLHTTPRequest
* *

14
chrome/content/zotero/xpcom/schema.js
View File

@ -33,7 +33,7 @@ Zotero.Schema = new function(){
var _dbVersions = []; var _dbVersions = [];
var _schemaVersions = []; var _schemaVersions = [];
var _maxCompatibility = 1; var _maxCompatibility = 2;
var _repositoryTimer; var _repositoryTimer;
var _remoteUpdateInProgress = false, _localUpdateInProgress = false; var _remoteUpdateInProgress = false, _localUpdateInProgress = false;
@ -2280,6 +2280,18 @@ Zotero.Schema = new function(){
yield Zotero.DB.queryAsync("CREATE TABLE feeds (\n libraryID INTEGER PRIMARY KEY,\n name TEXT NOT NULL,\n url TEXT NOT NULL UNIQUE,\n lastUpdate TIMESTAMP,\n lastCheck TIMESTAMP,\n lastCheckError TEXT,\n cleanupAfter INT,\n refreshInterval INT,\n FOREIGN KEY (libraryID) REFERENCES libraries(libraryID) ON DELETE CASCADE\n)"); yield Zotero.DB.queryAsync("CREATE TABLE feeds (\n libraryID INTEGER PRIMARY KEY,\n name TEXT NOT NULL,\n url TEXT NOT NULL UNIQUE,\n lastUpdate TIMESTAMP,\n lastCheck TIMESTAMP,\n lastCheckError TEXT,\n cleanupAfter INT,\n refreshInterval INT,\n FOREIGN KEY (libraryID) REFERENCES libraries(libraryID) ON DELETE CASCADE\n)");
yield Zotero.DB.queryAsync("CREATE TABLE feedItems (\n itemID INTEGER PRIMARY KEY,\n guid TEXT NOT NULL UNIQUE,\n readTime TIMESTAMP,\n FOREIGN KEY (itemID) REFERENCES items(itemID) ON DELETE CASCADE\n)"); yield Zotero.DB.queryAsync("CREATE TABLE feedItems (\n itemID INTEGER PRIMARY KEY,\n guid TEXT NOT NULL UNIQUE,\n readTime TIMESTAMP,\n FOREIGN KEY (itemID) REFERENCES items(itemID) ON DELETE CASCADE\n)");
} }
if (i == 81) {
yield _updateDBVersion('compatibility', 2);
yield Zotero.DB.queryAsync("ALTER TABLE libraries RENAME TO librariesOld");
yield Zotero.DB.queryAsync("CREATE TABLE libraries (\n libraryID INTEGER PRIMARY KEY,\n type TEXT NOT NULL,\n editable INT NOT NULL,\n filesEditable INT NOT NULL,\n version INT NOT NULL DEFAULT 0,\n storageVersion INT NOT NULL DEFAULT 0,\n lastSync INT NOT NULL DEFAULT 0\n)");
yield Zotero.DB.queryAsync("INSERT INTO libraries SELECT libraryID, type, editable, filesEditable, version, 0, lastSync FROM librariesOld");
yield Zotero.DB.queryAsync("DROP TABLE librariesOld");
yield Zotero.DB.queryAsync("PRAGMA foreign_keys = ON");
yield Zotero.DB.queryAsync("DELETE FROM version WHERE schema LIKE ?", "storage_%");
}
} }
yield _updateDBVersion('userdata', toVersion); yield _updateDBVersion('userdata', toVersion);

113
chrome/content/zotero/xpcom/storage.js
View File

@ -25,34 +25,6 @@
Zotero.Sync.Storage = new function () { Zotero.Sync.Storage = new function () {
//
// Constants
//
this.SYNC_STATE_TO_UPLOAD = 0;
this.SYNC_STATE_TO_DOWNLOAD = 1;
this.SYNC_STATE_IN_SYNC = 2;
this.SYNC_STATE_FORCE_UPLOAD = 3;
this.SYNC_STATE_FORCE_DOWNLOAD = 4;
this.SYNC_STATE_IN_CONFLICT = 5;
this.SUCCESS = 1;
this.ERROR_NO_URL = -1;
this.ERROR_NO_USERNAME = -2;
this.ERROR_NO_PASSWORD = -3;
this.ERROR_OFFLINE = -4;
this.ERROR_UNREACHABLE = -5;
this.ERROR_SERVER_ERROR = -6;
this.ERROR_NOT_DAV = -7;
this.ERROR_BAD_REQUEST = -8;
this.ERROR_AUTH_FAILED = -9;
this.ERROR_FORBIDDEN = -10;
this.ERROR_PARENT_DIR_NOT_FOUND = -11;
this.ERROR_ZOTERO_DIR_NOT_FOUND = -12;
this.ERROR_ZOTERO_DIR_NOT_WRITABLE = -13;
this.ERROR_NOT_ALLOWED = -14;
this.ERROR_UNKNOWN = -15;
this.ERROR_FILE_MISSING_AFTER_UPLOAD = -16;
this.ERROR_NONEXISTENT_FILE_NOT_MISSING = -17;
// TEMP // TEMP
this.__defineGetter__("defaultError", function () Zotero.getString('sync.storage.error.default', Zotero.appName)); this.__defineGetter__("defaultError", function () Zotero.getString('sync.storage.error.default', Zotero.appName));
@ -111,40 +83,6 @@ Zotero.Sync.Storage = new function () {
} }
this.checkServerPromise = function (mode) {
return mode.checkServer()
.spread(function (uri, status) {
var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
var lastWin = wm.getMostRecentWindow("navigator:browser");
var success = mode.checkServerCallback(uri, status, lastWin, true);
if (!success) {
Zotero.debug(mode.name + " verification failed");
var e = new Zotero.Error(
Zotero.getString('sync.storage.error.verificationFailed', mode.name),
0,
{
dialogButtonText: Zotero.getString('sync.openSyncPreferences'),
dialogButtonCallback: function () {
var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
var lastWin = wm.getMostRecentWindow("navigator:browser");
lastWin.ZoteroPane.openPreferences('zotero-prefpane-sync');
}
}
);
throw e;
}
})
.then(function () {
Zotero.debug(mode.name + " file sync is successfully set up");
Zotero.Prefs.set("sync.storage.verified", true);
});
}
this.getItemDownloadImageNumber = function (item) { this.getItemDownloadImageNumber = function (item) {
var numImages = 64; var numImages = 64;
@ -191,57 +129,6 @@ Zotero.Sync.Storage = new function () {
} }
this.resetAllSyncStates = function (syncState, includeUserFiles, includeGroupFiles) {
if (!includeUserFiles && !includeGroupFiles) {
includeUserFiles = true;
includeGroupFiles = true;
}
if (!syncState) {
syncState = this.SYNC_STATE_TO_UPLOAD;
}
switch (syncState) {
case this.SYNC_STATE_TO_UPLOAD:
case this.SYNC_STATE_TO_DOWNLOAD:
case this.SYNC_STATE_IN_SYNC:
break;
default:
throw ("Invalid sync state '" + syncState + "' in "
+ "Zotero.Sync.Storage.resetAllSyncStates()");
}
//var sql = "UPDATE itemAttachments SET syncState=?, storageModTime=NULL, storageHash=NULL";
var sql = "UPDATE itemAttachments SET syncState=?";
var params = [syncState];
if (includeUserFiles && !includeGroupFiles) {
sql += " WHERE itemID IN (SELECT itemID FROM items WHERE libraryID = ?)";
params.push(Zotero.Libraries.userLibraryID);
}
else if (!includeUserFiles && includeGroupFiles) {
sql += " WHERE itemID IN (SELECT itemID FROM items WHERE libraryID != ?)";
params.push(Zotero.Libraries.userLibraryID);
}
Zotero.DB.query(sql, [syncState]);
var sql = "DELETE FROM version WHERE schema LIKE 'storage_%'";
Zotero.DB.query(sql);
}
function error(e) { function error(e) {
if (_syncInProgress) { if (_syncInProgress) {
Zotero.Sync.Storage.QueueManager.cancel(true); Zotero.Sync.Storage.QueueManager.cancel(true);

109
chrome/content/zotero/xpcom/storage/storageEngine.js
View File

@ -32,31 +32,29 @@ if (!Zotero.Sync.Storage) {
* An Engine manages file sync processes for a given library * An Engine manages file sync processes for a given library
* *
* @param {Object} options * @param {Object} options
* @param {Zotero.Sync.APIClient} options.apiClient
* @param {Integer} options.libraryID * @param {Integer} options.libraryID
* @param {Object} options.controller - Storage controller instance (ZFS_Controller/WebDAV_Controller)
* @param {Function} [onError] - Function to run on error * @param {Function} [onError] - Function to run on error
* @param {Boolean} [stopOnError] * @param {Boolean} [stopOnError]
*/ */
Zotero.Sync.Storage.Engine = function (options) { Zotero.Sync.Storage.Engine = function (options) {
if (options.apiClient == undefined) {
throw new Error("options.apiClient not set");
}
if (options.libraryID == undefined) { if (options.libraryID == undefined) {
throw new Error("options.libraryID not set"); throw new Error("options.libraryID not set");
} }
if (options.controller == undefined) {
throw new Error("options.controller not set");
}
this.apiClient = options.apiClient;
this.background = options.background; this.background = options.background;
this.firstInSession = options.firstInSession; this.firstInSession = options.firstInSession;
this.lastFullFileCheck = options.lastFullFileCheck; this.lastFullFileCheck = options.lastFullFileCheck;
this.libraryID = options.libraryID; this.libraryID = options.libraryID;
this.library = Zotero.Libraries.get(options.libraryID); this.library = Zotero.Libraries.get(options.libraryID);
this.controller = options.controller;
this.local = Zotero.Sync.Storage.Local; this.local = Zotero.Sync.Storage.Local;
this.utils = Zotero.Sync.Storage.Utilities; this.utils = Zotero.Sync.Storage.Utilities;
this.mode = this.local.getModeForLibrary(this.libraryID);
var modeClass = this.utils.getClassForMode(this.mode);
this.controller = new modeClass(options);
this.setStatus = options.setStatus || function () {}; this.setStatus = options.setStatus || function () {};
this.onError = options.onError || function (e) {}; this.onError = options.onError || function (e) {};
this.stopOnError = options.stopOnError || false; this.stopOnError = options.stopOnError || false;
@ -87,12 +85,37 @@ Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function*
Zotero.debug("Starting file sync for " + this.library.name); Zotero.debug("Starting file sync for " + this.library.name);
if (!this.controller.verified) { if (!this.controller.verified) {
Zotero.debug(`${this.mode} file sync is not active`); Zotero.debug(`${this.controller.name} file sync is not active -- verifying`);
throw new Error("Storage mode verification not implemented"); try {
yield this.controller.checkServer();
// TODO: Check server }
catch (e) {
let wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
let lastWin = wm.getMostRecentWindow("navigator:browser");
let success = yield this.controller.handleVerificationError(e, lastWin, true);
if (!success) {
Zotero.debug(this.controller.name + " verification failed", 2);
throw new Zotero.Error(
Zotero.getString('sync.storage.error.verificationFailed', this.controller.name),
0,
{
dialogButtonText: Zotero.getString('sync.openSyncPreferences'),
dialogButtonCallback: function () {
let wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
let lastWin = wm.getMostRecentWindow("navigator:browser");
lastWin.ZoteroPane.openPreferences('zotero-prefpane-sync');
}
}
);
}
}
} }
if (this.controller.cacheCredentials) { if (this.controller.cacheCredentials) {
yield this.controller.cacheCredentials(); yield this.controller.cacheCredentials();
} }
@ -101,7 +124,10 @@ Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function*
var lastSyncTime = null; var lastSyncTime = null;
var downloadAll = this.local.downloadOnSync(libraryID); var downloadAll = this.local.downloadOnSync(libraryID);
if (downloadAll) { if (downloadAll) {
lastSyncTime = yield this.controller.getLastSyncTime(libraryID); if (!this.library.storageDownloadNeeded) {
this.library.storageVersion = this.library.libraryVersion;
yield this.library.saveTx();
}
} }
// Check for updated files to upload // Check for updated files to upload
@ -131,18 +157,11 @@ Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function*
var downloadForced = yield this.local.checkForForcedDownloads(libraryID); var downloadForced = yield this.local.checkForForcedDownloads(libraryID);
// If we don't have any forced downloads, we can skip downloads if the last sync time hasn't // If we don't have any forced downloads, we can skip downloads if no storage metadata has
// changed or doesn't exist on the server (meaning there are no files) // changed (meaning nothing else has uploaded files since the last successful file sync)
if (downloadAll && !downloadForced) { if (downloadAll && !downloadForced) {
if (lastSyncTime) { if (this.library.storageVersion == this.library.libraryVersion) {
if (this.library.lastStorageSync == lastSyncTime) { Zotero.debug("No remote storage changes for " + this.library.name
Zotero.debug("Last " + this.mode.toUpperCase() + " sync id hasn't changed for "
+ this.library.name + " -- skipping file downloads");
downloadAll = false;
}
}
else {
Zotero.debug(`No last ${this.mode} sync time for ${this.library.name}`
+ " -- skipping file downloads"); + " -- skipping file downloads");
downloadAll = false; downloadAll = false;
} }
@ -189,6 +208,7 @@ Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function*
} }
// Process the results // Process the results
var downloadSuccessful = false;
var changes = new Zotero.Sync.Storage.Result; var changes = new Zotero.Sync.Storage.Result;
for (let type of ['download', 'upload']) { for (let type of ['download', 'upload']) {
let results = yield promises[type]; let results = yield promises[type];
@ -202,32 +222,33 @@ Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function*
} }
} }
} }
Zotero.debug(`File ${type} sync finished for ${this.library.name}`); Zotero.debug(`File ${type} sync finished for ${this.library.name}`);
changes.updateFromResults(results.filter(p => p.isFulfilled()).map(p => p.value())); changes.updateFromResults(results.filter(p => p.isFulfilled()).map(p => p.value()));
}
if (type == 'download' && results.every(p => !p.isRejected())) {
// If files were uploaded, update the remote last-sync time downloadSuccessful = true;
if (changes.remoteChanges) {
lastSyncTime = yield this.controller.setLastSyncTime(libraryID);
if (!lastSyncTime) {
throw new Error("Last sync time not set after sync");
} }
} }
// If there's a remote last-sync time from either the check before downloads or when it if (downloadSuccessful) {
// was changed after uploads, store that locally so we know we can skip download checks this.library.storageDownloadNeeded = false;
// next time this.library.storageVersion = this.library.libraryVersion;
if (lastSyncTime) {
this.library.lastStorageSync = lastSyncTime;
yield this.library.saveTx(); yield this.library.saveTx();
} }
// If WebDAV sync, purge deleted and orphaned files // For ZFS, this purges all files on server based on flag set when switching from ZFS
if (this.mode == 'webdav') { // to WebDAV in prefs. For WebDAV, this purges locally deleted files on server.
try {
yield this.controller.purgeDeletedStorageFiles(libraryID);
}
catch (e) {
Zotero.logError(e);
}
// If WebDAV sync, purge orphaned files
if (this.controller.mode == 'webdav') {
try { try {
yield this.controller.purgeDeletedStorageFiles(libraryID);
yield this.controller.purgeOrphanedStorageFiles(libraryID); yield this.controller.purgeOrphanedStorageFiles(libraryID);
} }
catch (e) { catch (e) {
@ -253,16 +274,16 @@ Zotero.Sync.Storage.Engine.prototype.stop = function () {
Zotero.Sync.Storage.Engine.prototype.queueItem = Zotero.Promise.coroutine(function* (item) { Zotero.Sync.Storage.Engine.prototype.queueItem = Zotero.Promise.coroutine(function* (item) {
switch (yield this.local.getSyncState(item.id)) { switch (yield this.local.getSyncState(item.id)) {
case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD: case Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD: case Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_DOWNLOAD:
var type = 'download'; var type = 'download';
var onStart = Zotero.Promise.method(function (request) { var onStart = Zotero.Promise.method(function (request) {
return this.controller.downloadFile(request); return this.controller.downloadFile(request);
}.bind(this)); }.bind(this));
break; break;
case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD: case Zotero.Sync.Storage.Local.SYNC_STATE_TO_UPLOAD:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD: case Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_UPLOAD:
var type = 'upload'; var type = 'upload';
var onStart = Zotero.Promise.method(function (request) { var onStart = Zotero.Promise.method(function (request) {
return this.controller.uploadFile(request); return this.controller.uploadFile(request);

92
chrome/content/zotero/xpcom/storage/storageLocal.js
View File

@ -1,4 +1,14 @@
Zotero.Sync.Storage.Local = { Zotero.Sync.Storage.Local = {
//
// Constants
//
SYNC_STATE_TO_UPLOAD: 0,
SYNC_STATE_TO_DOWNLOAD: 1,
SYNC_STATE_IN_SYNC: 2,
SYNC_STATE_FORCE_UPLOAD: 3,
SYNC_STATE_FORCE_DOWNLOAD: 4,
SYNC_STATE_IN_CONFLICT: 5,
lastFullFileCheck: {}, lastFullFileCheck: {},
uploadCheckFiles: [], uploadCheckFiles: [],
@ -101,7 +111,7 @@ Zotero.Sync.Storage.Local = {
libraryID, libraryID,
Zotero.Attachments.LINK_MODE_IMPORTED_FILE, Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
Zotero.Attachments.LINK_MODE_IMPORTED_URL, Zotero.Attachments.LINK_MODE_IMPORTED_URL,
Zotero.Sync.Storage.SYNC_STATE_IN_SYNC, this.SYNC_STATE_IN_SYNC,
minTime minTime
]; ];
var itemIDs = yield Zotero.DB.columnQueryAsync(sql, params); var itemIDs = yield Zotero.DB.columnQueryAsync(sql, params);
@ -174,8 +184,8 @@ Zotero.Sync.Storage.Local = {
let params = [ let params = [
Zotero.Attachments.LINK_MODE_IMPORTED_FILE, Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
Zotero.Attachments.LINK_MODE_IMPORTED_URL, Zotero.Attachments.LINK_MODE_IMPORTED_URL,
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD, this.SYNC_STATE_TO_UPLOAD,
Zotero.Sync.Storage.SYNC_STATE_IN_SYNC this.SYNC_STATE_IN_SYNC
]; ];
if (libraryID !== false) { if (libraryID !== false) {
sql += " AND libraryID=?"; sql += " AND libraryID=?";
@ -251,7 +261,7 @@ Zotero.Sync.Storage.Local = {
var path = item.getFilePath(); var path = item.getFilePath();
if (!path) { if (!path) {
Zotero.debug("Marking pathless attachment " + lk + " as in-sync"); Zotero.debug("Marking pathless attachment " + lk + " as in-sync");
return Zotero.Sync.Storage.SYNC_STATE_IN_SYNC; return this.SYNC_STATE_IN_SYNC;
} }
var fileName = OS.Path.basename(path); var fileName = OS.Path.basename(path);
var file; var file;
@ -272,7 +282,7 @@ Zotero.Sync.Storage.Local = {
// If file is already marked for upload, skip check. Even if the file was changed // If file is already marked for upload, skip check. Even if the file was changed
// both locally and remotely, conflicts are checked at upload time, so we don't need // both locally and remotely, conflicts are checked at upload time, so we don't need
// to worry about it here. // to worry about it here.
if ((yield this.getSyncState(item.id)) == Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD) { if ((yield this.getSyncState(item.id)) == this.SYNC_STATE_TO_UPLOAD) {
Zotero.debug("File is already marked for upload"); Zotero.debug("File is already marked for upload");
return false; return false;
} }
@ -296,7 +306,7 @@ Zotero.Sync.Storage.Local = {
Zotero.debug(`Marking attachment ${lk} for download (stored mtime: ${mtime})`); Zotero.debug(`Marking attachment ${lk} for download (stored mtime: ${mtime})`);
// DEBUG: Always set here, or allow further steps? // DEBUG: Always set here, or allow further steps?
return Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD; return this.SYNC_STATE_FORCE_DOWNLOAD;
} }
var same = !this.checkFileModTime(item, fmtime, mtime); var same = !this.checkFileModTime(item, fmtime, mtime);
@ -330,7 +340,7 @@ Zotero.Sync.Storage.Local = {
// Mark file for upload // Mark file for upload
Zotero.debug("Marking attachment " + lk + " as changed " Zotero.debug("Marking attachment " + lk + " as changed "
+ "(" + mtime + " != " + fmtime + ")"); + "(" + mtime + " != " + fmtime + ")");
return Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD; return this.SYNC_STATE_TO_UPLOAD;
} }
catch (e) { catch (e) {
if (e instanceof OS.File.Error && if (e instanceof OS.File.Error &&
@ -342,7 +352,7 @@ Zotero.Sync.Storage.Local = {
// Handle long filenames on OS X/Linux // Handle long filenames on OS X/Linux
|| (e.unixErrno && e.unixErrno == 63))) { || (e.unixErrno && e.unixErrno == 63))) {
Zotero.debug("Marking attachment " + lk + " as missing"); Zotero.debug("Marking attachment " + lk + " as missing");
return Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD; return this.SYNC_STATE_TO_DOWNLOAD;
} }
if (e instanceof OS.File.Error) { if (e instanceof OS.File.Error) {
@ -372,7 +382,7 @@ Zotero.Sync.Storage.Local = {
* @return {Boolean} - True if file modification time differs from remote mod time, * @return {Boolean} - True if file modification time differs from remote mod time,
* false otherwise * false otherwise
*/ */
checkFileModTime(item, fmtime, mtime) { checkFileModTime: function (item, fmtime, mtime) {
var libraryKey = item.libraryKey; var libraryKey = item.libraryKey;
if (fmtime == mtime) { if (fmtime == mtime) {
@ -406,7 +416,7 @@ Zotero.Sync.Storage.Local = {
var sql = "SELECT COUNT(*) FROM items JOIN itemAttachments USING (itemID) " var sql = "SELECT COUNT(*) FROM items JOIN itemAttachments USING (itemID) "
+ "WHERE libraryID=? AND syncState=?"; + "WHERE libraryID=? AND syncState=?";
return !!(yield Zotero.DB.valueQueryAsync( return !!(yield Zotero.DB.valueQueryAsync(
sql, [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD] sql, [libraryID, this.SYNC_STATE_FORCE_DOWNLOAD]
)); ));
}), }),
@ -420,10 +430,10 @@ Zotero.Sync.Storage.Local = {
getFilesToDownload: function (libraryID, forcedOnly) { getFilesToDownload: function (libraryID, forcedOnly) {
var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) " var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
+ "WHERE libraryID=? AND syncState IN (?"; + "WHERE libraryID=? AND syncState IN (?";
var params = [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD]; var params = [libraryID, this.SYNC_STATE_FORCE_DOWNLOAD];
if (!forcedOnly) { if (!forcedOnly) {
sql += ",?"; sql += ",?";
params.push(Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD); params.push(this.SYNC_STATE_TO_DOWNLOAD);
} }
sql += ") " sql += ") "
// Skip attachments with empty path, which can't be saved, and files with .zotero* // Skip attachments with empty path, which can't be saved, and files with .zotero*
@ -444,8 +454,8 @@ Zotero.Sync.Storage.Local = {
+ "WHERE libraryID=? AND syncState IN (?,?) AND linkMode IN (?,?)"; + "WHERE libraryID=? AND syncState IN (?,?) AND linkMode IN (?,?)";
var params = [ var params = [
libraryID, libraryID,
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD, this.SYNC_STATE_TO_UPLOAD,
Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD, this.SYNC_STATE_FORCE_UPLOAD,
Zotero.Attachments.LINK_MODE_IMPORTED_FILE, Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
Zotero.Attachments.LINK_MODE_IMPORTED_URL Zotero.Attachments.LINK_MODE_IMPORTED_URL
]; ];
@ -473,17 +483,22 @@ Zotero.Sync.Storage.Local = {
/** /**
* @param {Integer} itemID * @param {Integer} itemID
* @param {Integer} syncState Constant from Zotero.Sync.Storage * @param {Integer|String} syncState - Zotero.Sync.Storage.Local.SYNC_STATE_* or last part
* as string (e.g., "TO_UPLOAD")
*/ */
setSyncState: Zotero.Promise.method(function (itemID, syncState) { setSyncState: Zotero.Promise.method(function (itemID, syncState) {
if (typeof syncState == 'string') {
syncState = this["SYNC_STATE_" + syncState.toUpperCase()];
}
switch (syncState) { switch (syncState) {
case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD: case this.SYNC_STATE_TO_UPLOAD:
case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD: case this.SYNC_STATE_TO_DOWNLOAD:
case Zotero.Sync.Storage.SYNC_STATE_IN_SYNC: case this.SYNC_STATE_IN_SYNC:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD: case this.SYNC_STATE_FORCE_UPLOAD:
case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD: case this.SYNC_STATE_FORCE_DOWNLOAD:
case Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT: case this.SYNC_STATE_IN_CONFLICT:
break; break;
default: default:
@ -495,6 +510,14 @@ Zotero.Sync.Storage.Local = {
}), }),
resetModeSyncStates: Zotero.Promise.coroutine(function* (mode) {
var sql = "UPDATE itemAttachments SET syncState=? "
+ "WHERE itemID IN (SELECT itemID FROM items WHERE libraryID=?)";
var params = [this.SYNC_STATE_TO_UPLOAD, Zotero.Libraries.userLibraryID];
yield Zotero.DB.queryAsync(sql, params);
}),
/** /**
* @param {Integer} itemID * @param {Integer} itemID
* @return {Integer|NULL} Mod time as timestamp in ms, * @return {Integer|NULL} Mod time as timestamp in ms,
@ -513,7 +536,7 @@ Zotero.Sync.Storage.Local = {
/** /**
* @param {Integer} itemID * @param {Integer} itemID
* @param {Integer} mtime - File modification time as timestamp in ms * @param {Integer} mtime - File modification time as timestamp in ms
* @param {Boolean} [updateItem=FALSE] - Update clientDateModified field of attachment item * @param {Boolean} [updateItem=FALSE] - Mark attachment item as unsynced
*/ */
setSyncedModificationTime: Zotero.Promise.coroutine(function* (itemID, mtime, updateItem) { setSyncedModificationTime: Zotero.Promise.coroutine(function* (itemID, mtime, updateItem) {
if (mtime < 0) { if (mtime < 0) {
@ -528,9 +551,8 @@ Zotero.Sync.Storage.Local = {
yield Zotero.DB.queryAsync(sql, [mtime, itemID]); yield Zotero.DB.queryAsync(sql, [mtime, itemID]);
if (updateItem) { if (updateItem) {
// Update item date modified so the new mod time will be synced let item = yield Zotero.Items.getAsync(itemID);
let sql = "UPDATE items SET clientDateModified=? WHERE itemID=?"; yield item.updateSynced(false);
yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]);
} }
}), }),
@ -553,8 +575,7 @@ Zotero.Sync.Storage.Local = {
/** /**
* @param {Integer} itemID * @param {Integer} itemID
* @param {String} hash File hash * @param {String} hash File hash
* @param {Boolean} [updateItem=FALSE] Update dateModified field of * @param {Boolean} [updateItem=FALSE] - Mark attachment item as unsynced
* attachment item
*/ */
setSyncedHash: Zotero.Promise.coroutine(function* (itemID, hash, updateItem) { setSyncedHash: Zotero.Promise.coroutine(function* (itemID, hash, updateItem) {
if (hash !== null && hash.length != 32) { if (hash !== null && hash.length != 32) {
@ -567,9 +588,8 @@ Zotero.Sync.Storage.Local = {
yield Zotero.DB.queryAsync(sql, [hash, itemID]); yield Zotero.DB.queryAsync(sql, [hash, itemID]);
if (updateItem) { if (updateItem) {
// Update item date modified so the new mod time will be synced let item = yield Zotero.Items.getAsync(itemID);
var sql = "UPDATE items SET clientDateModified=? WHERE itemID=?"; yield item.updateSynced(false);
yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]);
} }
}), }),
@ -658,7 +678,7 @@ Zotero.Sync.Storage.Local = {
yield Zotero.DB.executeTransaction(function* () { yield Zotero.DB.executeTransaction(function* () {
yield this.setSyncedHash(item.id, md5); yield this.setSyncedHash(item.id, md5);
yield this.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); yield this.setSyncState(item.id, this.SYNC_STATE_IN_SYNC);
yield this.setSyncedModificationTime(item.id, mtime); yield this.setSyncedModificationTime(item.id, mtime);
}.bind(this)); }.bind(this));
@ -998,7 +1018,7 @@ Zotero.Sync.Storage.Local = {
sql, sql,
[ [
{ int: libraryID }, { int: libraryID },
Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT this.SYNC_STATE_IN_CONFLICT
] ]
); );
var keyVersionPairs = rows.map(function (row) { var keyVersionPairs = rows.map(function (row) {
@ -1073,16 +1093,16 @@ Zotero.Sync.Storage.Local = {
let mtime = io.dataOut[i].dateModified; let mtime = io.dataOut[i].dateModified;
// Local // Local
if (mtime == conflict.left.dateModified) { if (mtime == conflict.left.dateModified) {
syncState = Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD; syncState = this.SYNC_STATE_FORCE_UPLOAD;
} }
// Remote // Remote
else { else {
syncState = Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD; syncState = this.SYNC_STATE_FORCE_DOWNLOAD;
} }
let itemID = Zotero.Items.getIDFromLibraryAndKey(libraryID, conflict.left.key); let itemID = Zotero.Items.getIDFromLibraryAndKey(libraryID, conflict.left.key);
yield Zotero.Sync.Storage.Local.setSyncState(itemID, syncState); yield Zotero.Sync.Storage.Local.setSyncState(itemID, syncState);
} }
}); }.bind(this));
return true; return true;
}) })
} }

4
chrome/content/zotero/xpcom/storage/storageUtilities.js
View File

@ -2,10 +2,10 @@ Zotero.Sync.Storage.Utilities = {
getClassForMode: function (mode) { getClassForMode: function (mode) {
switch (mode) { switch (mode) {
case 'zfs': case 'zfs':
return Zotero.Sync.Storage.ZFS_Module; return Zotero.Sync.Storage.Mode.ZFS;
case 'webdav': case 'webdav':
return Zotero.Sync.Storage.WebDAV_Module; return Zotero.Sync.Storage.Mode.WebDAV;
default: default:
throw new Error("Invalid storage mode '" + mode + "'"); throw new Error("Invalid storage mode '" + mode + "'");

2218
chrome/content/zotero/xpcom/storage/webdav.js
View File

File diff suppressed because it is too large Load Diff

124
chrome/content/zotero/xpcom/storage/zfs.js
View File

@ -23,8 +23,11 @@
***** END LICENSE BLOCK ***** ***** END LICENSE BLOCK *****
*/ */
if (!Zotero.Sync.Storage.Mode) {
Zotero.Sync.Storage.Mode = {};
}
Zotero.Sync.Storage.ZFS_Module = function (options) { Zotero.Sync.Storage.Mode.ZFS = function (options) {
this.options = options; this.options = options;
this.apiClient = options.apiClient; this.apiClient = options.apiClient;
@ -33,76 +36,17 @@ Zotero.Sync.Storage.ZFS_Module = function (options) {
this._maxS3Backoff = 60; this._maxS3Backoff = 60;
this._maxS3ConsecutiveFailures = 5; this._maxS3ConsecutiveFailures = 5;
}; };
Zotero.Sync.Storage.ZFS_Module.prototype = { Zotero.Sync.Storage.Mode.ZFS.prototype = {
mode: "zfs",
name: "ZFS", name: "ZFS",
verified: true, verified: true,
/**
* @return {Promise} A promise for the last sync time
*/
getLastSyncTime: Zotero.Promise.coroutine(function* (libraryID) {
var params = this._getRequestParams(libraryID, "laststoragesync");
var uri = this.apiClient.buildRequestURI(params);
try {
let req = yield this.apiClient.makeRequest(
"GET", uri, { successCodes: [200, 404], debug: true }
);
// Not yet synced
if (req.status == 404) {
Zotero.debug("No last sync time for library " + libraryID);
return null;
}
let ts = req.responseText;
let date = new Date(ts * 1000);
Zotero.debug("Last successful ZFS sync for library " + libraryID + " was " + date);
return ts;
}
catch (e) {
Zotero.logError(e);
throw e;
}
}),
setLastSyncTime: Zotero.Promise.coroutine(function* (libraryID) {
var params = this._getRequestParams(libraryID, "laststoragesync");
var uri = this.apiClient.buildRequestURI(params);
try {
var req = yield this.apiClient.makeRequest(
"POST", uri, { successCodes: [200, 404], debug: true }
);
}
catch (e) {
var msg = "Unexpected status code " + e.xmlhttp.status + " setting last file sync time";
Zotero.logError(e);
throw new Error(Zotero.Sync.Storage.defaultError);
}
// Not yet synced
//
// TODO: Don't call this at all if no files uploaded
if (req.status == 404) {
return;
}
var time = req.responseText;
if (parseInt(time) != time) {
Zotero.logError(`Unexpected response ${time} setting last file sync time`);
throw new Error(Zotero.Sync.Storage.defaultError);
}
return parseInt(time);
}),
/** /**
* Begin download process for individual file * Begin download process for individual file
* *
* @param {Zotero.Sync.Storage.Request} request * @param {Zotero.Sync.Storage.Request} request
* @return {Promise<Boolean>} - True if file download, false if not * @return {Promise<Zotero.Sync.Storage.Result>}
*/ */
downloadFile: Zotero.Promise.coroutine(function* (request) { downloadFile: Zotero.Promise.coroutine(function* (request) {
var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request); var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
@ -140,7 +84,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
Zotero.debug("Download request " + request.name Zotero.debug("Download request " + request.name
+ " stopped before download started -- closing channel"); + " stopped before download started -- closing channel");
req.cancel(Components.results.NS_BINDING_ABORTED); req.cancel(Components.results.NS_BINDING_ABORTED);
deferred.resolve(false); deferred.resolve(new Zotero.Sync.Storage.Result);
} }
}, },
onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) { onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) {
@ -193,9 +137,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
yield Zotero.Sync.Storage.Local.setSyncedModificationTime( yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
item.id, requestData.mtime item.id, requestData.mtime
); );
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
}); });
return false; return false;
}), }),
@ -259,7 +201,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
if (request.isFinished()) { if (request.isFinished()) {
Zotero.debug("Download request " + request.name Zotero.debug("Download request " + request.name
+ " is no longer running after file download", 2); + " is no longer running after file download", 2);
deferred.resolve(false); deferred.resolve(new Zotero.Sync.Storage.Result);
return; return;
} }
@ -271,14 +213,13 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
); );
} }
catch (e) { catch (e) {
Zotero.debug("REJECTING");
deferred.reject(e); deferred.reject(e);
} }
}.bind(this), }.bind(this),
onCancel: function (req, status) { onCancel: function (req, status) {
Zotero.debug("Request cancelled"); Zotero.debug("Request cancelled");
if (deferred.promise.isPending()) { if (deferred.promise.isPending()) {
deferred.resolve(false); deferred.resolve(new Zotero.Sync.Storage.Result);
} }
} }
} }
@ -290,8 +231,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
Zotero.debug('Saving ' + uri); Zotero.debug('Saving ' + uri);
const nsIWBP = Components.interfaces.nsIWebBrowserPersist; const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
var wbp = Components var wbp = Components.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
.classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
.createInstance(nsIWBP); .createInstance(nsIWBP);
wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE; wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE;
wbp.progressListener = listener; wbp.progressListener = listener;
@ -308,7 +248,6 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
if (!created) { if (!created) {
return new Zotero.Sync.Storage.Result; return new Zotero.Sync.Storage.Result;
} }
return this._processUploadFile(request);
} }
return this._processUploadFile(request); return this._processUploadFile(request);
}), }),
@ -327,23 +266,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
Zotero.debug("Unlinking synced files on ZFS"); Zotero.debug("Unlinking synced files on ZFS");
var uri = this.userURI; var uri = this.userURI;
uri.spec += "removestoragefiles?"; uri.spec += "removestoragefiles";
// Unused
for each(var value in values) {
switch (value) {
case 'user':
uri.spec += "user=1&";
break;
case 'group':
uri.spec += "group=1&";
break;
default:
throw new Error("Invalid zfsPurge value '" + value + "'");
}
}
uri.spec = uri.spec.substr(0, uri.spec.length - 1);
yield Zotero.HTTP.request("POST", uri, ""); yield Zotero.HTTP.request("POST", uri, "");
@ -438,10 +361,9 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
} }
// Build POST body // Build POST body
var mtime = yield item.attachmentModificationTime;
var params = { var params = {
mtime: yield item.attachmentModificationTime,
md5: yield item.attachmentHash, md5: yield item.attachmentHash,
mtime,
filename, filename,
filesize: (yield OS.File.stat(uploadPath)).size filesize: (yield OS.File.stat(uploadPath)).size
}; };
@ -521,7 +443,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
// TEMP // TEMP
// //
// Passed through to _updateItemFileInfo() // Passed through to _updateItemFileInfo()
json.mtime = mtime; json.mtime = params.mtime;
json.md5 = params.md5; json.md5 = params.md5;
if (storedHash) { if (storedHash) {
json.storedHash = storedHash; json.storedHash = storedHash;
@ -619,16 +541,12 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
item.id, fileModTime item.id, fileModTime
); );
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, fileHash); yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, fileHash);
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
}); });
return new Zotero.Sync.Storage.Result; return new Zotero.Sync.Storage.Result;
} }
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_conflict");
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
return new Zotero.Sync.Storage.Result({ return new Zotero.Sync.Storage.Result({
fileSyncRequired: true fileSyncRequired: true
}); });
@ -847,9 +765,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
_updateItemFileInfo: Zotero.Promise.coroutine(function* (item, params) { _updateItemFileInfo: Zotero.Promise.coroutine(function* (item, params) {
// Mark as in-sync // Mark as in-sync
yield Zotero.DB.executeTransaction(function* () { yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
// Store file mod time and hash // Store file mod time and hash
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, params.mtime); yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, params.mtime);
@ -1016,7 +932,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
// Check for conflict // Check for conflict
if ((yield Zotero.Sync.Storage.Local.getSyncState(item.id)) if ((yield Zotero.Sync.Storage.Local.getSyncState(item.id))
!= Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) { != Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_UPLOAD) {
if (info) { if (info) {
// Local file time // Local file time
var fmtime = yield item.attachmentModificationTime; var fmtime = yield item.attachmentModificationTime;
@ -1036,7 +952,7 @@ Zotero.Sync.Storage.ZFS_Module.prototype = {
yield Zotero.DB.executeTransaction(function* () { yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime); yield Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime);
yield Zotero.Sync.Storage.setSyncState( yield Zotero.Sync.Storage.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC item.id, Zotero.Sync.Storage.Local.SYNC_STATE_IN_SYNC
); );
}); });
return { return {

2
chrome/content/zotero/xpcom/sync.js
View File

@ -1657,7 +1657,7 @@ Zotero.Sync.Server.Data = new function() {
// Mark new attachments for download // Mark new attachments for download
if (isNewObject) { if (isNewObject) {
obj.attachmentSyncState = obj.attachmentSyncState =
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD; Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD;
} }
// Set existing attachments for mtime update check // Set existing attachments for mtime update check
else { else {

10
chrome/content/zotero/xpcom/sync/syncEngine.js
View File

@ -709,9 +709,12 @@ Zotero.Sync.Data.Engine.prototype._uploadObjects = Zotero.Promise.coroutine(func
toCache.push(current); toCache.push(current);
} }
else { else {
let j = yield obj.toJSON(); // This won't reflect the actual version of the item on the server, but
j.version = json.libraryVersion; // it will guarantee that the item won't be redownloaded unnecessarily
toCache.push(j); // in the case of a full sync, because the version will be higher than
// whatever version is on the server.
batch[index].version = json.libraryVersion
toCache.push(batch[index]);
} }
numSuccessful++; numSuccessful++;
@ -891,6 +894,7 @@ Zotero.Sync.Data.Engine.prototype._getJSONForObject = function (objectType, id)
includeKey: true, includeKey: true,
includeVersion: true, // DEBUG: remove? includeVersion: true, // DEBUG: remove?
includeDate: true, includeDate: true,
syncedStorageProperties: true,
patchBase: cacheObj ? cacheObj.data : false patchBase: cacheObj ? cacheObj.data : false
}); });
}); });

85
chrome/content/zotero/xpcom/sync/syncEventListeners.js
View File

@ -43,58 +43,55 @@ Zotero.Sync.EventListeners.ChangeListener = new function () {
var storageForLibrary = {}; var storageForLibrary = {};
return Zotero.DB.executeTransaction(function* () { return Zotero.Utilities.Internal.forEachChunkAsync(
for (let i = 0; i < ids.length; i++) { ids,
let id = ids[i]; 100,
function (chunk) {
if (extraData[id] && extraData[id].skipDeleteLog) { return Zotero.DB.executeTransaction(function* () {
continue; for (let id of chunk) {
} if (extraData[id] && extraData[id].skipDeleteLog) {
continue;
var libraryID, key; }
if (type == 'setting') {
[libraryID, key] = ids[i].split("/"); if (type == 'setting') {
} var [libraryID, key] = id.split("/");
else { }
let d = extraData[ids[i]]; else {
libraryID = d.libraryID; var { libraryID, key } = extraData[id];
key = d.key; }
}
if (!key) {
if (!key) { throw new Error("Key not provided in notifier object");
throw new Error("Key not provided in notifier object"); }
}
yield Zotero.DB.queryAsync(
syncSQL,
[
syncObjectTypeID,
libraryID,
key
]
);
if (type == 'item') {
if (storageForLibrary[libraryID] === undefined) {
storageForLibrary[libraryID] =
Zotero.Sync.Storage.Local.getModeForLibrary(libraryID) == 'webdav';
}
if (storageForLibrary[libraryID] && oldItem.itemType == 'attachment' &&
[
Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
Zotero.Attachments.LINK_MODE_IMPORTED_URL
].indexOf(oldItem.linkMode) != -1) {
yield Zotero.DB.queryAsync( yield Zotero.DB.queryAsync(
storageSQL, syncSQL,
[ [
syncObjectTypeID,
libraryID, libraryID,
key key
] ]
); );
if (type == 'item') {
if (storageForLibrary[libraryID] === undefined) {
storageForLibrary[libraryID] =
Zotero.Sync.Storage.Local.getModeForLibrary(libraryID) == 'webdav';
}
if (storageForLibrary[libraryID] && extraData[id].storageDeleteLog) {
yield Zotero.DB.queryAsync(
storageSQL,
[
libraryID,
key
]
);
}
}
} }
} });
} }
}); );
}); });
} }

62
chrome/content/zotero/xpcom/sync/syncLocal.js
View File

@ -478,6 +478,7 @@ Zotero.Sync.Data.Local = {
let isNewObject = false; let isNewObject = false;
let skipCache = false; let skipCache = false;
let storageDetailsChanged = false;
let obj = yield objectsClass.getByLibraryAndKeyAsync( let obj = yield objectsClass.getByLibraryAndKeyAsync(
libraryID, objectKey, { noCache: true } libraryID, objectKey, { noCache: true }
); );
@ -495,12 +496,22 @@ Zotero.Sync.Data.Local = {
let jsonDataLocal = yield obj.toJSON(); let jsonDataLocal = yield obj.toJSON();
// For items, check if mtime or file hash changed in metadata,
// which would indicate that a remote storage sync took place and
// a download is needed
if (objectType == 'item' && obj.isImportedAttachment()) {
if (jsonDataLocal.mtime != jsonData.mtime
|| jsonDataLocal.md5 != jsonData.md5) {
storageDetailsChanged = true;
}
}
let result = this._reconcileChanges( let result = this._reconcileChanges(
objectType, objectType,
cachedJSON.data, cachedJSON.data,
jsonDataLocal, jsonDataLocal,
jsonData, jsonData,
['dateAdded', 'dateModified'] ['mtime', 'md5', 'dateAdded', 'dateModified']
); );
// If no changes, update local version number and mark as synced // If no changes, update local version number and mark as synced
@ -510,6 +521,15 @@ Zotero.Sync.Data.Local = {
obj.version = json.version; obj.version = json.version;
obj.synced = true; obj.synced = true;
yield obj.save(); yield obj.save();
if (objectType == 'item') {
yield this._onItemProcessed(
obj,
jsonData,
isNewObject,
storageDetailsChanged
);
}
continue; continue;
} }
@ -599,15 +619,12 @@ Zotero.Sync.Data.Local = {
obj, jsonData, options, { skipCache } obj, jsonData, options, { skipCache }
); );
if (saved) { if (saved) {
// Delete older versions of the item in the cache if (objectType == 'item') {
yield this.deleteCacheObjectVersions( yield this._onItemProcessed(
objectType, libraryID, jsonData.key, null, jsonData.version - 1 obj,
); jsonData,
isNewObject,
// Mark updated attachments for download storageDetailsChanged
if (objectType == 'item' && obj.isImportedAttachment()) {
yield this._checkAttachmentForDownload(
obj, jsonData.mtime, isNewObject
); );
} }
} }
@ -694,6 +711,27 @@ Zotero.Sync.Data.Local = {
}), }),
_onItemProcessed: Zotero.Promise.coroutine(function* (item, jsonData, isNewObject, storageDetailsChanged) {
// Delete older versions of the item in the cache
yield this.deleteCacheObjectVersions(
'item', item.libraryID, jsonData.key, null, jsonData.version - 1
);
// Mark updated attachments for download
if (item.isImportedAttachment()) {
// If storage changes were made (attachment mtime or hash), mark
// library as requiring download
if (isNewObject || storageDetailsChanged) {
Zotero.Libraries.get(item.libraryID).storageDownloadNeeded = true;
}
yield this._checkAttachmentForDownload(
item, jsonData.mtime, isNewObject
);
}
}),
_checkAttachmentForDownload: Zotero.Promise.coroutine(function* (item, mtime, isNewObject) { _checkAttachmentForDownload: Zotero.Promise.coroutine(function* (item, mtime, isNewObject) {
var markToDownload = false; var markToDownload = false;
if (!isNewObject) { if (!isNewObject) {
@ -724,9 +762,7 @@ Zotero.Sync.Data.Local = {
markToDownload = true; markToDownload = true;
} }
if (markToDownload) { if (markToDownload) {
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
} }
}), }),

23
chrome/content/zotero/xpcom/sync/syncRunner.js
View File

@ -62,6 +62,7 @@ Zotero.Sync.Runner_Module = function (options = {}) {
var _syncEngines = []; var _syncEngines = [];
var _storageEngines = []; var _storageEngines = [];
var _storageControllers = {};
var _lastSyncStatus; var _lastSyncStatus;
var _currentSyncStatusLabel; var _currentSyncStatusLabel;
@ -476,6 +477,9 @@ Zotero.Sync.Runner_Module = function (options = {}) {
Object.assign(opts, options); Object.assign(opts, options);
opts.libraryID = libraryID; opts.libraryID = libraryID;
let mode = Zotero.Sync.Storage.Local.getModeForLibrary(libraryID);
opts.controller = this.getStorageController(mode, opts);
let tries = 3; let tries = 3;
while (true) { while (true) {
if (tries == 0) { if (tries == 0) {
@ -549,6 +553,25 @@ Zotero.Sync.Runner_Module = function (options = {}) {
}.bind(this)); }.bind(this));
/**
* Get a storage controller for a given mode ('zfs', 'webdav'),
* caching it if necessary
*/
this.getStorageController = function (mode, options) {
if (_storageControllers[mode]) {
return _storageControllers[mode];
}
var modeClass = Zotero.Sync.Storage.Utilities.getClassForMode(mode);
return _storageControllers[mode] = new modeClass(options);
},
// TODO: Call on API key change
this.resetStorageController = function (mode) {
delete _storageControllers[mode];
},
/** /**
* Download a single file on demand (not within a sync process) * Download a single file on demand (not within a sync process)
*/ */

9
chrome/content/zotero/zoteroPane.js
View File

@ -235,15 +235,6 @@ var ZoteroPane = new function()
catch (e) {} catch (e) {}
} }
// Hide sync debugging menu by default
if (Zotero.Prefs.get('sync.debugMenu')) {
var sep = document.getElementById('zotero-tb-actions-sync-separator');
sep.hidden = false;
sep.nextSibling.hidden = false;
sep.nextSibling.nextSibling.hidden = false;
sep.nextSibling.nextSibling.nextSibling.hidden = false;
}
if (Zotero.openPane) { if (Zotero.openPane) {
Zotero.openPane = false; Zotero.openPane = false;
setTimeout(function () { setTimeout(function () {

4
chrome/content/zotero/zoteroPane.xul
View File

@ -114,10 +114,6 @@
label="Search for Shared Libraries" oncommand="Zotero.Zeroconf.findInstances()"/> label="Search for Shared Libraries" oncommand="Zotero.Zeroconf.findInstances()"/>
<menuseparator id="zotero-tb-actions-plugins-separator"/> <menuseparator id="zotero-tb-actions-plugins-separator"/>
<menuitem id="zotero-tb-actions-timeline" label="&zotero.toolbar.timeline.label;" command="cmd_zotero_createTimeline"/> <menuitem id="zotero-tb-actions-timeline" label="&zotero.toolbar.timeline.label;" command="cmd_zotero_createTimeline"/>
<menuseparator hidden="true" id="zotero-tb-actions-sync-separator"/>
<menuitem hidden="true" label="WebDAV Sync Debugging" disabled="true"/>
<menuitem hidden="true" label=" Purge Deleted Storage Files" oncommand="Zotero.Sync.Storage.purgeDeletedStorageFiles('webdav', function(results) { Zotero.debug(results); })"/>
<menuitem hidden="true" label=" Purge Orphaned Storage Files" oncommand="Zotero.Sync.Storage.purgeOrphanedStorageFiles('webdav', function(results) { Zotero.debug(results); })"/>
<menuseparator id="zotero-tb-actions-separator"/> <menuseparator id="zotero-tb-actions-separator"/>
<menuitem id="zotero-tb-actions-prefs" label="&zotero.toolbar.preferences.label;" <menuitem id="zotero-tb-actions-prefs" label="&zotero.toolbar.preferences.label;"
oncommand="ZoteroPane_Local.openPreferences()"/> oncommand="ZoteroPane_Local.openPreferences()"/>

3
chrome/locale/en-US/zotero/zotero.properties
View File

@ -920,6 +920,9 @@ sync.storage.error.webdav.fileMissingAfterUpload = A potential problem was foun
sync.storage.error.webdav.nonexistentFileNotMissing = Your WebDAV server is claiming that a nonexistent file exists. Contact your WebDAV server administrator for assistance. sync.storage.error.webdav.nonexistentFileNotMissing = Your WebDAV server is claiming that a nonexistent file exists. Contact your WebDAV server administrator for assistance.
sync.storage.error.webdav.serverConfig.title = WebDAV Server Configuration Error sync.storage.error.webdav.serverConfig.title = WebDAV Server Configuration Error
sync.storage.error.webdav.serverConfig = Your WebDAV server returned an internal error. sync.storage.error.webdav.serverConfig = Your WebDAV server returned an internal error.
sync.storage.error.webdav.requestError = Your WebDAV server returned an HTTP %1$S error for a %2$S request.
sync.storage.error.webdav.checkSettingsOrContactAdmin = If you receive this message repeatedly, check your WebDAV server settings or contact your WebDAV server administrator.
sync.storage.error.webdav.url = URL: %S
sync.storage.error.zfs.restart = A file sync error occurred. Please restart %S and/or your computer and try syncing again.\n\nIf the error persists, there may be a problem with either your computer or your network: security software, proxy server, VPN, etc. Try disabling any security/firewall software you're using or, if this is a laptop, try from a different network. sync.storage.error.zfs.restart = A file sync error occurred. Please restart %S and/or your computer and try syncing again.\n\nIf the error persists, there may be a problem with either your computer or your network: security software, proxy server, VPN, etc. Try disabling any security/firewall software you're using or, if this is a laptop, try from a different network.
sync.storage.error.zfs.tooManyQueuedUploads = You have too many queued uploads. Please try again in %S minutes. sync.storage.error.zfs.tooManyQueuedUploads = You have too many queued uploads. Please try again in %S minutes.

6
resource/schema/userdata.sql
View File

@ -1,4 +1,4 @@
-- 80 -- 81
-- Copyright (c) 2009 Center for History and New Media -- Copyright (c) 2009 Center for History and New Media
-- George Mason University, Fairfax, Virginia, USA -- George Mason University, Fairfax, Virginia, USA
@ -261,8 +261,8 @@ CREATE TABLE libraries (
editable INT NOT NULL, editable INT NOT NULL,
filesEditable INT NOT NULL, filesEditable INT NOT NULL,
version INT NOT NULL DEFAULT 0, version INT NOT NULL DEFAULT 0,
lastSync INT NOT NULL DEFAULT 0, storageVersion INT NOT NULL DEFAULT 0,
lastStorageSync INT NOT NULL DEFAULT 0 lastSync INT NOT NULL DEFAULT 0
); );
CREATE TABLE users ( CREATE TABLE users (

6
test/content/support.js
View File

@ -724,8 +724,14 @@ function setHTTPResponse(server, baseURL, response, responses) {
responseArray[1]["Content-Type"] = "text/plain"; responseArray[1]["Content-Type"] = "text/plain";
responseArray[2] = response.text || ""; responseArray[2] = response.text || "";
} }
if (!response.headers) {
response.headers = {};
}
response.headers["Fake-Server-Match"] = 1;
for (let i in response.headers) { for (let i in response.headers) {
responseArray[1][i] = response.headers[i]; responseArray[1][i] = response.headers[i];
} }
server.respondWith(response.method, baseURL + response.url, responseArray); server.respondWith(response.method, baseURL + response.url, responseArray);
} }

61
test/tests/itemTest.js
View File

@ -616,7 +616,7 @@ describe("Zotero.Item", function () {
// DEBUG: Is this necessary? // DEBUG: Is this necessary?
assert.equal( assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)), (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD Zotero.Sync.Storage.Local.SYNC_STATE_TO_UPLOAD
); );
assert.isNull(yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)); assert.isNull(yield Zotero.Sync.Storage.Local.getSyncedHash(item.id));
}) })
@ -874,6 +874,49 @@ describe("Zotero.Item", function () {
assert.strictEqual(json.deleted, 1); assert.strictEqual(json.deleted, 1);
}) })
it("should output attachment fields from file", function* () {
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
item.id, new Date().getTime()
);
yield Zotero.Sync.Storage.Local.setSyncedHash(
item.id, 'b32e33f529942d73bea4ed112310f804'
);
});
var json = yield item.toJSON();
assert.equal(json.linkMode, 'imported_file');
assert.equal(json.filename, 'test.png');
assert.isUndefined(json.path);
assert.equal(json.mtime, (yield item.attachmentModificationTime));
assert.equal(json.md5, (yield item.attachmentHash));
})
it("should output synced storage values with .syncedStorageProperties", function* () {
var item = new Zotero.Item('attachment');
item.attachmentLinkMode = 'imported_file';
item.fileName = 'test.txt';
yield item.saveTx();
var mtime = new Date().getTime();
var md5 = 'b32e33f529942d73bea4ed112310f804';
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, md5);
});
var json = yield item.toJSON({
syncedStorageProperties: true
});
assert.equal(json.mtime, mtime);
assert.equal(json.md5, md5);
})
it("should output unset storage properties as null", function* () { it("should output unset storage properties as null", function* () {
var item = new Zotero.Item('attachment'); var item = new Zotero.Item('attachment');
item.attachmentLinkMode = 'imported_file'; item.attachmentLinkMode = 'imported_file';
@ -881,7 +924,6 @@ describe("Zotero.Item", function () {
var id = yield item.saveTx(); var id = yield item.saveTx();
var json = yield item.toJSON(); var json = yield item.toJSON();
Zotero.debug(json);
assert.isNull(json.mtime); assert.isNull(json.mtime);
assert.isNull(json.md5); assert.isNull(json.md5);
}) })
@ -960,21 +1002,6 @@ describe("Zotero.Item", function () {
assert.strictEqual(json.deleted, 1); assert.strictEqual(json.deleted, 1);
}) })
}) })
// TODO: Expand to all fields
it("should handle attachment fields", function* () {
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({
file: file
});
var json = yield item.toJSON();
assert.equal(json.linkMode, 'imported_file');
assert.equal(json.filename, 'test.png');
assert.isUndefined(json.path);
assert.equal(json.md5, '93da8f1e5774c599f0942dcecf64b11c');
assert.typeOf(json.mtime, 'number');
})
}) })
describe("#fromJSON()", function () { describe("#fromJSON()", function () {

20
test/tests/libraryTest.js
View File

@ -63,26 +63,6 @@ describe("Zotero.Library", function() {
}); });
}); });
describe("#lastStorageSync", function () {
it("should set and get a time in seconds", function* () {
var library = yield createGroup();
var time = Math.round(new Date().getTime() / 1000);
library.lastStorageSync = time;
yield library.saveTx();
var dbTime = yield Zotero.DB.valueQueryAsync(
"SELECT lastStorageSync FROM libraries WHERE libraryID=?", library.libraryID
);
assert.equal(dbTime, time);
assert.equal(library.lastStorageSync, time);
});
it("should throw if setting time in milliseconds", function* () {
var library = Zotero.Libraries.userLibrary;
assert.throws(() => library.lastStorageSync = new Date().getTime(), "timestamp must be in seconds");
})
})
describe("#editable", function() { describe("#editable", function() {
it("should return editable status", function() { it("should return editable status", function() {
let library = Zotero.Libraries.get(Zotero.Libraries.userLibraryID); let library = Zotero.Libraries.get(Zotero.Libraries.userLibraryID);

819
test/tests/storageEngineTest.js
View File

@ -1,819 +0,0 @@
"use strict";
describe("Zotero.Sync.Storage.Engine", function () {
Components.utils.import("resource://zotero-unit/httpd.js");
var win;
var apiKey = Zotero.Utilities.randomString(24);
var port = 16213;
var baseURL = `http://localhost:${port}/`;
var server;
var responses = {};
var setup = Zotero.Promise.coroutine(function* (options = {}) {
server = sinon.fakeServer.create();
server.autoRespond = true;
Components.utils.import("resource://zotero/concurrentCaller.js");
var caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true;
Components.utils.import("resource://zotero/config.js");
var client = new Zotero.Sync.APIClient({
baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
apiKey,
caller,
background: options.background || true
});
var engine = new Zotero.Sync.Storage.Engine({
apiClient: client,
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
stopOnError: true
});
return { engine, client, caller };
});
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
function parseQueryString(str) {
var queryStringParams = str.split('&');
var params = {};
for (let param of queryStringParams) {
let [ key, val ] = param.split('=');
params[key] = decodeURIComponent(val);
}
return params;
}
function assertAPIKey(request) {
assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
}
//
// Tests
//
before(function* () {
})
beforeEach(function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
win = yield loadZoteroPane();
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
this.httpd = new HttpServer();
this.httpd.start(port);
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser");
// Set download-on-sync by default
Zotero.Sync.Storage.Local.downloadOnSync(
Zotero.Libraries.userLibraryID, true
);
})
afterEach(function* () {
var defer = new Zotero.Promise.defer();
this.httpd.stop(() => defer.resolve());
yield defer.promise;
win.close();
})
after(function* () {
this.timeout(60000);
//yield resetDB();
win.close();
})
describe("ZFS", function () {
describe("Syncing", function () {
it("should skip downloads if no last storage sync time", function* () {
var { engine, client, caller } = yield setup();
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 404
});
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.isFalse(Zotero.Libraries.userLibrary.lastStorageSync);
})
it("should skip downloads if unchanged last storage sync time", function* () {
var { engine, client, caller } = yield setup();
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
var library = Zotero.Libraries.userLibrary;
library.lastStorageSync = newStorageSyncTime;
yield library.saveTx();
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.equal(library.lastStorageSync, newStorageSyncTime);
})
it("should ignore a remotely missing file", function* () {
var { engine, client, caller } = yield setup();
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 404, null);
}
}
);
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
})
it("should handle a remotely failing file", function* () {
var { engine, client, caller } = yield setup();
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 500, null);
}
}
);
// TODO: In stopOnError mode, this the promise is rejected.
// This should probably test with stopOnError mode turned off instead.
var e = yield getPromiseError(engine.start());
assert.equal(e.message, Zotero.Sync.Storage.defaultError);
})
it("should download a missing file", function* () {
var { engine, client, caller } = yield setup();
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
// TODO: Test binary data
var text = Zotero.Utilities.randomString();
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var mtime = "1441252524905";
var md5 = Zotero.Utilities.Internal.md5(text)
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
var s3Path = `pretend-s3/${item.key}`;
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
if (!request.hasHeader('Zotero-API-Key')) {
response.setStatusLine(null, 403, "Forbidden");
return;
}
var key = request.getHeader('Zotero-API-Key');
if (key != apiKey) {
response.setStatusLine(null, 403, "Invalid key");
return;
}
response.setStatusLine(null, 302, "Found");
response.setHeader("Zotero-File-Modification-Time", mtime, false);
response.setHeader("Zotero-File-MD5", md5, false);
response.setHeader("Zotero-File-Compressed", "No", false);
response.setHeader("Location", baseURL + s3Path, false);
}
}
);
this.httpd.registerPathHandler(
"/" + s3Path,
{
handle: function (request, response) {
response.setStatusLine(null, 200, "OK");
response.write(text);
}
}
);
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync());
assert.equal(contents, text);
})
it("should upload new files", function* () {
var { engine, client, caller } = yield setup();
// Single file
var file1 = getTestDataDirectory();
file1.append('test.png');
var item1 = yield Zotero.Attachments.importFromFile({ file: file1 });
var mtime1 = yield item1.attachmentModificationTime;
var hash1 = yield item1.attachmentHash;
var path1 = item1.getFilePath();
var filename1 = 'test.png';
var size1 = (yield OS.File.stat(path1)).size;
var contentType1 = 'image/png';
var prefix1 = Zotero.Utilities.randomString();
var suffix1 = Zotero.Utilities.randomString();
var uploadKey1 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
// HTML file with auxiliary image
var file2 = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
var parentItem = yield createDataObject('item');
var item2 = yield Zotero.Attachments.importSnapshotFromFile({
file: file2,
url: 'http://example.com/',
parentItemID: parentItem.id,
title: 'Test',
contentType: 'text/html',
charset: 'utf-8'
});
var mtime2 = yield item2.attachmentModificationTime;
var hash2 = yield item2.attachmentHash;
var path2 = item2.getFilePath();
var filename2 = 'index.html';
var size2 = (yield OS.File.stat(path2)).size;
var contentType2 = 'text/html';
var charset2 = 'utf-8';
var prefix2 = Zotero.Utilities.randomString();
var suffix2 = Zotero.Utilities.randomString();
var uploadKey2 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
var deferreds = [];
setResponse({
method: "GET",
url: "users/1/laststoragesync",
status: 404
});
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.md5, hash1);
assert.equal(params.mtime, mtime1);
assert.equal(params.filename, filename1);
assert.equal(params.filesize, size1);
assert.equal(params.contentType, contentType1);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/1",
contentType: contentType1,
prefix: prefix1,
suffix: suffix1,
uploadKey: uploadKey1
})
);
}
// Get upload authorization for multi-file zip
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
// Verify ZIP hash
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
item2.key + '.zip'
);
deferreds.push({
promise: Zotero.Utilities.Internal.md5Async(tmpZipPath)
.then(function (md5) {
assert.equal(params.zipMD5, md5);
})
});
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
Zotero.debug(params);
assert.equal(params.md5, hash2);
assert.notEqual(params.zipMD5, hash2);
assert.equal(params.mtime, mtime2);
assert.equal(params.filename, filename2);
assert.equal(params.zipFilename, item2.key + ".zip");
assert.isTrue(parseInt(params.filesize) == params.filesize);
assert.equal(params.contentType, contentType2);
assert.equal(params.charset, charset2);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/2",
contentType: 'application/zip',
prefix: prefix2,
suffix: suffix2,
uploadKey: uploadKey2
})
);
}
// Upload single file to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
assert.equal(req.requestHeaders["Content-Type"], contentType1 + fixSinonBug);
assert.equal(req.requestBody.size, (new Blob([prefix1, File(file1), suffix1]).size));
req.respond(201, {}, "");
}
// Upload multi-file ZIP to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/2") {
assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
// Verify uploaded ZIP file
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
Zotero.Utilities.randomString() + '.zip'
);
let deferred = Zotero.Promise.defer();
deferreds.push(deferred);
var reader = new FileReader();
reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () {
try {
let file = yield OS.File.open(tmpZipPath, {
create: true
});
var contents = new Uint8Array(reader.result);
contents = contents.slice(prefix2.length, suffix2.length * -1);
yield file.write(contents);
yield file.close();
var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
.createInstance(Components.interfaces.nsIZipReader);
zr.open(Zotero.File.pathToFile(tmpZipPath));
zr.test(null);
var entries = zr.findEntries('*');
var entryNames = [];
while (entries.hasMore()) {
entryNames.push(entries.getNext());
}
assert.equal(entryNames.length, 2);
assert.sameMembers(entryNames, ['index.html', 'img.gif']);
assert.equal(zr.getEntry('index.html').realSize, size2);
assert.equal(zr.getEntry('img.gif').realSize, 42);
deferred.resolve();
}
catch (e) {
deferred.reject(e);
}
}));
reader.readAsArrayBuffer(req.requestBody);
req.respond(201, {}, "");
}
// Register single-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey1);
req.respond(
204,
{
"Last-Modified-Version": 10
},
""
);
}
// Register multi-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey2);
req.respond(
204,
{
"Last-Modified-Version": 15
},
""
);
}
})
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "POST",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
// TODO: One-step uploads
/*// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file`) {
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/json" + fixSinonBug
);
let params = JSON.parse(req.requestBody);
assert.equal(params.md5, hash);
assert.equal(params.mtime, mtime);
assert.equal(params.filename, filename);
assert.equal(params.size, size);
assert.equal(params.contentType, contentType);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3",
headers: {
"Content-Type": contentType,
"Content-MD5": hash,
//"Content-Length": params.size, process but don't return
//"x-amz-meta-"
},
uploadKey
})
);
}
else if (req.method == "PUT" && req.url == baseURL + "pretend-s3") {
assert.equal(req.requestHeaders["Content-Type"], contentType + fixSinonBug);
assert.instanceOf(req.requestBody, File);
req.respond(201, {}, "");
}
})*/
var result = yield engine.start();
yield Zotero.Promise.all(deferreds.map(d => d.promise));
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item1.id)), mtime1);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item1.id)), hash1);
assert.equal(item1.version, 10);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item2.id)), mtime2);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item2.id)), hash2);
assert.equal(item2.version, 15);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
})
it("should update local info for file that already exists on the server", function* () {
var { engine, client, caller } = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file: file });
item.version = 5;
yield item.saveTx();
var json = yield item.toJSON();
yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var newVersion = 10;
setResponse({
method: "POST",
url: "users/1/laststoragesync",
status: 200,
text: "" + (Math.round(new Date().getTime() / 1000) - 50000)
});
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
req.respond(
200,
{
"Content-Type": "application/json",
"Last-Modified-Version": newVersion
},
JSON.stringify({
exists: 1,
})
);
}
})
var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
setResponse({
method: "POST",
url: "users/1/laststoragesync",
status: 200,
text: "" + newStorageSyncTime
});
// TODO: One-step uploads
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.equal(item.version, newVersion);
// Check last sync time
assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
})
})
describe("#_processUploadFile()", function () {
it("should handle 412 with matching version and hash matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.ZFS_Module({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var itemJSON = yield item.toResponseJSON();
// Set saved hash to a different value, which should be overwritten
//
// We're also testing cases where a hash isn't set for a file (e.g., if the
// storage directory was transferred, the mtime doesn't match, but the file was
// never downloaded), but there's no difference in behavior
var dbHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, dbHash)
});
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-Match"] == dbHash) {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"ETag does not match current version of file"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncedHash(item.id), itemJSON.data.md5
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isFalse(result.fileSyncRequired);
})
it("should handle 412 with matching version and hash not matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.ZFS_Module({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var fileHash = yield item.attachmentHash;
var itemJSON = yield item.toResponseJSON();
itemJSON.data.md5 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"If-None-Match: * set but file exists"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.isNull(Zotero.Sync.Storage.Local.getSyncedHash(item.id));
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item.id),
Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isTrue(result.fileSyncRequired);
})
it("should handle 412 with greater version", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.ZFS_Module({
apiClient: client
})
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.version = 5;
item.synced = true;
yield item.saveTx();
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 10
},
"If-None-Match: * set but file exists"
);
}
})
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
assert.equal(item.version, 5);
assert.equal(item.synced, true);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isTrue(result.syncRequired);
})
})
})
})

34
test/tests/storageLocalTest.js
View File

@ -31,9 +31,7 @@ describe("Zotero.Sync.Storage.Local", function () {
yield Zotero.DB.executeTransaction(function* () { yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash); yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime); yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
}); });
// Update mtime and contents // Update mtime and contents
@ -50,7 +48,7 @@ describe("Zotero.Sync.Storage.Local", function () {
assert.equal(changed, true); assert.equal(changed, true);
assert.equal( assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)), (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD Zotero.Sync.Storage.Local.SYNC_STATE_TO_UPLOAD
); );
}) })
@ -64,9 +62,7 @@ describe("Zotero.Sync.Storage.Local", function () {
yield Zotero.DB.executeTransaction(function* () { yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash); yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime); yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
}); });
var libraryID = Zotero.Libraries.userLibraryID; var libraryID = Zotero.Libraries.userLibraryID;
@ -76,7 +72,7 @@ describe("Zotero.Sync.Storage.Local", function () {
yield item.eraseTx(); yield item.eraseTx();
assert.isFalse(changed); assert.isFalse(changed);
assert.equal(syncState, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); assert.equal(syncState, Zotero.Sync.Storage.Local.SYNC_STATE_IN_SYNC);
}) })
it("should skip a file if mod time has changed but contents haven't", function* () { it("should skip a file if mod time has changed but contents haven't", function* () {
@ -91,9 +87,7 @@ describe("Zotero.Sync.Storage.Local", function () {
yield Zotero.DB.executeTransaction(function* () { yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash); yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime); yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
}); });
// Update mtime, but not contents // Update mtime, but not contents
@ -109,7 +103,7 @@ describe("Zotero.Sync.Storage.Local", function () {
yield item.eraseTx(); yield item.eraseTx();
assert.isFalse(changed); assert.isFalse(changed);
assert.equal(syncState, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); assert.equal(syncState, Zotero.Sync.Storage.Local.SYNC_STATE_IN_SYNC);
assert.equal(syncedModTime, newModTime); assert.equal(syncedModTime, newModTime);
}) })
}) })
@ -217,12 +211,8 @@ describe("Zotero.Sync.Storage.Local", function () {
json3.mtime = now - 20000; json3.mtime = now - 20000;
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]); yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]);
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(item1.id, "in_conflict");
item1.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT yield Zotero.Sync.Storage.Local.setSyncState(item3.id, "in_conflict");
);
yield Zotero.Sync.Storage.Local.setSyncState(
item3.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
);
var conflicts = yield Zotero.Sync.Storage.Local.getConflicts(libraryID); var conflicts = yield Zotero.Sync.Storage.Local.getConflicts(libraryID);
assert.lengthOf(conflicts, 2); assert.lengthOf(conflicts, 2);
@ -269,10 +259,10 @@ describe("Zotero.Sync.Storage.Local", function () {
yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]); yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]);
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(
item1.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT item1.id, "in_conflict"
); );
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(
item3.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT item3.id, "in_conflict"
); );
var promise = waitForWindow('chrome://zotero/content/merge.xul', function (dialog) { var promise = waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
@ -317,11 +307,11 @@ describe("Zotero.Sync.Storage.Local", function () {
yield assert.eventually.equal( yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item1.id), Zotero.Sync.Storage.Local.getSyncState(item1.id),
Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_UPLOAD
); );
yield assert.eventually.equal( yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item3.id), Zotero.Sync.Storage.Local.getSyncState(item3.id),
Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD Zotero.Sync.Storage.Local.SYNC_STATE_FORCE_DOWNLOAD
); );
}) })
}) })

69
test/tests/syncEngineTest.js
View File

@ -107,11 +107,6 @@ describe("Zotero.Sync.Data.Engine", function () {
yield Zotero.Users.setCurrentUserID(1); yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser"); yield Zotero.Users.setCurrentUsername("testuser");
}) })
after(function* () {
yield resetDB({
thisArg: this
});
})
describe("Syncing", function () { describe("Syncing", function () {
it("should download items into a new library", function* () { it("should download items into a new library", function* () {
@ -415,6 +410,70 @@ describe("Zotero.Sync.Data.Engine", function () {
} }
}) })
it("should upload synced storage properties", function* () {
({ engine, client, caller } = yield setup());
var libraryID = Zotero.Libraries.userLibraryID;
var lastLibraryVersion = 2;
yield Zotero.Libraries.setVersion(libraryID, lastLibraryVersion);
var item = new Zotero.Item('attachment');
item.attachmentLinkMode = 'imported_file';
item.attachmentFilename = 'test1.txt';
yield item.saveTx();
var mtime = new Date().getTime();
var md5 = '57f8a4fda823187b91e1191487b87fe6';
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, md5);
});
var itemResponseJSON = yield item.toResponseJSON();
itemResponseJSON.version = itemResponseJSON.data.version = lastLibraryVersion;
itemResponseJSON.data.mtime = mtime;
itemResponseJSON.data.md5 = md5;
server.respond(function (req) {
if (req.method == "POST") {
if (req.url == baseURL + "users/1/items") {
let json = JSON.parse(req.requestBody);
assert.lengthOf(json, 1);
let itemJSON = json[0];
assert.equal(itemJSON.key, item.key);
assert.equal(itemJSON.version, 0);
assert.equal(itemJSON.mtime, mtime);
assert.equal(itemJSON.md5, md5);
req.respond(
200,
{
"Content-Type": "application/json",
"Last-Modified-Version": lastLibraryVersion
},
JSON.stringify({
successful: {
"0": itemResponseJSON
},
unchanged: {},
failed: {}
})
);
return;
}
}
})
yield engine.start();
// Check data in cache
var json = yield Zotero.Sync.Data.Local.getCacheObject(
'item', libraryID, item.key, lastLibraryVersion
);
assert.equal(json.data.mtime, mtime);
assert.equal(json.data.md5, md5);
})
it("should update local objects with remotely saved version after uploading if necessary", function* () { it("should update local objects with remotely saved version after uploading if necessary", function* () {
({ engine, client, caller } = yield setup()); ({ engine, client, caller } = yield setup());

14
test/tests/syncLocalTest.js
View File

@ -149,7 +149,7 @@ describe("Zotero.Sync.Data.Local", function() {
var id = Zotero.Items.getIDFromLibraryAndKey(libraryID, key); var id = Zotero.Items.getIDFromLibraryAndKey(libraryID, key);
assert.equal( assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(id)), (yield Zotero.Sync.Storage.Local.getSyncState(id)),
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD
); );
}) })
@ -170,9 +170,7 @@ describe("Zotero.Sync.Data.Local", function() {
yield Zotero.Sync.Storage.Local.setSyncedHash( yield Zotero.Sync.Storage.Local.setSyncedHash(
item.id, (yield item.attachmentHash) item.id, (yield item.attachmentHash)
); );
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
}); });
// Simulate download of version with updated attachment // Simulate download of version with updated attachment
@ -191,7 +189,7 @@ describe("Zotero.Sync.Data.Local", function() {
assert.equal( assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)), (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD
); );
}) })
@ -213,9 +211,7 @@ describe("Zotero.Sync.Data.Local", function() {
yield Zotero.Sync.Storage.Local.setSyncedHash( yield Zotero.Sync.Storage.Local.setSyncedHash(
item.id, (yield item.attachmentHash) item.id, (yield item.attachmentHash)
); );
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(item.id, "in_sync");
item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
);
}); });
// Modify title locally, leaving item unsynced // Modify title locally, leaving item unsynced
@ -237,7 +233,7 @@ describe("Zotero.Sync.Data.Local", function() {
assert.equal(item.getField('title'), newTitle); assert.equal(item.getField('title'), newTitle);
assert.equal( assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)), (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD Zotero.Sync.Storage.Local.SYNC_STATE_TO_DOWNLOAD
); );
}) })
}) })

764
test/tests/webdavTest.js Normal file
View File

@ -0,0 +1,764 @@
"use strict";
describe("Zotero.Sync.Storage.Mode.WebDAV", function () {
//
// Setup
//
Components.utils.import("resource://zotero-unit/httpd.js");
var apiKey = Zotero.Utilities.randomString(24);
var apiPort = 16213;
var apiURL = `http://localhost:${apiPort}/`;
var davScheme = "http";
var davPort = 16214;
var davBasePath = "/webdav/";
var davHostPath = `localhost:${davPort}${davBasePath}`;
var davUsername = "user";
var davPassword = "password";
var davURL = `${davScheme}://${davUsername}:${davPassword}@${davHostPath}`;
var win, controller, server, requestCount;
var responses = {};
function setResponse(response) {
setHTTPResponse(server, davURL, response, responses);
}
function resetRequestCount() {
requestCount = server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length;
}
function assertRequestCount(count) {
assert.equal(
server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length - requestCount,
count
);
}
function generateLastSyncID() {
return "" + Zotero.Utilities.randomString(controller._lastSyncIDLength);
}
function parseQueryString(str) {
var queryStringParams = str.split('&');
var params = {};
for (let param of queryStringParams) {
let [ key, val ] = param.split('=');
params[key] = decodeURIComponent(val);
}
return params;
}
function assertAPIKey(request) {
assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
}
before(function* () {
controller = new Zotero.Sync.Storage.Mode.WebDAV;
Zotero.Prefs.set("sync.storage.scheme", davScheme);
Zotero.Prefs.set("sync.storage.url", davHostPath);
Zotero.Prefs.set("sync.storage.username", davUsername);
controller.password = davPassword;
})
beforeEach(function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
server = sinon.fakeServer.create();
server.autoRespond = true;
this.httpd = new HttpServer();
this.httpd.start(davPort);
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser");
Zotero.Sync.Storage.Local.setModeForLibrary(Zotero.Libraries.userLibraryID, 'webdav');
// Set download-on-sync by default
Zotero.Sync.Storage.Local.downloadOnSync(
Zotero.Libraries.userLibraryID, true
);
})
var setup = Zotero.Promise.coroutine(function* (options = {}) {
var engine = new Zotero.Sync.Storage.Engine({
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
controller,
stopOnError: true
});
if (!controller.verified) {
setResponse({
method: "OPTIONS",
url: "zotero/",
headers: {
DAV: 1
},
status: 200
})
setResponse({
method: "PROPFIND",
url: "zotero/",
status: 207
})
setResponse({
method: "PUT",
url: "zotero/zotero-test-file.prop",
status: 201
})
setResponse({
method: "GET",
url: "zotero/zotero-test-file.prop",
status: 200
})
setResponse({
method: "DELETE",
url: "zotero/zotero-test-file.prop",
status: 200
})
yield controller.checkServer();
yield controller.cacheCredentials();
}
resetRequestCount();
return engine;
})
afterEach(function* () {
var defer = new Zotero.Promise.defer();
this.httpd.stop(() => defer.resolve());
yield defer.promise;
})
after(function* () {
if (win) {
win.close();
}
})
//
// Tests
//
describe("Syncing", function () {
beforeEach(function* () {
win = yield loadZoteroPane();
})
afterEach(function () {
win.close();
})
it("should skip downloads if not marked as needed", function* () {
var engine = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
var result = yield engine.start();
assertRequestCount(0);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should ignore a remotely missing file", function* () {
var engine = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 404
});
var result = yield engine.start();
assertRequestCount(1);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isFalse(library.storageDownloadNeeded);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should handle a remotely failing .prop file", function* () {
var engine = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 500
});
// TODO: In stopOnError mode, the promise is rejected.
// This should probably test with stopOnError mode turned off instead.
var e = yield getPromiseError(engine.start());
assert.include(
e.message,
Zotero.getString('sync.storage.error.webdav.requestError', [500, "GET"])
);
assertRequestCount(1);
assert.isTrue(library.storageDownloadNeeded);
assert.equal(library.storageVersion, 0);
})
it("should handle a remotely failing .zip file", function* () {
var engine = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 200,
text: '<properties version="1">'
+ '<mtime>1234567890</mtime>'
+ '<hash>8286300a280f64a4b5cfaac547c21d32</hash>'
+ '</properties>'
});
this.httpd.registerPathHandler(
`${davBasePath}zotero/${item.key}.zip`,
{
handle: function (request, response) {
response.setStatusLine(null, 500, null);
}
}
);
// TODO: In stopOnError mode, the promise is rejected.
// This should probably test with stopOnError mode turned off instead.
var e = yield getPromiseError(engine.start());
assert.include(
e.message,
Zotero.getString('sync.storage.error.webdav.requestError', [500, "GET"])
);
assert.isTrue(library.storageDownloadNeeded);
assert.equal(library.storageVersion, 0);
})
it("should download a missing file", function* () {
var engine = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var fileName = "test.txt";
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:' + fileName;
// TODO: Test binary data
var text = Zotero.Utilities.randomString();
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
// Create ZIP file containing above text file
var tmpPath = Zotero.getTempDirectory().path;
var tmpID = "webdav_download_" + Zotero.Utilities.randomString();
var zipDirPath = OS.Path.join(tmpPath, tmpID);
var zipPath = OS.Path.join(tmpPath, tmpID + ".zip");
yield OS.File.makeDir(zipDirPath);
yield Zotero.File.putContentsAsync(OS.Path.join(zipDirPath, fileName), text);
yield Zotero.File.zipDirectory(zipDirPath, zipPath);
yield OS.File.removeDir(zipDirPath);
yield Zotero.Promise.delay(1000);
var zipContents = yield Zotero.File.getBinaryContentsAsync(zipPath);
var mtime = "1441252524905";
var md5 = yield Zotero.Utilities.Internal.md5Async(zipPath);
yield OS.File.remove(zipPath);
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 200,
text: '<properties version="1">'
+ `<mtime>${mtime}</mtime>`
+ `<hash>${md5}</hash>`
+ '</properties>'
});
this.httpd.registerPathHandler(
`${davBasePath}zotero/${item.key}.zip`,
{
handle: function (request, response) {
response.setStatusLine(null, 200, "OK");
response.write(zipContents);
}
}
);
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync());
assert.equal(contents, text);
assert.isFalse(library.storageDownloadNeeded);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should upload new files", function* () {
var engine = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.synced = true;
yield item.saveTx();
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var fileContents = yield Zotero.File.getContentsAsync(path);
var deferreds = [];
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 404
});
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
if (req.method == "PUT" && req.url == `${davURL}zotero/${item.key}.zip`) {
assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
let deferred = Zotero.Promise.defer();
deferreds.push(deferred);
var reader = new FileReader();
reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () {
try {
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
Zotero.Utilities.randomString() + '.zip'
);
let file = yield OS.File.open(tmpZipPath, {
create: true
});
var contents = new Uint8Array(reader.result);
yield file.write(contents);
yield file.close();
// Make sure ZIP file contains the necessary entries
var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
.createInstance(Components.interfaces.nsIZipReader);
zr.open(Zotero.File.pathToFile(tmpZipPath));
zr.test(null);
var entries = zr.findEntries('*');
var entryNames = [];
while (entries.hasMore()) {
entryNames.push(entries.getNext());
}
assert.equal(entryNames.length, 1);
assert.sameMembers(entryNames, [filename]);
assert.equal(zr.getEntry(filename).realSize, size);
yield OS.File.remove(tmpZipPath);
deferred.resolve();
}
catch (e) {
deferred.reject(e);
}
}));
reader.readAsArrayBuffer(req.requestBody);
req.respond(201, { "Fake-Server-Match": 1 }, "");
}
else if (req.method == "PUT" && req.url == `${davURL}zotero/${item.key}.prop`) {
var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
.createInstance(Components.interfaces.nsIDOMParser);
var doc = parser.parseFromString(req.requestBody, "text/xml");
assert.equal(
doc.documentElement.getElementsByTagName('mtime')[0].textContent, mtime
);
assert.equal(
doc.documentElement.getElementsByTagName('hash')[0].textContent, hash
);
req.respond(204, { "Fake-Server-Match": 1 }, "");
}
});
var result = yield engine.start();
yield Zotero.Promise.all(deferreds.map(d => d.promise));
assertRequestCount(3);
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isTrue(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.isFalse(item.synced);
})
it("should upload an updated file", function* () {
var engine = yield setup();
var file = getTestDataDirectory();
file.append('test.txt');
var item = yield Zotero.Attachments.importFromFile({ file });
item.synced = true;
yield item.saveTx();
yield Zotero.DB.executeTransaction(function* () {
// Set an mtime in the past
yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
item.id,
new Date(Date.now() - 10000)
);
// And a different hash
yield Zotero.Sync.Storage.Local.setSyncedHash(
item.id, "3a2f092dd62178eb8bbfda42e07e64da"
);
});
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
setResponse({
method: "DELETE",
url: `zotero/${item.key}.prop`,
status: 204
});
setResponse({
method: "PUT",
url: `zotero/${item.key}.zip`,
status: 204
});
setResponse({
method: "PUT",
url: `zotero/${item.key}.prop`,
status: 204
});
var result = yield engine.start();
assertRequestCount(3);
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isTrue(result.syncRequired);
assert.isFalse(result.fileSyncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.isFalse(item.synced);
})
it("should skip upload that already exists on the server", function* () {
var engine = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.synced = true;
yield item.saveTx();
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var fileContents = yield Zotero.File.getContentsAsync(path);
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 200,
text: '<properties version="1">'
+ `<mtime>${mtime}</mtime>`
+ `<hash>${hash}</hash>`
+ '</properties>'
});
var result = yield engine.start();
assertRequestCount(1);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local object
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.isFalse(item.synced);
})
it("should mark item as in conflict if mod time and hash on storage server don't match synced values", function* () {
var engine = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.synced = true;
yield item.saveTx();
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var fileContents = yield Zotero.File.getContentsAsync(path);
var newModTime = mtime + 5000;
var newHash = "4f69f43d8ac8788190b13ff7f4a0a915";
setResponse({
method: "GET",
url: `zotero/${item.key}.prop`,
status: 200,
text: '<properties version="1">'
+ `<mtime>${newModTime}</mtime>`
+ `<hash>${newHash}</hash>`
+ '</properties>'
});
var result = yield engine.start();
assertRequestCount(1);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isTrue(result.fileSyncRequired);
// Check local object
//
// Item should be marked as in conflict
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.Local.SYNC_STATE_IN_CONFLICT
);
// Synced mod time should have been changed, because that's what's shown in the
// conflict dialog
assert.equal(
(yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), newModTime
);
assert.isTrue(item.synced);
})
})
describe("#purgeDeletedStorageFiles()", function () {
beforeEach(function () {
resetRequestCount();
})
it("should delete files on storage server that were deleted locally", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.synced = true;
yield item.saveTx();
yield item.eraseTx();
assert.lengthOf((yield Zotero.Sync.Storage.Local.getDeletedFiles(libraryID)), 1);
setResponse({
method: "DELETE",
url: `zotero/${item.key}.prop`,
status: 204
});
setResponse({
method: "DELETE",
url: `zotero/${item.key}.zip`,
status: 204
});
var results = yield controller.purgeDeletedStorageFiles(libraryID);
assertRequestCount(2);
assert.lengthOf(results.deleted, 2);
assert.sameMembers(results.deleted, [`${item.key}.prop`, `${item.key}.zip`]);
assert.lengthOf(results.missing, 0);
assert.lengthOf(results.error, 0);
// Storage delete log should be empty
assert.lengthOf((yield Zotero.Sync.Storage.Local.getDeletedFiles(libraryID)), 0);
})
})
describe("#purgeOrphanedStorageFiles()", function () {
beforeEach(function () {
resetRequestCount();
Zotero.Prefs.clear('lastWebDAVOrphanPurge');
})
it("should delete orphaned files more than a week older than the last sync time", function* () {
var library = Zotero.Libraries.userLibrary;
library.updateLastSyncTime();
yield library.saveTx();
const daysBeforeSyncTime = 7;
var beforeTime = new Date(Date.now() - (daysBeforeSyncTime * 86400 * 1000 + 1)).toUTCString();
var currentTime = new Date(Date.now() - 3600000).toUTCString();
setResponse({
method: "PROPFIND",
url: `zotero/`,
status: 207,
headers: {
"Content-Type": 'text/xml; charset="utf-8"'
},
text: '<?xml version="1.0" encoding="utf-8"?>'
+ '<D:multistatus xmlns:D="DAV:" xmlns:ns0="DAV:">'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${beforeTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/lastsync.txt</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${beforeTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/lastsync</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${beforeTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/AAAAAAAA.zip</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${beforeTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/AAAAAAAA.prop</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${beforeTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/BBBBBBBB.zip</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${currentTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '<D:response xmlns:lp1="DAV:" xmlns:lp2="http://apache.org/dav/props/">'
+ `<D:href>${davBasePath}zotero/BBBBBBBB.prop</D:href>`
+ '<D:propstat>'
+ '<D:prop>'
+ `<lp1:getlastmodified>${currentTime}</lp1:getlastmodified>`
+ '</D:prop>'
+ '<D:status>HTTP/1.1 200 OK</D:status>'
+ '</D:propstat>'
+ '</D:response>'
+ '</D:multistatus>'
});
setResponse({
method: "DELETE",
url: 'zotero/AAAAAAAA.prop',
status: 204
});
setResponse({
method: "DELETE",
url: 'zotero/AAAAAAAA.zip',
status: 204
});
setResponse({
method: "DELETE",
url: 'zotero/lastsync.txt',
status: 204
});
setResponse({
method: "DELETE",
url: 'zotero/lastsync',
status: 204
});
yield controller.purgeOrphanedStorageFiles();
assertRequestCount(5);
})
it("shouldn't purge if purged recently", function* () {
Zotero.Prefs.set("lastWebDAVOrphanPurge", Math.round(new Date().getTime() / 1000) - 3600);
yield assert.eventually.equal(controller.purgeOrphanedStorageFiles(), false);
assertRequestCount(0);
})
})
})

777
test/tests/zfsTest.js Normal file
View File

@ -0,0 +1,777 @@
"use strict";
describe("Zotero.Sync.Storage.Mode.ZFS", function () {
//
// Setup
//
Components.utils.import("resource://zotero-unit/httpd.js");
var apiKey = Zotero.Utilities.randomString(24);
var port = 16213;
var baseURL = `http://localhost:${port}/`;
var win, server, requestCount;
var responses = {};
function setResponse(response) {
setHTTPResponse(server, baseURL, response, responses);
}
function resetRequestCount() {
requestCount = server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length;
}
function assertRequestCount(count) {
assert.equal(
server.requests.filter(r => r.responseHeaders["Fake-Server-Match"]).length - requestCount,
count
);
}
function parseQueryString(str) {
var queryStringParams = str.split('&');
var params = {};
for (let param of queryStringParams) {
let [ key, val ] = param.split('=');
params[key] = decodeURIComponent(val);
}
return params;
}
function assertAPIKey(request) {
assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
}
//
// Tests
//
beforeEach(function* () {
yield resetDB({
thisArg: this,
skipBundledFiles: true
});
win = yield loadZoteroPane();
Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
server = sinon.fakeServer.create();
server.autoRespond = true;
this.httpd = new HttpServer();
this.httpd.start(port);
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("testuser");
Zotero.Sync.Storage.Local.setModeForLibrary(Zotero.Libraries.userLibraryID, 'zfs');
// Set download-on-sync by default
Zotero.Sync.Storage.Local.downloadOnSync(
Zotero.Libraries.userLibraryID, true
);
resetRequestCount();
})
var setup = Zotero.Promise.coroutine(function* (options = {}) {
Components.utils.import("resource://zotero/concurrentCaller.js");
var caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true;
Components.utils.import("resource://zotero/config.js");
var client = new Zotero.Sync.APIClient({
baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
apiKey,
caller,
background: options.background || true
});
var engine = new Zotero.Sync.Storage.Engine({
libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
controller: new Zotero.Sync.Storage.Mode.ZFS({
apiClient: client
}),
stopOnError: true
});
return { engine, client, caller };
})
afterEach(function* () {
var defer = new Zotero.Promise.defer();
this.httpd.stop(() => defer.resolve());
yield defer.promise;
win.close();
})
after(function* () {
this.timeout(60000);
//yield resetDB();
win.close();
})
describe("Syncing", function () {
it("should skip downloads if not marked as needed", function* () {
var { engine, client, caller } = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
var result = yield engine.start();
assertRequestCount(0);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should ignore a remotely missing file", function* () {
var { engine, client, caller } = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 404, null);
}
}
);
var result = yield engine.start();
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isFalse(library.storageDownloadNeeded);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should handle a remotely failing file", function* () {
var { engine, client, caller } = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
response.setStatusLine(null, 500, null);
}
}
);
// TODO: In stopOnError mode, this the promise is rejected.
// This should probably test with stopOnError mode turned off instead.
var e = yield getPromiseError(engine.start());
assert.equal(e.message, Zotero.Sync.Storage.defaultError);
assert.isTrue(library.storageDownloadNeeded);
assert.equal(library.storageVersion, 0);
})
it("should download a missing file", function* () {
var { engine, client, caller } = yield setup();
var library = Zotero.Libraries.userLibrary;
library.libraryVersion = 5;
yield library.saveTx();
library.storageDownloadNeeded = true;
var item = new Zotero.Item("attachment");
item.attachmentLinkMode = 'imported_file';
item.attachmentPath = 'storage:test.txt';
// TODO: Test binary data
var text = Zotero.Utilities.randomString();
yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
var mtime = "1441252524905";
var md5 = Zotero.Utilities.Internal.md5(text)
var s3Path = `pretend-s3/${item.key}`;
this.httpd.registerPathHandler(
`/users/1/items/${item.key}/file`,
{
handle: function (request, response) {
if (!request.hasHeader('Zotero-API-Key')) {
response.setStatusLine(null, 403, "Forbidden");
return;
}
var key = request.getHeader('Zotero-API-Key');
if (key != apiKey) {
response.setStatusLine(null, 403, "Invalid key");
return;
}
response.setStatusLine(null, 302, "Found");
response.setHeader("Zotero-File-Modification-Time", mtime, false);
response.setHeader("Zotero-File-MD5", md5, false);
response.setHeader("Zotero-File-Compressed", "No", false);
response.setHeader("Location", baseURL + s3Path, false);
}
}
);
this.httpd.registerPathHandler(
"/" + s3Path,
{
handle: function (request, response) {
response.setStatusLine(null, 200, "OK");
response.write(text);
}
}
);
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync());
assert.equal(contents, text);
assert.isFalse(library.storageDownloadNeeded);
assert.equal(library.storageVersion, library.libraryVersion);
})
it("should upload new files", function* () {
var { engine, client, caller } = yield setup();
// Single file
var file1 = getTestDataDirectory();
file1.append('test.png');
var item1 = yield Zotero.Attachments.importFromFile({ file: file1 });
var mtime1 = yield item1.attachmentModificationTime;
var hash1 = yield item1.attachmentHash;
var path1 = item1.getFilePath();
var filename1 = 'test.png';
var size1 = (yield OS.File.stat(path1)).size;
var contentType1 = 'image/png';
var prefix1 = Zotero.Utilities.randomString();
var suffix1 = Zotero.Utilities.randomString();
var uploadKey1 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
// HTML file with auxiliary image
var file2 = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
var parentItem = yield createDataObject('item');
var item2 = yield Zotero.Attachments.importSnapshotFromFile({
file: file2,
url: 'http://example.com/',
parentItemID: parentItem.id,
title: 'Test',
contentType: 'text/html',
charset: 'utf-8'
});
var mtime2 = yield item2.attachmentModificationTime;
var hash2 = yield item2.attachmentHash;
var path2 = item2.getFilePath();
var filename2 = 'index.html';
var size2 = (yield OS.File.stat(path2)).size;
var contentType2 = 'text/html';
var charset2 = 'utf-8';
var prefix2 = Zotero.Utilities.randomString();
var suffix2 = Zotero.Utilities.randomString();
var uploadKey2 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
var deferreds = [];
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.md5, hash1);
assert.equal(params.mtime, mtime1);
assert.equal(params.filename, filename1);
assert.equal(params.filesize, size1);
assert.equal(params.contentType, contentType1);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/1",
contentType: contentType1,
prefix: prefix1,
suffix: suffix1,
uploadKey: uploadKey1
})
);
}
// Get upload authorization for multi-file zip
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
// Verify ZIP hash
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
item2.key + '.zip'
);
deferreds.push({
promise: Zotero.Utilities.Internal.md5Async(tmpZipPath)
.then(function (md5) {
assert.equal(params.zipMD5, md5);
})
});
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
Zotero.debug(params);
assert.equal(params.md5, hash2);
assert.notEqual(params.zipMD5, hash2);
assert.equal(params.mtime, mtime2);
assert.equal(params.filename, filename2);
assert.equal(params.zipFilename, item2.key + ".zip");
assert.isTrue(parseInt(params.filesize) == params.filesize);
assert.equal(params.contentType, contentType2);
assert.equal(params.charset, charset2);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3/2",
contentType: 'application/zip',
prefix: prefix2,
suffix: suffix2,
uploadKey: uploadKey2
})
);
}
// Upload single file to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
assert.equal(req.requestHeaders["Content-Type"], contentType1 + fixSinonBug);
assert.equal(req.requestBody.size, (new Blob([prefix1, File(file1), suffix1]).size));
req.respond(201, {}, "");
}
// Upload multi-file ZIP to S3
else if (req.method == "POST" && req.url == baseURL + "pretend-s3/2") {
assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
// Verify uploaded ZIP file
let tmpZipPath = OS.Path.join(
Zotero.getTempDirectory().path,
Zotero.Utilities.randomString() + '.zip'
);
let deferred = Zotero.Promise.defer();
deferreds.push(deferred);
var reader = new FileReader();
reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () {
try {
let file = yield OS.File.open(tmpZipPath, {
create: true
});
var contents = new Uint8Array(reader.result);
contents = contents.slice(prefix2.length, suffix2.length * -1);
yield file.write(contents);
yield file.close();
var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
.createInstance(Components.interfaces.nsIZipReader);
zr.open(Zotero.File.pathToFile(tmpZipPath));
zr.test(null);
var entries = zr.findEntries('*');
var entryNames = [];
while (entries.hasMore()) {
entryNames.push(entries.getNext());
}
assert.equal(entryNames.length, 2);
assert.sameMembers(entryNames, ['index.html', 'img.gif']);
assert.equal(zr.getEntry('index.html').realSize, size2);
assert.equal(zr.getEntry('img.gif').realSize, 42);
deferred.resolve();
}
catch (e) {
deferred.reject(e);
}
}));
reader.readAsArrayBuffer(req.requestBody);
req.respond(201, {}, "");
}
// Register single-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item1.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey1);
req.respond(
204,
{
"Last-Modified-Version": 10
},
""
);
}
// Register multi-file upload
else if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item2.key}/file`
&& req.requestBody.indexOf('upload=') != -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
let parts = req.requestBody.split('&');
let params = {};
for (let part of parts) {
let [key, val] = part.split('=');
params[key] = decodeURIComponent(val);
}
assert.equal(params.upload, uploadKey2);
req.respond(
204,
{
"Last-Modified-Version": 15
},
""
);
}
})
// TODO: One-step uploads
/*// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file`) {
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/json" + fixSinonBug
);
let params = JSON.parse(req.requestBody);
assert.equal(params.md5, hash);
assert.equal(params.mtime, mtime);
assert.equal(params.filename, filename);
assert.equal(params.size, size);
assert.equal(params.contentType, contentType);
req.respond(
200,
{
"Content-Type": "application/json"
},
JSON.stringify({
url: baseURL + "pretend-s3",
headers: {
"Content-Type": contentType,
"Content-MD5": hash,
//"Content-Length": params.size, process but don't return
//"x-amz-meta-"
},
uploadKey
})
);
}
else if (req.method == "PUT" && req.url == baseURL + "pretend-s3") {
assert.equal(req.requestHeaders["Content-Type"], contentType + fixSinonBug);
assert.instanceOf(req.requestBody, File);
req.respond(201, {}, "");
}
})*/
var result = yield engine.start();
yield Zotero.Promise.all(deferreds.map(d => d.promise));
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item1.id)), mtime1);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item1.id)), hash1);
assert.equal(item1.version, 10);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item2.id)), mtime2);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item2.id)), hash2);
assert.equal(item2.version, 15);
})
it("should update local info for file that already exists on the server", function* () {
var { engine, client, caller } = yield setup();
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file: file });
item.version = 5;
yield item.saveTx();
var json = yield item.toJSON();
yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
var mtime = yield item.attachmentModificationTime;
var hash = yield item.attachmentHash;
var path = item.getFilePath();
var filename = 'test.png';
var size = (yield OS.File.stat(path)).size;
var contentType = 'image/png';
var newVersion = 10;
// https://github.com/cjohansen/Sinon.JS/issues/607
let fixSinonBug = ";charset=utf-8";
server.respond(function (req) {
// Get upload authorization for single file
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1) {
assertAPIKey(req);
assert.equal(req.requestHeaders["If-None-Match"], "*");
assert.equal(
req.requestHeaders["Content-Type"],
"application/x-www-form-urlencoded" + fixSinonBug
);
req.respond(
200,
{
"Content-Type": "application/json",
"Last-Modified-Version": newVersion
},
JSON.stringify({
exists: 1,
})
);
}
})
// TODO: One-step uploads
var result = yield engine.start();
assert.isTrue(result.localChanges);
assert.isTrue(result.remoteChanges);
assert.isFalse(result.syncRequired);
// Check local objects
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
assert.equal(item.version, newVersion);
})
})
describe("#_processUploadFile()", function () {
it("should handle 412 with matching version and hash matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var itemJSON = yield item.toResponseJSON();
itemJSON.data.mtime = yield item.attachmentModificationTime;
itemJSON.data.md5 = yield item.attachmentHash;
// Set saved hash to a different value, which should be overwritten
//
// We're also testing cases where a hash isn't set for a file (e.g., if the
// storage directory was transferred, the mtime doesn't match, but the file was
// never downloaded), but there's no difference in behavior
var dbHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
yield Zotero.DB.executeTransaction(function* () {
yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, dbHash)
});
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-Match"] == dbHash) {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"ETag does not match current version of file"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncedHash(item.id),
(yield item.attachmentHash)
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isFalse(result.fileSyncRequired);
})
it("should handle 412 with matching version and hash not matching local file", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
apiClient: client
})
var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
var item = yield Zotero.Attachments.importFromFile({ file: filePath });
item.version = 5;
item.synced = true;
yield item.saveTx();
var fileHash = yield item.attachmentHash;
var itemJSON = yield item.toResponseJSON();
itemJSON.data.md5 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 5
},
"If-None-Match: * set but file exists"
);
}
})
setResponse({
method: "GET",
url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
status: 200,
text: JSON.stringify([itemJSON])
});
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
yield assert.eventually.isNull(Zotero.Sync.Storage.Local.getSyncedHash(item.id));
yield assert.eventually.equal(
Zotero.Sync.Storage.Local.getSyncState(item.id),
Zotero.Sync.Storage.Local.SYNC_STATE_IN_CONFLICT
);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isFalse(result.syncRequired);
assert.isTrue(result.fileSyncRequired);
})
it("should handle 412 with greater version", function* () {
var { engine, client, caller } = yield setup();
var zfs = new Zotero.Sync.Storage.Mode.ZFS({
apiClient: client
})
var file = getTestDataDirectory();
file.append('test.png');
var item = yield Zotero.Attachments.importFromFile({ file });
item.version = 5;
item.synced = true;
yield item.saveTx();
server.respond(function (req) {
if (req.method == "POST"
&& req.url == `${baseURL}users/1/items/${item.key}/file`
&& req.requestBody.indexOf('upload=') == -1
&& req.requestHeaders["If-None-Match"] == "*") {
req.respond(
412,
{
"Content-Type": "application/json",
"Last-Modified-Version": 10
},
"If-None-Match: * set but file exists"
);
}
})
var result = yield zfs._processUploadFile({
name: item.libraryKey
});
assert.equal(item.version, 5);
assert.equal(item.synced, true);
assert.isFalse(result.localChanges);
assert.isFalse(result.remoteChanges);
assert.isTrue(result.syncRequired);
})
})
})

4
test/tests/zoteroPaneTest.js
View File

@ -148,9 +148,7 @@ describe("ZoteroPane", function() {
// TODO: Test binary data // TODO: Test binary data
var text = Zotero.Utilities.randomString(); var text = Zotero.Utilities.randomString();
yield item.saveTx(); yield item.saveTx();
yield Zotero.Sync.Storage.Local.setSyncState( yield Zotero.Sync.Storage.Local.setSyncState(item.id, "to_download");
item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
);
var mtime = "1441252524000"; var mtime = "1441252524000";
var md5 = Zotero.Utilities.Internal.md5(text) var md5 = Zotero.Utilities.Internal.md5(text)