diff --git a/chrome/content/zotero/bindings/attachmentbox.xml b/chrome/content/zotero/bindings/attachmentbox.xml
index 6d98e7a5f..fd53b1a06 100644
--- a/chrome/content/zotero/bindings/attachmentbox.xml
+++ b/chrome/content/zotero/bindings/attachmentbox.xml
@@ -57,6 +57,7 @@
Zotero.debug("Setting mode to '" + val + "'");
this.editable = false;
+ this.synchronous = false;
this.displayURL = false;
this.displayFileName = false;
this.clickableLink = false;
@@ -93,6 +94,7 @@
break;
case 'merge':
+ this.synchronous = true;
this.displayURL = true;
this.displayFileName = true;
this.displayAccessed = true;
@@ -102,6 +104,7 @@
break;
case 'mergeedit':
+ this.synchronous = true;
this.editable = true;
this.displayURL = true;
this.displayFileName = true;
@@ -112,6 +115,13 @@
this.displayDateModified = true;
break;
+ case 'filemerge':
+ this.synchronous = true;
+ this.displayURL = true;
+ this.displayFileName = true;
+ this.displayDateModified = true;
+ break;
+
default:
throw ("Invalid mode '" + val + "' in attachmentbox.xml");
}
@@ -123,18 +133,16 @@
</property>
<field name="_item"/>
- <property name="item"
- onget="return this._item;"
- onset="this._item = val; this.refresh();">
+ <property name="item" onget="return this._item;">
+ <setter><![CDATA[
+ if (!(val instanceof Zotero.Item)) {
+ throw new Error("'item' must be a Zotero.Item");
+ }
+ this._item = val;
+ this.refresh();
+ ]]></setter>
</property>
- <!-- .ref is an alias for .item -->
- <property name="ref"
- onget="return this._item;"
- onset="this._item = val; this.refresh();">
- </property>
-
-
<!-- Methods -->
<constructor>
@@ -167,125 +175,122 @@
<method name="refresh">
<body><![CDATA[
- Zotero.spawn(function* () {
- Zotero.debug('Refreshing attachment box');
+ Zotero.debug('Refreshing attachment box');
+
+ var attachmentBox = document.getAnonymousNodes(this)[0];
+ var title = this._id('title');
+ var fileNameRow = this._id('fileNameRow');
+ var urlField = this._id('url');
+ var accessed = this._id('accessedRow');
+ var pagesRow = this._id('pagesRow');
+ var dateModifiedRow = this._id('dateModifiedRow');
+ var indexStatusRow = this._id('indexStatusRow');
+ var selectButton = this._id('select-button');
+
+ // DEBUG: this is annoying -- we really want to use an abstracted
+ // version of createValueElement() from itemPane.js
+ // (ideally in an XBL binding)
+
+ // Wrap title to multiple lines if necessary
+ while (title.hasChildNodes()) {
+ title.removeChild(title.firstChild);
+ }
+ var val = this.item.getField('title');
+
+ if (typeof val != 'string') {
+ val += "";
+ }
+
+ var firstSpace = val.indexOf(" ");
+ // Crop long uninterrupted text
+ if ((firstSpace == -1 && val.length > 29 ) || firstSpace > 29) {
+ title.setAttribute('crop', 'end');
+ title.setAttribute('value', val);
+ }
+ // Create a <description> element, essentially
+ else {
+ title.removeAttribute('value');
+ title.appendChild(document.createTextNode(val));
+ }
+
+ if (this.editable) {
+ title.className = 'zotero-clicky';
- yield Zotero.Promise.all([this.item.loadItemData(), this.item.loadNote()])
- .tap(() => Zotero.Promise.check(this.item));
+ // For the time being, use a silly little popup
+ title.addEventListener('click', this.editTitle, false);
+ }
+
+ var isImportedURL = this.item.attachmentLinkMode ==
+ Zotero.Attachments.LINK_MODE_IMPORTED_URL;
+
+ // Metadata for URL's
+ if (this.item.attachmentLinkMode == Zotero.Attachments.LINK_MODE_LINKED_URL
+ || isImportedURL) {
- var attachmentBox = document.getAnonymousNodes(this)[0];
- var title = this._id('title');
- var fileNameRow = this._id('fileNameRow');
- var urlField = this._id('url');
- var accessed = this._id('accessedRow');
- var pagesRow = this._id('pagesRow');
- var dateModifiedRow = this._id('dateModifiedRow');
- var indexStatusRow = this._id('indexStatusRow');
- var selectButton = this._id('select-button');
-
- // DEBUG: this is annoying -- we really want to use an abstracted
- // version of createValueElement() from itemPane.js
- // (ideally in an XBL binding)
-
- // Wrap title to multiple lines if necessary
- while (title.hasChildNodes()) {
- title.removeChild(title.firstChild);
- }
- var val = this.item.getField('title');
-
- if (typeof val != 'string') {
- val += "";
- }
-
- var firstSpace = val.indexOf(" ");
- // Crop long uninterrupted text
- if ((firstSpace == -1 && val.length > 29 ) || firstSpace > 29) {
- title.setAttribute('crop', 'end');
- title.setAttribute('value', val);
- }
- // Create a <description> element, essentially
- else {
- title.removeAttribute('value');
- title.appendChild(document.createTextNode(val));
- }
-
- if (this.editable) {
- title.className = 'zotero-clicky';
-
- // For the time being, use a silly little popup
- title.addEventListener('click', this.editTitle, false);
- }
-
- var isImportedURL = this.item.attachmentLinkMode ==
- Zotero.Attachments.LINK_MODE_IMPORTED_URL;
-
- // Metadata for URL's
- if (this.item.attachmentLinkMode == Zotero.Attachments.LINK_MODE_LINKED_URL
- || isImportedURL) {
-
- // URL
- if (this.displayURL) {
- var urlSpec = this.item.getField('url');
- urlField.setAttribute('value', urlSpec);
- urlField.setAttribute('hidden', false);
- if (this.clickableLink) {
- urlField.onclick = function (event) {
- ZoteroPane_Local.loadURI(this.value, event)
- };
- urlField.className = 'zotero-text-link';
- }
- else {
- urlField.className = '';
- }
- urlField.hidden = false;
+ // URL
+ if (this.displayURL) {
+ var urlSpec = this.item.getField('url');
+ urlField.setAttribute('value', urlSpec);
+ urlField.setAttribute('hidden', false);
+ if (this.clickableLink) {
+ urlField.onclick = function (event) {
+ ZoteroPane_Local.loadURI(this.value, event)
+ };
+ urlField.className = 'zotero-text-link';
}
else {
- urlField.hidden = true;
- }
-
- // Access date
- if (this.displayAccessed) {
- this._id("accessed-label").value = Zotero.getString('itemFields.accessDate')
- + Zotero.getString('punctuation.colon');
- this._id("accessed").value = Zotero.Date.sqlToDate(
- this.item.getField('accessDate'), true
- ).toLocaleString();
- accessed.hidden = false;
- }
- else {
- accessed.hidden = true;
+ urlField.className = '';
}
+ urlField.hidden = false;
}
- // Metadata for files
else {
urlField.hidden = true;
- accessed.hidden = true;
}
- if (this.item.attachmentLinkMode
- != Zotero.Attachments.LINK_MODE_LINKED_URL
- && this.displayFileName) {
- var fileName = this.item.getFilename();
-
- if (fileName) {
- this._id("fileName-label").value = Zotero.getString('pane.item.attachments.filename')
- + Zotero.getString('punctuation.colon');
- this._id("fileName").value = fileName;
- fileNameRow.hidden = false;
- }
- else {
- fileNameRow.hidden = true;
- }
+ // Access date
+ if (this.displayAccessed) {
+ this._id("accessed-label").value = Zotero.getString('itemFields.accessDate')
+ + Zotero.getString('punctuation.colon');
+ this._id("accessed").value = Zotero.Date.sqlToDate(
+ this.item.getField('accessDate'), true
+ ).toLocaleString();
+ accessed.hidden = false;
+ }
+ else {
+ accessed.hidden = true;
+ }
+ }
+ // Metadata for files
+ else {
+ urlField.hidden = true;
+ accessed.hidden = true;
+ }
+
+ if (this.item.attachmentLinkMode
+ != Zotero.Attachments.LINK_MODE_LINKED_URL
+ && this.displayFileName) {
+ var fileName = this.item.attachmentFilename;
+
+ if (fileName) {
+ this._id("fileName-label").value = Zotero.getString('pane.item.attachments.filename')
+ + Zotero.getString('punctuation.colon');
+ this._id("fileName").value = fileName;
+ fileNameRow.hidden = false;
}
else {
fileNameRow.hidden = true;
}
-
- // Page count
- if (this.displayPages) {
- var pages = yield Zotero.Fulltext.getPages(this.item.id)
- .tap(() => Zotero.Promise.check(this.item));
- var pages = pages ? pages.total : null;
+ }
+ else {
+ fileNameRow.hidden = true;
+ }
+
+ // Page count
+ if (this.displayPages) {
+ Zotero.Fulltext.getPages(this.item.id)
+ .tap(() => Zotero.Promise.check(this.item))
+ .then(function (pages) {
+ pages = pages ? pages.total : null;
if (pages) {
this._id("pages-label").value = Zotero.getString('itemFields.pages')
+ Zotero.getString('punctuation.colon');
@@ -295,77 +300,85 @@
else {
pagesRow.hidden = true;
}
- }
- else {
- pagesRow.hidden = true;
- }
-
- if (this.displayDateModified) {
- this._id("dateModified-label").value = Zotero.getString('itemFields.dateModified')
- + Zotero.getString('punctuation.colon');
- var mtime = yield this.item.attachmentModificationTime
- .tap(() => Zotero.Promise.check(this.item));
- if (mtime) {
- this._id("dateModified").value = new Date(mtime).toLocaleString();
- }
- // Use the item's mod time as a backup (e.g., when sync
- // passes in the mod time for the nonexistent remote file)
- else {
- this._id("dateModified").value = Zotero.Date.sqlToDate(
- this.item.getField('dateModified'), true
- ).toLocaleString();
- }
+ });
+ }
+ else {
+ pagesRow.hidden = true;
+ }
+
+ if (this.displayDateModified) {
+ this._id("dateModified-label").value = Zotero.getString('itemFields.dateModified')
+ + Zotero.getString('punctuation.colon');
+ // Conflict resolution uses a modal window, so promises won't work, but
+ // the sync process passes in the file mod time as dateModified
+ if (this.synchronous) {
+ this._id("dateModified").value = Zotero.Date.sqlToDate(
+ this.item.getField('dateModified'), true
+ ).toLocaleString();
dateModifiedRow.hidden = false;
}
else {
- dateModifiedRow.hidden = true;
+ this.item.attachmentModificationTime
+ .tap(() => Zotero.Promise.check(this._id))
+ .then(function (mtime) {
+ if (!this._id) return;
+ if (mtime) {
+ this._id("dateModified").value = new Date(mtime).toLocaleString();
+ }
+ dateModifiedRow.hidden = false;
+ });
}
-
- // Full-text index information
- if (this.displayIndexed) {
- yield this.updateItemIndexedState()
- .tap(() => Zotero.Promise.check(this.item));
+ }
+ else {
+ dateModifiedRow.hidden = true;
+ }
+
+ // Full-text index information
+ if (this.displayIndexed) {
+ this.updateItemIndexedState()
+ .tap(() => Zotero.Promise.check(this.item))
+ .then(function () {
indexStatusRow.hidden = false;
- }
- else {
- indexStatusRow.hidden = true;
- }
-
- // Note editor
- var noteEditor = this._id('attachment-note-editor');
- if (this.displayNote) {
- if (this.displayNoteIfEmpty || this.item.getNote() != '') {
- Zotero.debug("setting links on top");
- noteEditor.linksOnTop = true;
- noteEditor.hidden = false;
-
- // Don't make note editable (at least for now)
- if (this.mode == 'merge' || this.mode == 'mergeedit') {
- noteEditor.mode = 'merge';
- noteEditor.displayButton = false;
- }
- else {
- noteEditor.mode = this.mode;
- }
- noteEditor.parent = null;
- noteEditor.item = this.item;
- }
- }
- else {
- noteEditor.hidden = true;
- }
+ });
+ }
+ else {
+ indexStatusRow.hidden = true;
+ }
+
+ // Note editor
+ var noteEditor = this._id('attachment-note-editor');
+ if (this.displayNote) {
+ if (this.displayNoteIfEmpty || this.item.getNote() != '') {
+ Zotero.debug("setting links on top");
+ noteEditor.linksOnTop = true;
+ noteEditor.hidden = false;
+ // Don't make note editable (at least for now)
+ if (this.mode == 'merge' || this.mode == 'mergeedit') {
+ noteEditor.mode = 'merge';
+ noteEditor.displayButton = false;
+ }
+ else {
+ noteEditor.mode = this.mode;
+ }
+ noteEditor.parent = null;
+ noteEditor.item = this.item;
+ }
+ }
+ else {
+ noteEditor.hidden = true;
+ }
- if (this.displayButton) {
- selectButton.label = this.buttonCaption;
- selectButton.hidden = false;
- selectButton.setAttribute('oncommand',
- 'document.getBindingParent(this).clickHandler(this)');
- }
- else {
- selectButton.hidden = true;
- }
- }, this);
+
+ if (this.displayButton) {
+ selectButton.label = this.buttonCaption;
+ selectButton.hidden = false;
+ selectButton.setAttribute('oncommand',
+ 'document.getBindingParent(this).clickHandler(this)');
+ }
+ else {
+ selectButton.hidden = true;
+ }
]]></body>
</method>
diff --git a/chrome/content/zotero/bindings/itembox.xml b/chrome/content/zotero/bindings/itembox.xml
index c1ec439b3..622fd0b66 100644
--- a/chrome/content/zotero/bindings/itembox.xml
+++ b/chrome/content/zotero/bindings/itembox.xml
@@ -71,6 +71,7 @@
switch (val) {
case 'view':
+ case 'merge':
break;
case 'edit':
@@ -99,10 +100,9 @@
<field name="_item"/>
<property name="item" onget="return this._item;">
- <setter>
- <![CDATA[
+ <setter><![CDATA[
if (!(val instanceof Zotero.Item)) {
- throw ("<zoteroitembox>.item must be a Zotero.Item");
+ throw new Error("'item' must be a Zotero.Item");
}
// When changing items, reset truncation of creator list
@@ -112,8 +112,7 @@
this._item = val;
this.refresh();
- ]]>
- </setter>
+ ]]></setter>
</property>
<!-- .ref is an alias for .item -->
diff --git a/chrome/content/zotero/bindings/merge.xml b/chrome/content/zotero/bindings/merge.xml
index 045fcca39..3cf96f722 100644
--- a/chrome/content/zotero/bindings/merge.xml
+++ b/chrome/content/zotero/bindings/merge.xml
@@ -99,9 +99,11 @@
}
// Check for note or attachment
- this.type = this._getTypeFromObject(
- this._data.left.deleted ? this._data.right : this._data.left
- );
+ if (!this.type) {
+ this.type = this._getTypeFromObject(
+ this._data.left.deleted ? this._data.right : this._data.left
+ );
+ }
var showButton = this.type != 'item';
@@ -109,7 +111,6 @@
this._rightpane.showButton = showButton;
this._leftpane.data = this._data.left;
this._rightpane.data = this._data.right;
- this._mergepane.type = this.type;
this._mergepane.data = this._data.merge;
if (this._data.selected == 'left') {
@@ -313,6 +314,7 @@
break;
case 'attachment':
+ case 'file':
elementName = 'zoteroattachmentbox';
break;
@@ -320,13 +322,8 @@
elementName = 'zoteronoteeditor';
break;
- case 'file':
- elementName = 'zoterostoragefilebox';
- break;
-
default:
- throw ("Object type '" + this.type
- + "' not supported in <zoteromergepane>.ref");
+ throw new Error("Object type '" + this.type + "' not supported");
}
var objbox = document.createElement(elementName);
@@ -342,8 +339,7 @@
objbox.setAttribute("anonid", "objectbox");
objbox.setAttribute("flex", "1");
-
- objbox.mode = 'view';
+ objbox.mode = this.type == 'file' ? 'filemerge' : 'merge';
var button = this._id('choose-button');
if (this.showButton) {
@@ -363,7 +359,7 @@
// Create item from JSON for metadata box
var item = new Zotero.Item(val.itemType);
item.fromJSON(val);
- objbox.ref = item;
+ objbox.item = item;
]]>
</setter>
</property>
diff --git a/chrome/content/zotero/bindings/noteeditor.xml b/chrome/content/zotero/bindings/noteeditor.xml
index a0a13a43b..d9b17c03e 100644
--- a/chrome/content/zotero/bindings/noteeditor.xml
+++ b/chrome/content/zotero/bindings/noteeditor.xml
@@ -64,6 +64,7 @@
switch (val) {
case 'view':
+ case 'merge':
break;
case 'edit':
diff --git a/chrome/content/zotero/bindings/relatedbox.xml b/chrome/content/zotero/bindings/relatedbox.xml
index 8ecbfba43..aa8cc30ef 100644
--- a/chrome/content/zotero/bindings/relatedbox.xml
+++ b/chrome/content/zotero/bindings/relatedbox.xml
@@ -109,6 +109,7 @@
]]>
</destructor>
+ <!-- TODO: Asyncify -->
<method name="notify">
<parameter name="event"/>
<parameter name="type"/>
diff --git a/chrome/content/zotero/merge.js b/chrome/content/zotero/merge.js
index f35acc5ce..ee527c4d4 100644
--- a/chrome/content/zotero/merge.js
+++ b/chrome/content/zotero/merge.js
@@ -47,13 +47,20 @@ var Zotero_Merge_Window = new function () {
_wizard.getButton('cancel').setAttribute('label', Zotero.getString('sync.cancel'));
- _io = window.arguments[0].wrappedJSObject;
+ _io = window.arguments[0];
+ // Not totally clear when this is necessary
+ if (window.arguments[0].wrappedJSObject) {
+ _io = window.arguments[0].wrappedJSObject;
+ }
_conflicts = _io.dataIn.conflicts;
if (!_conflicts.length) {
// TODO: handle no conflicts
return;
}
+ if (_io.dataIn.type) {
+ _mergeGroup.type = _io.dataIn.type;
+ }
_mergeGroup.leftCaption = _io.dataIn.captions[0];
_mergeGroup.rightCaption = _io.dataIn.captions[1];
_mergeGroup.mergeCaption = _io.dataIn.captions[2];
@@ -240,7 +247,7 @@ var Zotero_Merge_Window = new function () {
}
// Apply changes from each side and pick most recent version for conflicting fields
var mergeInfo = {
- data: {}
+ data: {}
};
Object.assign(mergeInfo.data, _conflicts[pos].left)
Zotero.DataObjectUtilities.applyChanges(mergeInfo.data, _conflicts[pos].changes);
@@ -251,7 +258,9 @@ var Zotero_Merge_Window = new function () {
else {
var side = 1;
}
- Zotero.DataObjectUtilities.applyChanges(mergeInfo.data, _conflicts[pos].conflicts.map(x => x[side]));
+ Zotero.DataObjectUtilities.applyChanges(
+ mergeInfo.data, _conflicts[pos].conflicts.map(x => x[side])
+ );
mergeInfo.selected = side ? 'right' : 'left';
return mergeInfo;
}
@@ -284,13 +293,22 @@ var Zotero_Merge_Window = new function () {
function _updateResolveAllCheckbox() {
- if (_mergeGroup.rightpane.getAttribute("selected") == 'true') {
- var label = 'resolveAllRemoteFields';
+ if (_mergeGroup.type == 'file') {
+ if (_mergeGroup.rightpane.getAttribute("selected") == 'true') {
+ var label = 'resolveAllRemote';
+ }
+ else {
+ var label = 'resolveAllLocal';
+ }
}
else {
- var label = 'resolveAllLocalFields';
+ if (_mergeGroup.rightpane.getAttribute("selected") == 'true') {
+ var label = 'resolveAllRemoteFields';
+ }
+ else {
+ var label = 'resolveAllLocalFields';
+ }
}
- // TODO: files
_resolveAllCheckbox.label = Zotero.getString('sync.conflict.' + label);
}
diff --git a/chrome/content/zotero/xpcom/data/item.js b/chrome/content/zotero/xpcom/data/item.js
index c8d4ee8cf..b5565ac35 100644
--- a/chrome/content/zotero/xpcom/data/item.js
+++ b/chrome/content/zotero/xpcom/data/item.js
@@ -50,7 +50,6 @@ Zotero.Item = function(itemTypeOrID) {
this._attachmentLinkMode = null;
this._attachmentContentType = null;
this._attachmentPath = null;
- this._attachmentSyncState = null;
// loadCreators
this._creators = [];
@@ -1453,14 +1452,13 @@ Zotero.Item.prototype._saveData = Zotero.Promise.coroutine(function* (env) {
if (this._changed.attachmentData) {
let sql = "REPLACE INTO itemAttachments (itemID, parentItemID, linkMode, "
- + "contentType, charsetID, path, syncState) VALUES (?,?,?,?,?,?,?)";
+ + "contentType, charsetID, path) VALUES (?,?,?,?,?,?)";
let linkMode = this.attachmentLinkMode;
let contentType = this.attachmentContentType;
let charsetID = this.attachmentCharset
? Zotero.CharacterSets.getID(this.attachmentCharset)
: null;
let path = this.attachmentPath;
- let syncState = this.attachmentSyncState;
if (linkMode == Zotero.Attachments.LINK_MODE_LINKED_FILE && libraryType != 'user') {
throw new Error("Linked files can only be added to user library");
@@ -1472,8 +1470,7 @@ Zotero.Item.prototype._saveData = Zotero.Promise.coroutine(function* (env) {
{ int: linkMode },
contentType ? { string: contentType } : null,
charsetID ? { int: charsetID } : null,
- path ? { string: path } : null,
- syncState ? { int: syncState } : 0
+ path ? { string: path } : null
];
yield Zotero.DB.queryAsync(sql, params);
@@ -2295,8 +2292,10 @@ Zotero.Item.prototype.renameAttachmentFile = Zotero.Promise.coroutine(function*
yield this.relinkAttachmentFile(destPath);
yield Zotero.DB.executeTransaction(function* () {
- yield Zotero.Sync.Storage.setSyncedHash(this.id, null, false);
- yield Zotero.Sync.Storage.setSyncState(this.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD);
+ yield Zotero.Sync.Storage.Local.setSyncedHash(this.id, null, false);
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ this.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
+ );
}.bind(this));
return true;
@@ -2317,11 +2316,10 @@ Zotero.Item.prototype.renameAttachmentFile = Zotero.Promise.coroutine(function*
/**
* @param {string} path File path
- * @param {Boolean} [skipItemUpdate] Don't update attachment item mod time,
- * so that item doesn't sync. Used when a file
- * needs to be renamed to be accessible but the
- * user doesn't have access to modify the
- * attachment metadata
+ * @param {Boolean} [skipItemUpdate] Don't update attachment item mod time, so that item doesn't
+ * sync. Used when a file needs to be renamed to be accessible but the user doesn't have
+ * access to modify the attachment metadata. This also allows a save when the library is
+ * read-only.
*/
Zotero.Item.prototype.relinkAttachmentFile = Zotero.Promise.coroutine(function* (path, skipItemUpdate) {
if (path instanceof Components.interfaces.nsIFile) {
@@ -2382,7 +2380,8 @@ Zotero.Item.prototype.relinkAttachmentFile = Zotero.Promise.coroutine(function*
yield this.saveTx({
skipDateModifiedUpdate: true,
- skipClientDateModifiedUpdate: skipItemUpdate
+ skipClientDateModifiedUpdate: skipItemUpdate,
+ skipEditCheck: skipItemUpdate
});
return true;
@@ -3606,9 +3605,6 @@ Zotero.Item.prototype.clone = Zotero.Promise.coroutine(function* (libraryID, ski
if (this.attachmentPath) {
newItem.attachmentPath = this.attachmentPath;
}
- if (this.attachmentSyncState) {
- newItem.attachmentSyncState = this.attachmentSyncState;
- }
}
}
}
diff --git a/chrome/content/zotero/xpcom/data/items.js b/chrome/content/zotero/xpcom/data/items.js
index dacd06967..8733dd03c 100644
--- a/chrome/content/zotero/xpcom/data/items.js
+++ b/chrome/content/zotero/xpcom/data/items.js
@@ -84,8 +84,7 @@ Zotero.Items = function() {
attachmentCharset: "CS.charset AS attachmentCharset",
attachmentLinkMode: "IA.linkMode AS attachmentLinkMode",
attachmentContentType: "IA.contentType AS attachmentContentType",
- attachmentPath: "IA.path AS attachmentPath",
- attachmentSyncState: "IA.syncState AS attachmentSyncState"
+ attachmentPath: "IA.path AS attachmentPath"
};
}
}, {lazy: true});
diff --git a/chrome/content/zotero/xpcom/file.js b/chrome/content/zotero/xpcom/file.js
index b289dab54..82976e9da 100644
--- a/chrome/content/zotero/xpcom/file.js
+++ b/chrome/content/zotero/xpcom/file.js
@@ -48,7 +48,7 @@ Zotero.File = new function(){
else if (pathOrFile instanceof Ci.nsIFile) {
return pathOrFile;
}
- throw new Error('Unexpected value provided to Zotero.File.pathToFile() (' + pathOrFile + ')');
+ throw new Error("Unexpected value '" + pathOrFile + "'");
}
@@ -348,7 +348,7 @@ Zotero.File = new function(){
* @param {String} [charset] - The character set; defaults to UTF-8
* @return {Promise} - A promise that is resolved when the file has been written
*/
- this.putContentsAsync = function putContentsAsync(path, data, charset) {
+ this.putContentsAsync = function (path, data, charset) {
if (path instanceof Ci.nsIFile) {
path = path.path;
}
@@ -424,18 +424,17 @@ Zotero.File = new function(){
* iterator when done
*
* The DirectoryInterator is passed as the first parameter to the generator.
- * A StopIteration error will be caught automatically.
*
* Zotero.File.iterateDirectory(path, function* (iterator) {
* while (true) {
* var entry = yield iterator.next();
* [...]
* }
- * }).done()
+ * })
*
* @return {Promise}
*/
- this.iterateDirectory = function iterateDirectory(path, generator) {
+ this.iterateDirectory = function (path, generator) {
var iterator = new OS.File.DirectoryIterator(path);
return Zotero.Promise.coroutine(generator)(iterator)
.catch(function (e) {
@@ -470,6 +469,8 @@ Zotero.File = new function(){
this.createShortened = function (file, type, mode, maxBytes) {
+ file = this.pathToFile(file);
+
if (!maxBytes) {
maxBytes = 255;
}
@@ -575,6 +576,8 @@ Zotero.File = new function(){
}
break;
}
+
+ return file.leafName;
}
@@ -902,29 +905,28 @@ Zotero.File = new function(){
this.checkFileAccessError = function (e, file, operation) {
+ var str = 'file.accessError.';
if (file) {
- var str = Zotero.getString('file.accessError.theFile', file.path);
+ str += 'theFile'
}
else {
- var str = Zotero.getString('file.accessError.aFile');
+ str += 'aFile'
}
+ str += 'CannotBe';
switch (operation) {
case 'create':
- var opWord = Zotero.getString('file.accessError.created');
- break;
-
- case 'update':
- var opWord = Zotero.getString('file.accessError.updated');
+ str += 'Created';
break;
case 'delete':
- var opWord = Zotero.getString('file.accessError.deleted');
+ str += 'Deleted';
break;
default:
- var opWord = Zotero.getString('file.accessError.updated');
+ str += 'Updated';
}
+ str = Zotero.getString(str, file.path ? file.path : undefined);
Zotero.debug(file.path);
Zotero.debug(e, 1);
@@ -962,4 +964,64 @@ Zotero.File = new function(){
throw (e);
}
+
+
+ this.checkPathAccessError = function (e, path, operation) {
+ var str = 'file.accessError.';
+ if (path) {
+ str += 'theFile'
+ }
+ else {
+ str += 'aFile'
+ }
+ str += 'CannotBe';
+
+ switch (operation) {
+ case 'create':
+ str += 'Created';
+ break;
+
+ case 'delete':
+ str += 'Deleted';
+ break;
+
+ default:
+ str += 'Updated';
+ }
+ str = Zotero.getString(str, path ? path : undefined);
+
+ Zotero.debug(path);
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e);
+
+ // TODO: Check for specific errors?
+ if (e instanceof OS.File.Error) {
+ let checkFileWindows = Zotero.getString('file.accessError.message.windows');
+ let checkFileOther = Zotero.getString('file.accessError.message.other');
+ var msg = str + "\n\n"
+ + (Zotero.isWin ? checkFileWindows : checkFileOther)
+ + "\n\n"
+ + Zotero.getString('file.accessError.restart');
+
+ var e = new Zotero.Error(
+ msg,
+ 0,
+ {
+ dialogButtonText: Zotero.getString('file.accessError.showParentDir'),
+ dialogButtonCallback: function () {
+ try {
+ file.parent.QueryInterface(Components.interfaces.nsILocalFile);
+ file.parent.reveal();
+ }
+ // Unsupported on some platforms
+ catch (e2) {
+ Zotero.launchFile(file.parent);
+ }
+ }
+ }
+ );
+ }
+
+ throw e;
+ }
}
diff --git a/chrome/content/zotero/xpcom/storage.js b/chrome/content/zotero/xpcom/storage.js
index f382bcfb1..719bb47ed 100644
--- a/chrome/content/zotero/xpcom/storage.js
+++ b/chrome/content/zotero/xpcom/storage.js
@@ -33,6 +33,7 @@ Zotero.Sync.Storage = new function () {
this.SYNC_STATE_IN_SYNC = 2;
this.SYNC_STATE_FORCE_UPLOAD = 3;
this.SYNC_STATE_FORCE_DOWNLOAD = 4;
+ this.SYNC_STATE_IN_CONFLICT = 5;
this.SUCCESS = 1;
this.ERROR_NO_URL = -1;
@@ -57,14 +58,11 @@ Zotero.Sync.Storage = new function () {
this.__defineGetter__("defaultError", function () Zotero.getString('sync.storage.error.default', Zotero.appName));
this.__defineGetter__("defaultErrorRestart", function () Zotero.getString('sync.storage.error.defaultRestart', Zotero.appName));
+ var _itemDownloadPercentages = {};
+
//
// Public properties
//
-
-
- this.__defineGetter__("syncInProgress", function () _syncInProgress);
- this.__defineGetter__("updatesInProgress", function () _updatesInProgress);
-
this.compressionTracker = {
compressed: 0,
uncompressed: 0,
@@ -75,539 +73,6 @@ Zotero.Sync.Storage = new function () {
}
}
- Zotero.Notifier.registerObserver(this, ['file']);
-
-
- //
- // Private properties
- //
- var _maxCheckAgeInSeconds = 10800; // maximum age for upload modification check (3 hours)
- var _syncInProgress;
- var _updatesInProgress;
- var _itemDownloadPercentages = {};
- var _uploadCheckFiles = [];
- var _lastFullFileCheck = {};
-
-
- this.sync = function (options) {
- if (options.libraries) {
- Zotero.debug("Starting file sync for libraries " + options.libraries);
- }
- else {
- Zotero.debug("Starting file sync");
- }
-
- var self = this;
-
- var libraryModes = {};
- var librarySyncTimes = {};
-
- // Get personal library file sync mode
- return Zotero.Promise.try(function () {
- // TODO: Make sure modes are active
-
- if (options.libraries && options.libraries.indexOf(0) == -1) {
- return;
- }
-
- if (Zotero.Sync.Storage.ZFS.includeUserFiles) {
- libraryModes[0] = Zotero.Sync.Storage.ZFS;
- }
- else if (Zotero.Sync.Storage.WebDAV.includeUserFiles) {
- libraryModes[0] = Zotero.Sync.Storage.WebDAV;
- }
- })
- .then(function () {
- // Get group library file sync modes
- if (Zotero.Sync.Storage.ZFS.includeGroupFiles) {
- var groups = Zotero.Groups.getAll();
- for each(var group in groups) {
- if (options.libraries && options.libraries.indexOf(group.libraryID) == -1) {
- continue;
- }
- // TODO: if library file syncing enabled
- libraryModes[group.libraryID] = Zotero.Sync.Storage.ZFS;
- }
- }
-
- // Cache auth credentials for each mode
- var modes = [];
- var promises = [];
- for each(var mode in libraryModes) {
- if (modes.indexOf(mode) == -1) {
- modes.push(mode);
-
- // Try to verify WebDAV server first if it hasn't been
- if (mode == Zotero.Sync.Storage.WebDAV
- && !Zotero.Sync.Storage.WebDAV.verified) {
- Zotero.debug("WebDAV file sync is not active");
- var promise = Zotero.Sync.Storage.checkServerPromise(Zotero.Sync.Storage.WebDAV)
- .then(function () {
- return mode.cacheCredentials();
- });
- }
- else {
- var promise = mode.cacheCredentials();
- }
- promises.push(Zotero.Promise.allSettled([mode, promise]));
- }
- }
-
- return Zotero.Promise.all(promises)
- // Get library last-sync times
- .then(function (cacheCredentialsPromises) {
- var promises = [];
-
- // Mark WebDAV verification failure as user library error.
- // We ignore credentials-caching errors for ZFS and let the
- // later requests fail.
- cacheCredentialsPromises.forEach(function (results) {
- let mode = results[0].value;
- if (mode == Zotero.Sync.Storage.WebDAV) {
- if (results[1].state == "rejected") {
- promises.push(Zotero.Promise.allSettled(
- [0, Zotero.Promise.reject(results[1].reason)]
- ));
- // Skip further syncing of user library
- delete libraryModes[0];
- }
- }
- });
-
- for (var libraryID in libraryModes) {
- libraryID = parseInt(libraryID);
-
- // Get the last sync time for each library
- if (self.downloadOnSync(libraryID)) {
- promises.push(Zotero.Promise.allSettled(
- [libraryID, libraryModes[libraryID].getLastSyncTime(libraryID)]
- ));
- }
- // If download-as-needed, we don't need the last sync time
- else {
- promises.push(Zotero.Promise.allSettled([libraryID, null]));
- }
- }
- return Zotero.Promise.all(promises);
- });
- })
- .then(function (promises) {
- if (!promises.length) {
- Zotero.debug("No libraries are active for file sync");
- return [];
- }
-
- var libraryQueues = [];
-
- // Get the libraries we have sync times for
- promises.forEach(function (results) {
- let libraryID = results[0].value;
- let lastSyncTime = results[1].value;
- if (results[1].state == "fulfilled") {
- librarySyncTimes[libraryID] = lastSyncTime;
- }
- else {
- Zotero.debug(lastSyncTime.reason);
- Components.utils.reportError(lastSyncTime.reason);
- // Pass rejected promise through
- libraryQueues.push(results);
- }
- });
-
- // Check for updated files to upload in each library
- var promises = [];
- for (let libraryID in librarySyncTimes) {
- let promise;
- libraryID = parseInt(libraryID);
-
- if (!Zotero.Libraries.isFilesEditable(libraryID)) {
- Zotero.debug("No file editing access -- skipping file "
- + "modification check for library " + libraryID);
- continue;
- }
- // If this is a background sync, it's not the first sync of
- // the session, the library has had at least one full check
- // this session, and it's been less than _maxCheckAgeInSeconds
- // since the last full check of this library, check only files
- // that were previously modified or opened recently
- else if (options.background
- && !options.firstInSession
- && _lastFullFileCheck[libraryID]
- && (_lastFullFileCheck[libraryID] + (_maxCheckAgeInSeconds * 1000))
- > new Date().getTime()) {
- let itemIDs = _getFilesToCheck(libraryID);
- promise = self.checkForUpdatedFiles(libraryID, itemIDs);
- }
- // Otherwise check all files in the library
- else {
- _lastFullFileCheck[libraryID] = new Date().getTime();
- promise = self.checkForUpdatedFiles(libraryID);
- }
- promises.push(promise);
- }
- return Zotero.Promise.all(promises)
- .then(function () {
- // Queue files to download and upload from each library
- for (let libraryID in librarySyncTimes) {
- libraryID = parseInt(libraryID);
-
- var downloadAll = self.downloadOnSync(libraryID);
-
- // Forced downloads happen even in on-demand mode
- var sql = "SELECT COUNT(*) FROM items "
- + "JOIN itemAttachments USING (itemID) "
- + "WHERE libraryID=? AND syncState=?";
- var downloadForced = !!Zotero.DB.valueQuery(
- sql,
- [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD]
- );
-
- // If we don't have any forced downloads, we can skip
- // downloads if the last sync time hasn't changed
- // or doesn't exist on the server (meaning there are no files)
- if (downloadAll && !downloadForced) {
- let lastSyncTime = librarySyncTimes[libraryID];
- if (lastSyncTime) {
- var version = self.getStoredLastSyncTime(
- libraryModes[libraryID], libraryID
- );
- if (version == lastSyncTime) {
- Zotero.debug("Last " + libraryModes[libraryID].name
- + " sync id hasn't changed for library "
- + libraryID + " -- skipping file downloads");
- downloadAll = false;
- }
- }
- else {
- Zotero.debug("No last " + libraryModes[libraryID].name
- + " sync time for library " + libraryID
- + " -- skipping file downloads");
- downloadAll = false;
- }
- }
-
- if (downloadAll || downloadForced) {
- for each(var itemID in _getFilesToDownload(libraryID, !downloadAll)) {
- var item = Zotero.Items.get(itemID);
- self.queueItem(item);
- }
- }
-
- // Get files to upload
- if (Zotero.Libraries.isFilesEditable(libraryID)) {
- for each(var itemID in _getFilesToUpload(libraryID)) {
- var item = Zotero.Items.get(itemID);
- self.queueItem(item);
- }
- }
- else {
- Zotero.debug("No file editing access -- skipping file uploads for library " + libraryID);
- }
- }
-
- // Start queues for each library
- for (let libraryID in librarySyncTimes) {
- libraryID = parseInt(libraryID);
- libraryQueues.push(Zotero.Promise.allSettled(
- [libraryID, Zotero.Sync.Storage.QueueManager.start(libraryID)]
- ));
- }
-
- // The promise is done when all libraries are done
- return Zotero.Promise.all(libraryQueues);
- });
- })
- .then(function (promises) {
- Zotero.debug('Queue manager is finished');
-
- var changedLibraries = [];
- var finalPromises = [];
-
- promises.forEach(function (results) {
- var libraryID = results[0].value;
- var libraryQueues = results[1].value;
-
- if (results[1].state == "fulfilled") {
- libraryQueues.forEach(function (queuePromise) {
- if (queueZotero.Promise.isFulfilled()) {
- let result = queueZotero.Promise.value();
- Zotero.debug("File " + result.type + " sync finished "
- + "for library " + libraryID);
- if (result.localChanges) {
- changedLibraries.push(libraryID);
- }
- finalPromises.push(Zotero.Promise.allSettled([
- libraryID,
- libraryModes[libraryID].setLastSyncTime(
- libraryID,
- result.remoteChanges ? false : librarySyncTimes[libraryID]
- )
- ]));
- }
- else {
- let e = queueZotero.Promise.reason();
- Zotero.debug("File " + e.type + " sync failed "
- + "for library " + libraryID);
- finalPromises.push(Zotero.Promise.allSettled(
- [libraryID, Zotero.Promise.reject(e)]
- ));
- }
- });
- }
- else {
- Zotero.debug("File sync failed for library " + libraryID);
- finalPromises.push([libraryID, libraryQueues]);
- }
-
- // If WebDAV sync enabled, purge deleted and orphaned files
- if (libraryID == Zotero.Libraries.userLibraryID
- && Zotero.Sync.Storage.WebDAV.includeUserFiles) {
- Zotero.Sync.Storage.WebDAV.purgeDeletedStorageFiles()
- .then(function () {
- return Zotero.Sync.Storage.WebDAV.purgeOrphanedStorageFiles();
- })
- .catch(function (e) {
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
- });
- }
- });
-
- Zotero.Sync.Storage.ZFS.purgeDeletedStorageFiles()
- .catch(function (e) {
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
- });
-
- if (promises.length && !changedLibraries.length) {
- Zotero.debug("No local changes made during file sync");
- }
-
- return Zotero.Promise.all(finalPromises)
- .then(function (promises) {
- var results = {
- changesMade: !!changedLibraries.length,
- errors: []
- };
-
- promises.forEach(function (promiseResults) {
- var libraryID = promiseResults[0].value;
- if (promiseResults[1].state == "rejected") {
- let e = promiseResults[1].reason;
- if (typeof e == 'string') {
- e = new Error(e);
- }
- e.libraryID = libraryID;
- results.errors.push(e);
- }
- });
-
- return results;
- });
- });
- }
-
-
- //
- // Public methods
- //
- this.queueItem = function (item, highPriority) {
- var library = item.libraryID;
- if (libraryID) {
- var mode = Zotero.Sync.Storage.ZFS;
- }
- else {
- var mode = Zotero.Sync.Storage.ZFS.includeUserFiles
- ? Zotero.Sync.Storage.ZFS : Zotero.Sync.Storage.WebDAV;
- }
- switch (Zotero.Sync.Storage.getSyncState(item.id)) {
- case this.SYNC_STATE_TO_DOWNLOAD:
- case this.SYNC_STATE_FORCE_DOWNLOAD:
- var queue = 'download';
- var callbacks = {
- onStart: function (request) {
- return mode.downloadFile(request);
- }
- };
- break;
-
- case this.SYNC_STATE_TO_UPLOAD:
- case this.SYNC_STATE_FORCE_UPLOAD:
- var queue = 'upload';
- var callbacks = {
- onStart: function (request) {
- return mode.uploadFile(request);
- }
- };
- break;
-
- case false:
- Zotero.debug("Sync state for item " + item.id + " not found", 2);
- return;
- }
-
- var queue = Zotero.Sync.Storage.QueueManager.get(queue, library);
- var request = new Zotero.Sync.Storage.Request(
- item.libraryID + '/' + item.key, callbacks
- );
- if (queue.type == 'upload') {
- try {
- request.setMaxSize(Zotero.Attachments.getTotalFileSize(item));
- }
- // If this fails, ignore it, though we might fail later
- catch (e) {
- // But if the file doesn't exist yet, don't try to upload it
- //
- // This isn't a perfect test, because the file could still be
- // in the process of being downloaded. It'd be better to
- // download files to a temp directory and move them into place.
- if (!item.getFile()) {
- Zotero.debug("File " + item.libraryKey + " not yet available to upload -- skipping");
- return;
- }
-
- Components.utils.reportError(e);
- Zotero.debug(e, 1);
- }
- }
- queue.addRequest(request, highPriority);
- };
-
-
- this.getStoredLastSyncTime = function (mode, libraryID) {
- var sql = "SELECT version FROM version WHERE schema=?";
- return Zotero.DB.valueQuery(
- sql, "storage_" + mode.name.toLowerCase() + "_" + libraryID
- );
- };
-
-
- this.setStoredLastSyncTime = function (mode, libraryID, time) {
- var sql = "REPLACE INTO version SET version=? WHERE schema=?";
- Zotero.DB.query(
- sql,
- [
- time,
- "storage_" + mode.name.toLowerCase() + "_" + libraryID
- ]
- );
- };
-
-
- /**
- * @param {Integer} itemID
- */
- this.getSyncState = function (itemID) {
- var sql = "SELECT syncState FROM itemAttachments WHERE itemID=?";
- return Zotero.DB.valueQueryAsync(sql, itemID);
- }
-
-
- /**
- * @param {Integer} itemID
- * @param {Integer} syncState Constant from Zotero.Sync.Storage
- */
- this.setSyncState = Zotero.Promise.method(function (itemID, syncState) {
- switch (syncState) {
- case this.SYNC_STATE_TO_UPLOAD:
- case this.SYNC_STATE_TO_DOWNLOAD:
- case this.SYNC_STATE_IN_SYNC:
- case this.SYNC_STATE_FORCE_UPLOAD:
- case this.SYNC_STATE_FORCE_DOWNLOAD:
- break;
-
- default:
- throw new Error("Invalid sync state '" + syncState);
- }
-
- var sql = "UPDATE itemAttachments SET syncState=? WHERE itemID=?";
- return Zotero.DB.valueQueryAsync(sql, [syncState, itemID]);
- });
-
-
- /**
- * @param {Integer} itemID
- * @return {Integer|NULL} Mod time as timestamp in ms,
- * or NULL if never synced
- */
- this.getSyncedModificationTime = function (itemID) {
- var sql = "SELECT storageModTime FROM itemAttachments WHERE itemID=?";
- var mtime = Zotero.DB.valueQuery(sql, itemID);
- if (mtime === false) {
- throw "Item " + itemID + " not found in "
- + "Zotero.Sync.Storage.getSyncedModificationTime()";
- }
- return mtime;
- }
-
-
- /**
- * @param {Integer} itemID
- * @param {Integer} mtime File modification time as
- * timestamp in ms
- * @param {Boolean} [updateItem=FALSE] Update dateModified field of
- * attachment item
- */
- this.setSyncedModificationTime = function (itemID, mtime, updateItem) {
- if (mtime < 0) {
- Components.utils.reportError("Invalid file mod time " + mtime
- + " in Zotero.Storage.setSyncedModificationTime()");
- mtime = 0;
- }
-
- Zotero.DB.beginTransaction();
-
- var sql = "UPDATE itemAttachments SET storageModTime=? WHERE itemID=?";
- Zotero.DB.valueQuery(sql, [mtime, itemID]);
-
- if (updateItem) {
- // Update item date modified so the new mod time will be synced
- var sql = "UPDATE items SET clientDateModified=? WHERE itemID=?";
- Zotero.DB.query(sql, [Zotero.DB.transactionDateTime, itemID]);
- }
-
- Zotero.DB.commitTransaction();
- }
-
-
- /**
- * @param {Integer} itemID
- * @return {Promise<String|null|false>} - File hash, null if never synced, if false if
- * file doesn't exist
- */
- this.getSyncedHash = Zotero.Promise.coroutine(function* (itemID) {
- var sql = "SELECT storageHash FROM itemAttachments WHERE itemID=?";
- var hash = yield Zotero.DB.valueQueryAsync(sql, itemID);
- if (hash === false) {
- throw new Error("Item " + itemID + " not found");
- }
- return hash;
- })
-
-
- /**
- * @param {Integer} itemID
- * @param {String} hash File hash
- * @param {Boolean} [updateItem=FALSE] Update dateModified field of
- * attachment item
- */
- this.setSyncedHash = Zotero.Promise.coroutine(function* (itemID, hash, updateItem) {
- if (hash !== null && hash.length != 32) {
- throw ("Invalid file hash '" + hash + "' in Zotero.Storage.setSyncedHash()");
- }
-
- Zotero.DB.requireTransaction();
-
- var sql = "UPDATE itemAttachments SET storageHash=? WHERE itemID=?";
- yield Zotero.DB.queryAsync(sql, [hash, itemID]);
-
- if (updateItem) {
- // Update item date modified so the new mod time will be synced
- var sql = "UPDATE items SET clientDateModified=? WHERE itemID=?";
- yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]);
- }
- });
-
/**
* Check if modification time of file on disk matches the mod time
@@ -646,518 +111,6 @@ Zotero.Sync.Storage = new function () {
}
- /**
- * @param {Integer|'groups'} [libraryID]
- */
- this.downloadAsNeeded = function (libraryID) {
- // Personal library
- if (!libraryID) {
- return Zotero.Prefs.get('sync.storage.downloadMode.personal') == 'on-demand';
- }
- // Group library (groupID or 'groups')
- else {
- return Zotero.Prefs.get('sync.storage.downloadMode.groups') == 'on-demand';
- }
- }
-
-
- /**
- * @param {Integer|'groups'} [libraryID]
- */
- this.downloadOnSync = function (libraryID) {
- // Personal library
- if (!libraryID) {
- return Zotero.Prefs.get('sync.storage.downloadMode.personal') == 'on-sync';
- }
- // Group library (groupID or 'groups')
- else {
- return Zotero.Prefs.get('sync.storage.downloadMode.groups') == 'on-sync';
- }
- }
-
-
-
- /**
- * Scans local files and marks any that have changed for uploading
- * and any that are missing for downloading
- *
- * @param {Integer} [libraryID]
- * @param {Integer[]} [itemIDs]
- * @param {Object} [itemModTimes] Item mod times indexed by item ids;
- * items with stored mod times
- * that differ from the provided
- * time but file mod times
- * matching the stored time will
- * be marked for download
- * @return {Promise} Promise resolving to TRUE if any items changed state,
- * FALSE otherwise
- */
- this.checkForUpdatedFiles = function (libraryID, itemIDs, itemModTimes) {
- return Zotero.Promise.try(function () {
- libraryID = parseInt(libraryID);
- if (isNaN(libraryID)) {
- libraryID = false;
- }
-
- var msg = "Checking for locally changed attachment files";
-
- var memmgr = Components.classes["@mozilla.org/memory-reporter-manager;1"]
- .getService(Components.interfaces.nsIMemoryReporterManager);
- memmgr.init();
- //Zotero.debug("Memory usage: " + memmgr.resident);
-
- if (libraryID !== false) {
- if (itemIDs) {
- if (!itemIDs.length) {
- var msg = "No files to check for local changes in library " + libraryID;
- Zotero.debug(msg);
- return false;
- }
- }
- if (itemModTimes) {
- throw new Error("itemModTimes is not allowed when libraryID is set");
- }
-
- msg += " in library " + libraryID;
- }
- else if (itemIDs) {
- throw new Error("libraryID not provided");
- }
- else if (itemModTimes) {
- if (!Object.keys(itemModTimes).length) {
- return false;
- }
- msg += " in download-marking mode";
- }
- else {
- throw new Error("libraryID, itemIDs, or itemModTimes must be provided");
- }
- Zotero.debug(msg);
-
- var changed = false;
-
- if (!itemIDs) {
- itemIDs = Object.keys(itemModTimes ? itemModTimes : {});
- }
-
- // Can only handle a certain number of bound parameters at a time
- var numIDs = itemIDs.length;
- var maxIDs = Zotero.DB.MAX_BOUND_PARAMETERS - 10;
- var done = 0;
- var rows = [];
-
- Zotero.DB.beginTransaction();
-
- do {
- var chunk = itemIDs.splice(0, maxIDs);
- var sql = "SELECT itemID, linkMode, path, storageModTime, storageHash, syncState "
- + "FROM itemAttachments JOIN items USING (itemID) "
- + "WHERE linkMode IN (?,?) AND syncState IN (?,?)";
- var params = [];
- params.push(
- Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
- Zotero.Attachments.LINK_MODE_IMPORTED_URL,
- Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD,
- Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
- );
- if (libraryID !== false) {
- sql += " AND libraryID=?";
- params.push(libraryID);
- }
- if (chunk.length) {
- sql += " AND itemID IN (" + chunk.map(function () '?').join() + ")";
- params = params.concat(chunk);
- }
- var chunkRows = Zotero.DB.query(sql, params);
- if (chunkRows) {
- rows = rows.concat(chunkRows);
- }
- done += chunk.length;
- }
- while (done < numIDs);
-
- Zotero.DB.commitTransaction();
-
- // If no files, or everything is already marked for download,
- // we don't need to do anything
- if (!rows.length) {
- var msg = "No in-sync or to-upload files found";
- if (libraryID !== false) {
- msg += " in library " + libraryID;
- }
- Zotero.debug(msg);
- return false;
- }
-
- // Index attachment data by item id
- itemIDs = [];
- var attachmentData = {};
- for each(let row in rows) {
- var id = row.itemID;
- itemIDs.push(id);
- attachmentData[id] = {
- linkMode: row.linkMode,
- path: row.path,
- mtime: row.storageModTime,
- hash: row.storageHash,
- state: row.syncState
- };
- }
- rows = null;
-
- var t = new Date();
- var items = Zotero.Items.get(itemIDs);
- var numItems = items.length;
- var updatedStates = {};
-
- let checkItems = function () {
- if (!items.length) return Zotero.Promise.resolve();
-
- //Zotero.debug("Memory usage: " + memmgr.resident);
-
- let item = items.shift();
- let row = attachmentData[item.id];
- let lk = item.libraryKey;
- Zotero.debug("Checking attachment file for item " + lk);
-
- let nsIFile = item.getFile(row, true);
- if (!nsIFile) {
- Zotero.debug("Marking pathless attachment " + lk + " as in-sync");
- updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_IN_SYNC;
- return checkItems();
- }
- let file = null;
- return Zotero.Promise.resolve(OS.File.open(nsIFile.path))
- .then(function (promisedFile) {
- file = promisedFile;
- return file.stat()
- .then(function (info) {
- //Zotero.debug("Memory usage: " + memmgr.resident);
-
- var fmtime = info.lastModificationDate.getTime();
- //Zotero.debug("File modification time for item " + lk + " is " + fmtime);
-
- if (fmtime < 1) {
- Zotero.debug("File mod time " + fmtime + " is less than 1 -- interpreting as 1", 2);
- fmtime = 1;
- }
-
- // If file is already marked for upload, skip check. Even if this
- // is download-marking mode (itemModTimes) and the file was
- // changed remotely, conflicts are checked at upload time, so we
- // don't need to worry about it here.
- if (row.state == Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD) {
- return;
- }
-
- //Zotero.debug("Stored mtime is " + row.mtime);
- //Zotero.debug("File mtime is " + fmtime);
-
- // Download-marking mode
- if (itemModTimes) {
- Zotero.debug("Remote mod time for item " + lk + " is " + itemModTimes[item.id]);
-
- // Ignore attachments whose stored mod times haven't changed
- if (row.storageModTime == itemModTimes[item.id]) {
- Zotero.debug("Storage mod time (" + row.storageModTime + ") "
- + "hasn't changed for item " + lk);
- return;
- }
-
- Zotero.debug("Marking attachment " + lk + " for download "
- + "(stored mtime: " + itemModTimes[item.id] + ")");
- updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD;
- }
-
- var mtime = row.mtime;
-
- // If stored time matches file, it hasn't changed locally
- if (mtime == fmtime) {
- return;
- }
-
- // Allow floored timestamps for filesystems that don't support
- // millisecond precision (e.g., HFS+)
- if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
- Zotero.debug("File mod times are within one-second precision "
- + "(" + fmtime + " ≅ " + mtime + ") for " + file.leafName
- + " for item " + lk + " -- ignoring");
- return;
- }
-
- // Allow timestamp to be exactly one hour off to get around
- // time zone issues -- there may be a proper way to fix this
- if (Math.abs(fmtime - mtime) == 3600000
- // And check with one-second precision as well
- || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
- || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
- Zotero.debug("File mod time (" + fmtime + ") is exactly one "
- + "hour off remote file (" + mtime + ") for item " + lk
- + "-- assuming time zone issue and skipping upload");
- return;
- }
-
- // If file hash matches stored hash, only the mod time changed, so skip
- return Zotero.Utilities.Internal.md5Async(file)
- .then(function (fileHash) {
- if (row.hash && row.hash == fileHash) {
- // We have to close the file before modifying it from the main
- // thread (at least on Windows, where assigning lastModifiedTime
- // throws an NS_ERROR_FILE_IS_LOCKED otherwise)
- return Zotero.Promise.resolve(file.close())
- .then(function () {
- Zotero.debug("Mod time didn't match (" + fmtime + "!=" + mtime + ") "
- + "but hash did for " + nsIFile.leafName + " for item " + lk
- + " -- updating file mod time");
- try {
- nsIFile.lastModifiedTime = row.mtime;
- }
- catch (e) {
- Zotero.File.checkFileAccessError(e, nsIFile, 'update');
- }
- });
- }
-
- // Mark file for upload
- Zotero.debug("Marking attachment " + lk + " as changed "
- + "(" + mtime + " != " + fmtime + ")");
- updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD;
- });
- });
- })
- .finally(function () {
- if (file) {
- //Zotero.debug("Closing file for item " + lk);
- file.close();
- }
- })
- .catch(function (e) {
- if (e instanceof OS.File.Error &&
- (e.becauseNoSuchFile
- // This can happen if a path is too long on Windows,
- // e.g. a file is being accessed on a VM through a share
- // (and probably in other cases).
- || (e.winLastError && e.winLastError == 3)
- // Handle long filenames on OS X/Linux
- || (e.unixErrno && e.unixErrno == 63))) {
- Zotero.debug("Marking attachment " + lk + " as missing");
- updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD;
- return;
- }
-
- if (e instanceof OS.File.Error) {
- if (e.becauseClosed) {
- Zotero.debug("File was closed", 2);
- }
- Zotero.debug(e);
- Zotero.debug(e.toString());
- throw new Error("Error for operation '" + e.operation + "' for " + nsIFile.path);
- }
-
- throw e;
- })
- .then(function () {
- return checkItems();
- });
- };
-
- return checkItems()
- .then(function () {
- for (let itemID in updatedStates) {
- Zotero.Sync.Storage.setSyncState(itemID, updatedStates[itemID]);
- changed = true;
- }
-
- if (!changed) {
- Zotero.debug("No synced files have changed locally");
- }
-
- let msg = "Checked " + numItems + " files in ";
- if (libraryID !== false) {
- msg += "library " + libraryID + " in ";
- }
- msg += (new Date() - t) + "ms";
- Zotero.debug(msg);
-
- return changed;
- });
- });
- };
-
-
- /**
- * Download a single file
- *
- * If no queue is active, start one. Otherwise, add to existing queue.
- */
- this.downloadFile = function (item, requestCallbacks) {
- var itemID = item.id;
- var mode = getModeFromLibrary(item.libraryID);
-
- // TODO: verify WebDAV on-demand?
- if (!mode || !mode.verified) {
- Zotero.debug("File syncing is not active for item's library -- skipping download");
- return false;
- }
-
- if (!item.isImportedAttachment()) {
- throw new Error("Not an imported attachment");
- }
-
- if (item.getFile()) {
- Zotero.debug("File already exists -- replacing");
- }
-
- // TODO: start sync icon in cacheCredentials
- return Zotero.Promise.try(function () {
- return mode.cacheCredentials();
- })
- .then(function () {
- // TODO: start sync icon
- var library = item.libraryID;
- var queue = Zotero.Sync.Storage.QueueManager.get(
- 'download', library
- );
-
- if (!requestCallbacks) {
- requestCallbacks = {};
- }
- var onStart = function (request) {
- return mode.downloadFile(request);
- };
- requestCallbacks.onStart = requestCallbacks.onStart
- ? [onStart, requestCallbacks.onStart]
- : onStart;
-
- var request = new Zotero.Sync.Storage.Request(
- library + '/' + item.key, requestCallbacks
- );
-
- queue.addRequest(request, true);
- queue.start();
-
- return request.promise;
- });
- }
-
-
- /**
- * Extract a downloaded file and update the database metadata
- *
- * This is called from Zotero.Sync.Server.StreamListener.onStopRequest()
- *
- * @return {Promise<Object>} data - Promise for object with properties 'request', 'item',
- * 'compressed', 'syncModTime', 'syncHash'
- */
- this.processDownload = Zotero.Promise.coroutine(function* (data) {
- var funcName = "Zotero.Sync.Storage.processDownload()";
-
- if (!data) {
- throw "'data' not set in " + funcName;
- }
-
- if (!data.item) {
- throw "'data.item' not set in " + funcName;
- }
-
- if (!data.syncModTime) {
- throw "'data.syncModTime' not set in " + funcName;
- }
-
- if (!data.compressed && !data.syncHash) {
- throw "'data.syncHash' is required if 'data.compressed' is false in " + funcName;
- }
-
- var item = data.item;
- var syncModTime = data.syncModTime;
- var syncHash = data.syncHash;
-
- // TODO: Test file hash
-
- if (data.compressed) {
- var newFile = yield _processZipDownload(item);
- }
- else {
- var newFile = yield _processDownload(item);
- }
-
- // If |newFile| is set, the file was renamed, so set item filename to that
- // and mark for updated
- var file = item.getFile();
- if (newFile && file.leafName != newFile.leafName) {
- // Bypass library access check
- _updatesInProgress = true;
-
- // If library isn't editable but filename was changed, update
- // database without updating the item's mod time, which would result
- // in a library access error
- if (!Zotero.Items.isEditable(item)) {
- Zotero.debug("File renamed without library access -- "
- + "updating itemAttachments path", 3);
- item.relinkAttachmentFile(newFile, true);
- var useCurrentModTime = false;
- }
- else {
- item.relinkAttachmentFile(newFile);
-
- // TODO: use an integer counter instead of mod time for change detection
- var useCurrentModTime = true;
- }
-
- file = item.getFile();
- _updatesInProgress = false;
- }
- else {
- var useCurrentModTime = false;
- }
-
- if (!file) {
- // This can happen if an HTML snapshot filename was changed and synced
- // elsewhere but the renamed file wasn't synced, so the ZIP doesn't
- // contain a file with the known name
- var missingFile = item.getFile(null, true);
- Components.utils.reportError("File '" + missingFile.leafName
- + "' not found after processing download "
- + item.libraryID + "/" + item.key + " in " + funcName);
- return false;
- }
-
- Zotero.DB.beginTransaction();
-
- //var syncState = Zotero.Sync.Storage.getSyncState(item.id);
- //var updateItem = syncState != this.SYNC_STATE_TO_DOWNLOAD;
- var updateItem = false;
-
- try {
- if (useCurrentModTime) {
- file.lastModifiedTime = new Date();
-
- // Reset hash and sync state
- Zotero.Sync.Storage.setSyncedHash(item.id, null);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD);
- this.queueItem(item);
- }
- else {
- file.lastModifiedTime = syncModTime;
- // If hash not provided (e.g., WebDAV), calculate it now
- if (!syncHash) {
- syncHash = item.attachmentHash;
- }
- Zotero.Sync.Storage.setSyncedHash(item.id, syncHash);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- }
- }
- catch (e) {
- Zotero.File.checkFileAccessError(e, file, 'update');
- }
-
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, syncModTime, updateItem);
- Zotero.DB.commitTransaction();
-
- return true;
- });
-
-
this.checkServerPromise = function (mode) {
return mode.checkServer()
.spread(function (uri, status) {
@@ -1277,570 +230,16 @@ Zotero.Sync.Storage = new function () {
}
- this.getItemFromRequestName = function (name) {
- var [libraryID, key] = name.split('/');
- return Zotero.Items.getByLibraryAndKey(libraryID, key);
- }
-
-
- this.notify = function(event, type, ids, extraData) {
- if (event == 'open' && type == 'file') {
- let timestamp = new Date().getTime();
-
- for each(let id in ids) {
- _uploadCheckFiles.push({
- itemID: id,
- timestamp: timestamp
- });
- }
- }
- }
-
-
- //
- // Private methods
- //
- function getModeFromLibrary(libraryID) {
- if (libraryID === undefined) {
- throw new Error("libraryID not provided");
- }
-
- // Personal library
- if (!libraryID) {
- if (Zotero.Sync.Storage.ZFS.includeUserFiles) {
- return Zotero.Sync.Storage.ZFS;
- }
- if (Zotero.Sync.Storage.WebDAV.includeUserFiles) {
- return Zotero.Sync.Storage.WebDAV;
- }
- return false;
- }
-
- // Group library
- else {
- if (Zotero.Sync.Storage.ZFS.includeGroupFiles) {
- return Zotero.Sync.Storage.ZFS;
- }
- return false;
- }
- }
-
-
- var _processDownload = Zotero.Promise.coroutine(function* (item) {
- var funcName = "Zotero.Sync.Storage._processDownload()";
-
- var tempFile = Zotero.getTempDirectory();
- tempFile.append(item.key + '.tmp');
-
- if (!tempFile.exists()) {
- Zotero.debug(tempFile.path);
- throw ("Downloaded file not found in " + funcName);
- }
-
- var parentDir = Zotero.Attachments.getStorageDirectory(item);
- if (!parentDir.exists()) {
- yield Zotero.Attachments.createDirectoryForItem(item);
- }
-
- _deleteExistingAttachmentFiles(item);
-
- var file = item.getFile(null, true);
- if (!file) {
- throw ("Empty path for item " + item.key + " in " + funcName);
- }
- // Don't save Windows aliases
- if (file.leafName.endsWith('.lnk')) {
- return false;
- }
-
- var fileName = file.leafName;
- var renamed = false;
-
- // Make sure the new filename is valid, in case an invalid character made it over
- // (e.g., from before we checked for them)
- var filteredName = Zotero.File.getValidFileName(fileName);
- if (filteredName != fileName) {
- Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
- fileName = filteredName;
- file.leafName = fileName;
- renamed = true;
- }
-
- Zotero.debug("Moving download file " + tempFile.leafName + " into attachment directory as '" + fileName + "'");
- try {
- var destFile = parentDir.clone();
- destFile.append(fileName);
- Zotero.File.createShortened(destFile, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
- }
- catch (e) {
- Zotero.File.checkFileAccessError(e, destFile, 'create');
- }
-
- if (destFile.leafName != fileName) {
- Zotero.debug("Changed filename '" + fileName + "' to '" + destFile.leafName + "'");
-
- // Abort if Windows path limitation would cause filenames to be overly truncated
- if (Zotero.isWin && destFile.leafName.length < 40) {
- try {
- destFile.remove(false);
- }
- catch (e) {}
- // TODO: localize
- var msg = "Due to a Windows path length limitation, your Zotero data directory "
- + "is too deep in the filesystem for syncing to work reliably. "
- + "Please relocate your Zotero data to a higher directory.";
- Zotero.debug(msg, 1);
- throw new Error(msg);
- }
-
- renamed = true;
- }
-
- try {
- tempFile.moveTo(parentDir, destFile.leafName);
- }
- catch (e) {
- try {
- destFile.remove(false);
- }
- catch (e) {}
-
- Zotero.File.checkFileAccessError(e, destFile, 'create');
- }
-
- var returnFile = null;
- // processDownload() needs to know that we're renaming the file
- if (renamed) {
- returnFile = destFile.clone();
- }
- return returnFile;
- });
-
-
- var _processZipDownload = Zotero.Promise.coroutine(function* (item) {
- var funcName = "Zotero.Sync.Storage._processDownloadedZip()";
-
- var zipFile = Zotero.getTempDirectory();
- zipFile.append(item.key + '.zip.tmp');
-
- if (!zipFile.exists()) {
- throw ("Downloaded ZIP file not found in " + funcName);
- }
-
- var zipReader = Components.classes["@mozilla.org/libjar/zip-reader;1"].
- createInstance(Components.interfaces.nsIZipReader);
- try {
- zipReader.open(zipFile);
- zipReader.test(null);
-
- Zotero.debug("ZIP file is OK");
- }
- catch (e) {
- Zotero.debug(zipFile.leafName + " is not a valid ZIP file", 2);
- zipReader.close();
-
- try {
- zipFile.remove(false);
- }
- catch (e) {
- Zotero.File.checkFileAccessError(e, zipFile, 'delete');
- }
-
- // TODO: Remove prop file to trigger reuploading, in case it was an upload error?
-
- return false;
- }
-
- var parentDir = Zotero.Attachments.getStorageDirectory(item);
- if (!parentDir.exists()) {
- yield Zotero.Attachments.createDirectoryForItem(item);
- }
-
- try {
- _deleteExistingAttachmentFiles(item);
- }
- catch (e) {
- zipReader.close();
- throw (e);
- }
-
- var returnFile = null;
- var count = 0;
-
- var entries = zipReader.findEntries(null);
- while (entries.hasMore()) {
- count++;
- var entryName = entries.getNext();
- var b64re = /%ZB64$/;
- if (entryName.match(b64re)) {
- var fileName = Zotero.Utilities.Internal.Base64.decode(
- entryName.replace(b64re, '')
- );
- }
- else {
- var fileName = entryName;
- }
-
- if (fileName.startsWith('.zotero')) {
- Zotero.debug("Skipping " + fileName);
- continue;
- }
-
- Zotero.debug("Extracting " + fileName);
-
- var primaryFile = false;
- var filtered = false;
- var renamed = false;
-
- // Get the old filename
- var itemFileName = item.getFilename();
-
- // Make sure the new filename is valid, in case an invalid character
- // somehow make it into the ZIP (e.g., from before we checked for them)
- //
- // Do this before trying to use the relative descriptor, since otherwise
- // it might fail silently and select the parent directory
- var filteredName = Zotero.File.getValidFileName(fileName);
- if (filteredName != fileName) {
- Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
- fileName = filteredName;
- filtered = true;
- }
-
- // Name in ZIP is a relative descriptor, so file has to be reconstructed
- // using setRelativeDescriptor()
- var destFile = parentDir.clone();
- destFile.QueryInterface(Components.interfaces.nsILocalFile);
- destFile.setRelativeDescriptor(parentDir, fileName);
-
- fileName = destFile.leafName;
-
- // If only one file in zip and it doesn't match the known filename,
- // take our chances and use that name
- if (count == 1 && !entries.hasMore() && itemFileName) {
- // May not be necessary, but let's be safe
- itemFileName = Zotero.File.getValidFileName(itemFileName);
- if (itemFileName != fileName) {
- Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'", 2);
- Components.utils.reportError("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'");
- fileName = itemFileName;
- destFile.leafName = fileName;
- renamed = true;
- }
- }
-
- var primaryFile = itemFileName == fileName;
- if (primaryFile && filtered) {
- renamed = true;
- }
-
- if (destFile.exists()) {
- var msg = "ZIP entry '" + fileName + "' " + "already exists";
- Zotero.debug(msg, 2);
- Components.utils.reportError(msg + " in " + funcName);
- continue;
- }
-
- try {
- Zotero.File.createShortened(destFile, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
- }
- catch (e) {
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
-
- zipReader.close();
-
- Zotero.File.checkFileAccessError(e, destFile, 'create');
- }
-
- if (destFile.leafName != fileName) {
- Zotero.debug("Changed filename '" + fileName + "' to '" + destFile.leafName + "'");
-
- // Abort if Windows path limitation would cause filenames to be overly truncated
- if (Zotero.isWin && destFile.leafName.length < 40) {
- try {
- destFile.remove(false);
- }
- catch (e) {}
- zipReader.close();
- // TODO: localize
- var msg = "Due to a Windows path length limitation, your Zotero data directory "
- + "is too deep in the filesystem for syncing to work reliably. "
- + "Please relocate your Zotero data to a higher directory.";
- Zotero.debug(msg, 1);
- throw new Error(msg);
- }
-
- if (primaryFile) {
- renamed = true;
- }
- }
-
- try {
- zipReader.extract(entryName, destFile);
- }
- catch (e) {
- try {
- destFile.remove(false);
- }
- catch (e) {}
-
- // For advertising junk files, ignore a bug on Windows where
- // destFile.create() works but zipReader.extract() doesn't
- // when the path length is close to 255.
- if (destFile.leafName.match(/[a-zA-Z0-9+=]{130,}/)) {
- var msg = "Ignoring error extracting '" + destFile.path + "'";
- Zotero.debug(msg, 2);
- Zotero.debug(e, 2);
- Components.utils.reportError(msg + " in " + funcName);
- continue;
- }
-
- zipReader.close();
-
- Zotero.File.checkFileAccessError(e, destFile, 'create');
- }
-
- destFile.permissions = 0644;
-
- // If we're renaming the main file, processDownload() needs to know
- if (renamed) {
- returnFile = destFile;
- }
- }
- zipReader.close();
- zipFile.remove(false);
-
- return returnFile;
- });
-
-
- function _deleteExistingAttachmentFiles(item) {
- var funcName = "Zotero.Sync.Storage._deleteExistingAttachmentFiles()";
-
- var parentDir = Zotero.Attachments.getStorageDirectory(item);
-
- // Delete existing files
- var otherFiles = parentDir.directoryEntries;
- otherFiles.QueryInterface(Components.interfaces.nsIDirectoryEnumerator);
- var filesToDelete = [];
- var file;
- while (file = otherFiles.nextFile) {
- if (file.leafName.startsWith('.zotero')) {
- continue;
- }
-
- // Check symlink awareness, just to be safe
- if (!parentDir.contains(file, false)) {
- var msg = "Storage directory doesn't contain '" + file.leafName + "'";
- Zotero.debug(msg, 2);
- Components.utils.reportError(msg + " in " + funcName);
- continue;
- }
-
- filesToDelete.push(file);
- }
- otherFiles.close();
-
- // Do deletes outside of the enumerator to avoid an access error on Windows
- for each(var file in filesToDelete) {
- try {
- if (file.isFile()) {
- Zotero.debug("Deleting existing file " + file.leafName);
- file.remove(false);
- }
- else if (file.isDirectory()) {
- Zotero.debug("Deleting existing directory " + file.leafName);
- file.remove(true);
- }
- }
- catch (e) {
- Zotero.File.checkFileAccessError(e, file, 'delete');
- }
- }
- }
-
-
- /**
- * Create zip file of attachment directory
- *
- * @param {Zotero.Sync.Storage.Request} request
- * @param {Function} callback
- * @return {Boolean} TRUE if zip process started,
- * FALSE if storage was empty
- */
- this.createUploadFile = function (request, callback) {
- var item = Zotero.Sync.Storage.getItemFromRequestName(request.name);
- Zotero.debug("Creating zip file for item " + item.libraryID + "/" + item.key);
-
- try {
- switch (item.attachmentLinkMode) {
- case Zotero.Attachments.LINK_MODE_LINKED_FILE:
- case Zotero.Attachments.LINK_MODE_LINKED_URL:
- throw (new Error(
- "Upload file must be an imported snapshot or file in "
- + "Zotero.Sync.Storage.createUploadFile()"
- ));
- }
-
- var dir = Zotero.Attachments.getStorageDirectory(item);
-
- var tmpFile = Zotero.getTempDirectory();
- tmpFile.append(item.key + '.zip');
-
- var zw = Components.classes["@mozilla.org/zipwriter;1"]
- .createInstance(Components.interfaces.nsIZipWriter);
- zw.open(tmpFile, 0x04 | 0x08 | 0x20); // open rw, create, truncate
- var fileList = _zipDirectory(dir, dir, zw);
- if (fileList.length == 0) {
- Zotero.debug('No files to add -- removing zip file');
- zw.close();
- tmpFile.remove(null);
- return false;
- }
-
- Zotero.debug('Creating ' + tmpFile.leafName + ' with ' + fileList.length + ' file(s)');
-
- var observer = new Zotero.Sync.Storage.ZipWriterObserver(
- zw, callback, { request: request, files: fileList }
- );
- zw.processQueue(observer, null);
- return true;
- }
- // DEBUG: Do we want to catch this?
- catch (e) {
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
- return false;
- }
- }
-
- function _zipDirectory(rootDir, dir, zipWriter) {
- var fileList = [];
- dir = dir.directoryEntries;
- while (dir.hasMoreElements()) {
- var file = dir.getNext();
- file.QueryInterface(Components.interfaces.nsILocalFile);
- if (file.isDirectory()) {
- //Zotero.debug("Recursing into directory " + file.leafName);
- fileList.concat(_zipDirectory(rootDir, file, zipWriter));
- continue;
- }
- var fileName = file.getRelativeDescriptor(rootDir);
- if (fileName.startsWith('.zotero')) {
- Zotero.debug('Skipping file ' + fileName);
- continue;
- }
-
- //Zotero.debug("Adding file " + fileName);
-
- zipWriter.addEntryFile(
- fileName,
- Components.interfaces.nsIZipWriter.COMPRESSION_DEFAULT,
- file,
- true
- );
- fileList.push(fileName);
- }
- return fileList;
- }
- /**
- * Get files marked as ready to download
- *
- * @inner
- * @return {Number[]} Array of attachment itemIDs
- */
- function _getFilesToDownload(libraryID, forcedOnly) {
- var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
- + "WHERE libraryID=? AND syncState IN (?";
- var params = [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD];
- if (!forcedOnly) {
- sql += ",?";
- params.push(Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD);
- }
- sql += ") "
- // Skip attachments with empty path, which can't be saved, and files with .zotero*
- // paths, which have somehow ended up in some users' libraries
- + "AND path!='' AND path NOT LIKE 'storage:.zotero%'";
- var itemIDs = Zotero.DB.columnQuery(sql, params);
- if (!itemIDs) {
- return [];
- }
- return itemIDs;
- }
- /**
- * Get files marked as ready to upload
- *
- * @inner
- * @return {Number[]} Array of attachment itemIDs
- */
- function _getFilesToUpload(libraryID) {
- var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
- + "WHERE syncState IN (?,?) AND linkMode IN (?,?)";
- var params = [
- Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD,
- Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD,
- Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
- Zotero.Attachments.LINK_MODE_IMPORTED_URL
- ];
- if (typeof libraryID != 'undefined') {
- sql += " AND libraryID=?";
- params.push(libraryID);
- }
- else {
- throw new Error("libraryID not specified");
- }
- var itemIDs = Zotero.DB.columnQuery(sql, params);
- if (!itemIDs) {
- return [];
- }
- return itemIDs;
- }
- /**
- * Get files to check for local modifications for uploading
- *
- * This includes files previously modified and files opened externally
- * via Zotero within _maxCheckAgeInSeconds.
- */
- function _getFilesToCheck(libraryID) {
- var minTime = new Date().getTime() - (_maxCheckAgeInSeconds * 1000);
-
- // Get files by modification time
- var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
- + "WHERE libraryID=? AND linkMode IN (?,?) AND syncState IN (?) AND "
- + "storageModTime>=?";
- var params = [
- libraryID,
- Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
- Zotero.Attachments.LINK_MODE_IMPORTED_URL,
- Zotero.Sync.Storage.SYNC_STATE_IN_SYNC,
- minTime
- ];
- var itemIDs = Zotero.DB.columnQuery(sql, params) || [];
-
- // Get files by open time
- _uploadCheckFiles.filter(function (x) x.timestamp >= minTime);
- itemIDs = itemIDs.concat([x.itemID for each(x in _uploadCheckFiles)])
-
- return Zotero.Utilities.arrayUnique(itemIDs);
- }
- /**
- * @inner
- * @return {String[]|FALSE} Array of keys, or FALSE if none
- */
- this.getDeletedFiles = function () {
- var sql = "SELECT key FROM storageDeleteLog";
- return Zotero.DB.columnQuery(sql);
- }
+
function error(e) {
@@ -1892,56 +291,4 @@ Zotero.Sync.Storage = new function () {
}
-/**
- * Request observer for zip writing
- *
- * Implements nsIRequestObserver
- *
- * @param {nsIZipWriter} zipWriter
- * @param {Function} callback
- * @param {Object} data
- */
-Zotero.Sync.Storage.ZipWriterObserver = function (zipWriter, callback, data) {
- this._zipWriter = zipWriter;
- this._callback = callback;
- this._data = data;
-}
-Zotero.Sync.Storage.ZipWriterObserver.prototype = {
- onStartRequest: function () {},
-
- onStopRequest: function(req, context, status) {
- var zipFileName = this._zipWriter.file.leafName;
-
- var originalSize = 0;
- for each(var fileName in this._data.files) {
- var entry = this._zipWriter.getEntry(fileName);
- if (!entry) {
- var msg = "ZIP entry '" + fileName + "' not found for request '" + this._data.request.name + "'";
- Components.utils.reportError(msg);
- Zotero.debug(msg, 1);
- this._zipWriter.close();
- this._callback(false);
- return;
- }
- originalSize += entry.realSize;
- }
- delete this._data.files;
-
- this._zipWriter.close();
-
- Zotero.debug("Zip of " + zipFileName + " finished with status " + status
- + " (original " + Math.round(originalSize / 1024) + "KB, "
- + "compressed " + Math.round(this._zipWriter.file.fileSize / 1024) + "KB, "
- + Math.round(
- ((originalSize - this._zipWriter.file.fileSize) / originalSize) * 100
- ) + "% reduction)");
-
- Zotero.Sync.Storage.compressionTracker.compressed += this._zipWriter.file.fileSize;
- Zotero.Sync.Storage.compressionTracker.uncompressed += originalSize;
- Zotero.debug("Average compression so far: "
- + Math.round(Zotero.Sync.Storage.compressionTracker.ratio * 100) + "%");
-
- this._callback(this._data);
- }
-}
diff --git a/chrome/content/zotero/xpcom/storage/mode.js b/chrome/content/zotero/xpcom/storage/mode.js
deleted file mode 100644
index 472b6a101..000000000
--- a/chrome/content/zotero/xpcom/storage/mode.js
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- ***** BEGIN LICENSE BLOCK *****
-
- Copyright © 2009 Center for History and New Media
- George Mason University, Fairfax, Virginia, USA
- http://zotero.org
-
- This file is part of Zotero.
-
- Zotero is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- Zotero is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with Zotero. If not, see <http://www.gnu.org/licenses/>.
-
- ***** END LICENSE BLOCK *****
-*/
-
-
-Zotero.Sync.Storage.Mode = function () {};
-
-Zotero.Sync.Storage.Mode.prototype.__defineGetter__('verified', function () {
- return this._verified;
-});
-
-Zotero.Sync.Storage.Mode.prototype.__defineGetter__('username', function () {
- return this._username;
-});
-
-Zotero.Sync.Storage.Mode.prototype.__defineGetter__('password', function () {
- return this._password;
-});
-
-Zotero.Sync.Storage.Mode.prototype.__defineSetter__('password', function (val) {
- this._password = val;
-});
-
-Zotero.Sync.Storage.Mode.prototype.init = function () {
- return this._init();
-}
-
-Zotero.Sync.Storage.Mode.prototype.sync = function (observer) {
- return Zotero.Sync.Storage.sync(this.name, observer);
-}
-
-Zotero.Sync.Storage.Mode.prototype.downloadFile = function (request) {
- return this._downloadFile(request);
-}
-
-Zotero.Sync.Storage.Mode.prototype.uploadFile = function (request) {
- return this._uploadFile(request);
-}
-
-Zotero.Sync.Storage.Mode.prototype.getLastSyncTime = function (libraryID) {
- return this._getLastSyncTime(libraryID);
-}
-
-Zotero.Sync.Storage.Mode.prototype.setLastSyncTime = function (callback, useLastSyncTime) {
- return this._setLastSyncTime(callback, useLastSyncTime);
-}
-
-Zotero.Sync.Storage.Mode.prototype.checkServer = function (callback) {
- return this._checkServer(callback);
-}
-
-Zotero.Sync.Storage.Mode.prototype.checkServerCallback = function (uri, status, window, skipSuccessMessage) {
- return this._checkServerCallback(uri, status, window, skipSuccessMessage);
-}
-
-Zotero.Sync.Storage.Mode.prototype.cacheCredentials = function () {
- return this._cacheCredentials();
-}
-
-Zotero.Sync.Storage.Mode.prototype.purgeDeletedStorageFiles = function (callback) {
- return this._purgeDeletedStorageFiles(callback);
-}
-
-Zotero.Sync.Storage.Mode.prototype.purgeOrphanedStorageFiles = function (callback) {
- return this._purgeOrphanedStorageFiles(callback);
-}
diff --git a/chrome/content/zotero/xpcom/storage/queue.js b/chrome/content/zotero/xpcom/storage/queue.js
deleted file mode 100644
index 25a399d72..000000000
--- a/chrome/content/zotero/xpcom/storage/queue.js
+++ /dev/null
@@ -1,427 +0,0 @@
-/*
- ***** BEGIN LICENSE BLOCK *****
-
- Copyright © 2009 Center for History and New Media
- George Mason University, Fairfax, Virginia, USA
- http://zotero.org
-
- This file is part of Zotero.
-
- Zotero is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- Zotero is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with Zotero. If not, see <http://www.gnu.org/licenses/>.
-
- ***** END LICENSE BLOCK *****
-*/
-
-/**
- * Queue for storage sync transfer requests
- *
- * @param {String} type Queue type (e.g., 'download' or 'upload')
- */
-Zotero.Sync.Storage.Queue = function (type, libraryID) {
- Zotero.debug("Initializing " + type + " queue for library " + libraryID);
-
- // Public properties
- this.type = type;
- this.libraryID = libraryID;
- this.maxConcurrentRequests = 1;
- this.activeRequests = 0;
- this.totalRequests = 0;
-
- // Private properties
- this._requests = {};
- this._highPriority = [];
- this._running = false;
- this._stopping = false;
- this._finished = false;
- this._error = false;
- this._finishedReqs = 0;
- this._localChanges = false;
- this._remoteChanges = false;
- this._conflicts = [];
- this._cachedPercentage;
- this._cachedPercentageTime;
-}
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('name', function () {
- return this.type + "/" + this.libraryID;
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('Type', function () {
- return this.type[0].toUpperCase() + this.type.substr(1);
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('running', function () this._running);
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('stopping', function () this._stopping);
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('finished', function () this._finished);
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('unfinishedRequests', function () {
- return this.totalRequests - this.finishedRequests;
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('finishedRequests', function () {
- return this._finishedReqs;
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineSetter__('finishedRequests', function (val) {
- Zotero.debug("Finished requests: " + val);
- Zotero.debug("Total requests: " + this.totalRequests);
-
- this._finishedReqs = val;
-
- if (val == 0) {
- return;
- }
-
- // Last request
- if (val == this.totalRequests) {
- Zotero.debug(this.Type + " queue is done for library " + this.libraryID);
-
- // DEBUG info
- Zotero.debug("Active requests: " + this.activeRequests);
-
- if (this.activeRequests) {
- throw new Error(this.Type + " queue for library " + this.libraryID
- + " can't be done if there are active requests");
- }
-
- this._running = false;
- this._stopping = false;
- this._finished = true;
- this._requests = {};
- this._highPriority = [];
-
- var localChanges = this._localChanges;
- var remoteChanges = this._remoteChanges;
- var conflicts = this._conflicts.concat();
- var deferred = this._deferred;
- this._localChanges = false;
- this._remoteChanges = false;
- this._conflicts = [];
- this._deferred = null;
-
- if (!this._error) {
- Zotero.debug("Resolving promise for queue " + this.name);
- Zotero.debug(this._localChanges);
- Zotero.debug(this._remoteChanges);
- Zotero.debug(this._conflicts);
-
- deferred.resolve({
- libraryID: this.libraryID,
- type: this.type,
- localChanges: localChanges,
- remoteChanges: remoteChanges,
- conflicts: conflicts
- });
- }
- else {
- Zotero.debug("Rejecting promise for queue " + this.name);
- var e = this._error;
- this._error = false;
- e.libraryID = this.libraryID;
- e.type = this.type;
- deferred.reject(e);
- }
-
- return;
- }
-
- if (this._stopping) {
- return;
- }
- this.advance();
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('queuedRequests', function () {
- return this.unfinishedRequests - this.activeRequests;
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('remaining', function () {
- var remaining = 0;
- for each(var request in this._requests) {
- remaining += request.remaining;
- }
- return remaining;
-});
-
-Zotero.Sync.Storage.Queue.prototype.__defineGetter__('percentage', function () {
- if (this.totalRequests == 0) {
- return 0;
- }
- if (this._finished) {
- return 100;
- }
-
- // Cache percentage for a second
- if (this._cachedPercentage && (new Date() - this._cachedPercentageTime) < 1000) {
- return this._cachedPercentage;
- }
-
- var completedRequests = 0;
- for each(var request in this._requests) {
- completedRequests += request.percentage / 100;
- }
- this._cachedPercentage = Math.round((completedRequests / this.totalRequests) * 100);
- this._cachedPercentageTime = new Date();
- return this._cachedPercentage;
-});
-
-
-Zotero.Sync.Storage.Queue.prototype.isRunning = function () {
- return this._running;
-}
-
-Zotero.Sync.Storage.Queue.prototype.isStopping = function () {
- return this._stopping;
-}
-
-
-/**
- * Add a request to this queue
- *
- * @param {Zotero.Sync.Storage.Request} request
- * @param {Boolean} highPriority Add or move request to high priority queue
- */
-Zotero.Sync.Storage.Queue.prototype.addRequest = function (request, highPriority) {
- if (this._finished) {
- this.reset();
- }
-
- request.queue = this;
- var name = request.name;
- Zotero.debug("Queuing " + this.type + " request '" + name + "' for library " + this.libraryID);
-
- if (this._requests[name]) {
- if (highPriority) {
- Zotero.debug("Moving " + name + " to high-priority queue");
- this._requests[name].importCallbacks(request);
- this._highPriority.push(name);
- return;
- }
-
- Zotero.debug("Request '" + name + "' already exists");
- return;
- }
-
- this._requests[name] = request;
- this.totalRequests++;
-
- if (highPriority) {
- this._highPriority.push(name);
- }
-}
-
-
-Zotero.Sync.Storage.Queue.prototype.start = function () {
- if (!this._deferred || this._deferred.promise.isFulfilled()) {
- Zotero.debug("Creating deferred for queue " + this.name);
- this._deferred = Zotero.Promise.defer();
- }
- // The queue manager needs to know what queues were running in the
- // current session
- Zotero.Sync.Storage.QueueManager.addCurrentQueue(this);
-
- var self = this;
- setTimeout(function () {
- self.advance();
- }, 0);
-
- return this._deferred.promise;
-}
-
-
-
-/**
- * Start another request in this queue if there's an available slot
- */
-Zotero.Sync.Storage.Queue.prototype.advance = function () {
- this._running = true;
- this._finished = false;
-
- if (this._stopping) {
- Zotero.debug(this.Type + " queue for library " + this.libraryID
- + "is being stopped in Zotero.Sync.Storage.Queue.advance()", 2);
- return;
- }
-
- if (!this.queuedRequests) {
- Zotero.debug("No remaining requests in " + this.type
- + " queue for library " + this.libraryID + " ("
- + this.activeRequests + " active, "
- + this.finishedRequests + " finished)");
- return;
- }
-
- if (this.activeRequests >= this.maxConcurrentRequests) {
- Zotero.debug(this.Type + " queue for library " + this.libraryID
- + " is busy (" + this.activeRequests + "/"
- + this.maxConcurrentRequests + ")");
- return;
- }
-
-
-
- // Start the first unprocessed request
-
- // Try the high-priority queue first
- var self = this;
- var request, name;
- while (name = this._highPriority.shift()) {
- request = this._requests[name];
- if (request.isRunning() || request.isFinished()) {
- continue;
- }
-
- let requestName = name;
-
- Zotero.Promise.try(function () {
- var promise = request.start();
- self.advance();
- return promise;
- })
- .then(function (result) {
- if (result.localChanges) {
- self._localChanges = true;
- }
- if (result.remoteChanges) {
- self._remoteChanges = true;
- }
- if (result.conflict) {
- self.addConflict(
- requestName,
- result.conflict.local,
- result.conflict.remote
- );
- }
- })
- .catch(function (e) {
- self.error(e);
- });
-
- return;
- }
-
- // And then others
- for each(var request in this._requests) {
- if (request.isRunning() || request.isFinished()) {
- continue;
- }
-
- let requestName = request.name;
-
- // This isn't in a Zotero.Promise.try() because the request needs to get marked
- // as running immediately so that it doesn't get run again by a
- // subsequent advance() call.
- try {
- var promise = request.start();
- self.advance();
- }
- catch (e) {
- self.error(e);
- }
-
- promise.then(function (result) {
- if (result.localChanges) {
- self._localChanges = true;
- }
- if (result.remoteChanges) {
- self._remoteChanges = true;
- }
- if (result.conflict) {
- self.addConflict(
- requestName,
- result.conflict.local,
- result.conflict.remote
- );
- }
- })
- .catch(function (e) {
- self.error(e);
- });
-
- return;
- }
-}
-
-
-Zotero.Sync.Storage.Queue.prototype.updateProgress = function () {
- Zotero.Sync.Storage.QueueManager.updateProgress();
-}
-
-
-Zotero.Sync.Storage.Queue.prototype.addConflict = function (requestName, localData, remoteData) {
- Zotero.debug('===========');
- Zotero.debug(localData);
- Zotero.debug(remoteData);
-
- this._conflicts.push({
- name: requestName,
- localData: localData,
- remoteData: remoteData
- });
-}
-
-
-Zotero.Sync.Storage.Queue.prototype.error = function (e) {
- if (!this._error) {
- if (this.isRunning()) {
- this._error = e;
- }
- else {
- Zotero.debug("Queue " + this.name + " was no longer running -- not assigning error", 2);
- }
- }
- Zotero.debug(e, 1);
- this.stop();
-}
-
-
-/**
- * Stops all requests in this queue
- */
-Zotero.Sync.Storage.Queue.prototype.stop = function () {
- if (!this._running) {
- Zotero.debug(this.Type + " queue for library " + this.libraryID
- + " is not running");
- return;
- }
- if (this._stopping) {
- Zotero.debug("Already stopping " + this.type + " queue for library "
- + this.libraryID);
- return;
- }
-
- Zotero.debug("Stopping " + this.type + " queue for library " + this.libraryID);
-
- // If no requests, finish manually
- /*if (this.activeRequests == 0) {
- this._finishedRequests = this._finishedRequests;
- return;
- }*/
-
- this._stopping = true;
- for each(var request in this._requests) {
- if (!request.isFinished()) {
- request.stop(true);
- }
- }
-
- Zotero.debug("Queue is stopped");
-}
-
-
-Zotero.Sync.Storage.Queue.prototype.reset = function () {
- this._finished = false;
- this._finishedReqs = 0;
- this.totalRequests = 0;
-}
diff --git a/chrome/content/zotero/xpcom/storage/queueManager.js b/chrome/content/zotero/xpcom/storage/queueManager.js
deleted file mode 100644
index e338458d6..000000000
--- a/chrome/content/zotero/xpcom/storage/queueManager.js
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
- ***** BEGIN LICENSE BLOCK *****
-
- Copyright © 2009 Center for History and New Media
- George Mason University, Fairfax, Virginia, USA
- http://zotero.org
-
- This file is part of Zotero.
-
- Zotero is free software: you can redistribute it and/or modify
- it under the terms of the GNU Affero General Public License as published by
- the Free Software Foundation, either version 3 of the License, or
- (at your option) any later version.
-
- Zotero is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU Affero General Public License for more details.
-
- You should have received a copy of the GNU Affero General Public License
- along with Zotero. If not, see <http://www.gnu.org/licenses/>.
-
- ***** END LICENSE BLOCK *****
-*/
-
-
-Zotero.Sync.Storage.QueueManager = new function () {
- var _queues = {};
- var _currentQueues = [];
-
- this.start = Zotero.Promise.coroutine(function* (libraryID) {
- if (libraryID) {
- var queues = this.getAll(libraryID);
- var suffix = " for library " + libraryID;
- }
- else {
- var queues = this.getAll();
- var suffix = "";
- }
-
- Zotero.debug("Starting file sync queues" + suffix);
-
- var promises = [];
- for each(var queue in queues) {
- if (!queue.unfinishedRequests) {
- continue;
- }
- Zotero.debug("Starting queue " + queue.name);
- promises.push(queue.start());
- }
-
- if (!promises.length) {
- Zotero.debug("No files to sync" + suffix);
- }
-
- var results = yield Zotero.Promise.allSettled(promises);
- Zotero.debug("All storage queues are finished" + suffix);
-
- for (let i = 0; i < results.length; i++) {
- let result = results[i];
- // Check for conflicts to resolve
- if (result.state == "fulfilled") {
- result = result.value;
- if (result.conflicts.length) {
- Zotero.debug("Reconciling conflicts for library " + result.libraryID);
- Zotero.debug(result.conflicts);
- var data = yield _reconcileConflicts(result.conflicts);
- if (data) {
- _processMergeData(data);
- }
- }
- }
- }
-
- return promises;
- });
-
- this.stop = function (libraryID) {
- if (libraryID) {
- var queues = this.getAll(libraryID);
- }
- else {
- var queues = this.getAll();
- }
- for (var queue in queues) {
- queue.stop();
- }
- };
-
-
- /**
- * Retrieving a queue, creating a new one if necessary
- *
- * @param {String} queueName
- */
- this.get = function (queueName, libraryID, noInit) {
- if (typeof libraryID == 'undefined') {
- throw new Error("libraryID not specified");
- }
-
- var hash = queueName + "/" + libraryID;
-
- // Initialize the queue if it doesn't exist yet
- if (!_queues[hash]) {
- if (noInit) {
- return false;
- }
- var queue = new Zotero.Sync.Storage.Queue(queueName, libraryID);
- switch (queueName) {
- case 'download':
- queue.maxConcurrentRequests =
- Zotero.Prefs.get('sync.storage.maxDownloads')
- break;
-
- case 'upload':
- queue.maxConcurrentRequests =
- Zotero.Prefs.get('sync.storage.maxUploads')
- break;
-
- default:
- throw ("Invalid queue '" + queueName + "' in Zotero.Sync.Storage.QueueManager.get()");
- }
- _queues[hash] = queue;
- }
-
- return _queues[hash];
- };
-
-
- this.getAll = function (libraryID) {
- if (typeof libraryID == 'string') {
- throw new Error("libraryID must be a number or undefined");
- }
-
- var queues = [];
- for each(var queue in _queues) {
- if (typeof libraryID == 'undefined' || queue.libraryID === libraryID) {
- queues.push(queue);
- }
- }
- return queues;
- };
-
-
- this.addCurrentQueue = function (queue) {
- if (!this.hasCurrentQueue(queue)) {
- _currentQueues.push(queue.name);
- }
- }
-
-
- this.hasCurrentQueue = function (queue) {
- return _currentQueues.indexOf(queue.name) != -1;
- }
-
-
- /**
- * Stop all queues
- *
- * @param {Boolean} [skipStorageFinish=false] Don't call Zotero.Sync.Storage.finish()
- * when done (used when we stopped because of
- * an error)
- */
- this.cancel = function (skipStorageFinish) {
- Zotero.debug("Stopping all storage queues");
- for each(var queue in _queues) {
- if (queue.isRunning() && !queue.isStopping()) {
- queue.stop();
- }
- }
- }
-
-
- this.finish = function () {
- Zotero.debug("All storage queues are finished");
- _currentQueues = [];
- }
-
-
- /**
- * Calculate the current progress values and trigger a display update
- *
- * Also detects when all queues have finished and ends sync progress
- */
- this.updateProgress = function () {
- var activeRequests = 0;
- var allFinished = true;
- for each(var queue in _queues) {
- // Finished or never started
- if (!queue.isRunning() && !queue.isStopping()) {
- continue;
- }
- allFinished = false;
- activeRequests += queue.activeRequests;
- }
- if (activeRequests == 0) {
- _updateProgressMeters(0);
- if (allFinished) {
- this.finish();
- }
- return;
- }
-
- var status = {};
- for each(var queue in _queues) {
- if (!this.hasCurrentQueue(queue)) {
- continue;
- }
-
- if (!status[queue.libraryID]) {
- status[queue.libraryID] = {};
- }
- if (!status[queue.libraryID][queue.type]) {
- status[queue.libraryID][queue.type] = {};
- }
- status[queue.libraryID][queue.type].statusString = _getQueueStatus(queue);
- status[queue.libraryID][queue.type].percentage = queue.percentage;
- status[queue.libraryID][queue.type].totalRequests = queue.totalRequests;
- status[queue.libraryID][queue.type].finished = queue.finished;
- }
-
- _updateProgressMeters(activeRequests, status);
- }
-
-
- /**
- * Get a status string for a queue
- *
- * @param {Zotero.Sync.Storage.Queue} queue
- * @return {String}
- */
- function _getQueueStatus(queue) {
- var remaining = queue.remaining;
- var unfinishedRequests = queue.unfinishedRequests;
-
- if (!unfinishedRequests) {
- return Zotero.getString('sync.storage.none');
- }
-
- if (remaining > 1000) {
- var bytesRemaining = Zotero.getString(
- 'sync.storage.mbRemaining',
- Zotero.Utilities.numberFormat(remaining / 1000 / 1000, 1)
- );
- }
- else {
- var bytesRemaining = Zotero.getString(
- 'sync.storage.kbRemaining',
- Zotero.Utilities.numberFormat(remaining / 1000, 0)
- );
- }
- var totalRequests = queue.totalRequests;
- var filesRemaining = Zotero.getString(
- 'sync.storage.filesRemaining',
- [totalRequests - unfinishedRequests, totalRequests]
- );
- return bytesRemaining + ' (' + filesRemaining + ')';
- }
-
- /**
- * Cycle through windows, updating progress meters with new values
- */
- function _updateProgressMeters(activeRequests, status) {
- // Get overall percentage across queues
- var sum = 0, num = 0, percentage, total;
- for each(var libraryStatus in status) {
- for each(var queueStatus in libraryStatus) {
- percentage = queueStatus.percentage;
- total = queueStatus.totalRequests;
- sum += total * percentage;
- num += total;
- }
- }
- var percentage = Math.round(sum / num);
-
- var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
- .getService(Components.interfaces.nsIWindowMediator);
- var enumerator = wm.getEnumerator("navigator:browser");
- while (enumerator.hasMoreElements()) {
- var win = enumerator.getNext();
- if (!win.ZoteroPane) continue;
- var doc = win.ZoteroPane.document;
-
- var box = doc.getElementById("zotero-tb-sync-progress-box");
- var meter = doc.getElementById("zotero-tb-sync-progress");
-
- if (activeRequests == 0) {
- box.hidden = true;
- continue;
- }
-
- meter.setAttribute("value", percentage);
- box.hidden = false;
-
- var percentageLabel = doc.getElementById('zotero-tb-sync-progress-tooltip-progress');
- percentageLabel.lastChild.setAttribute('value', percentage + "%");
-
- var statusBox = doc.getElementById('zotero-tb-sync-progress-status');
- statusBox.data = status;
- }
- }
-
-
- var _reconcileConflicts = Zotero.Promise.coroutine(function* (conflicts) {
- var objectPairs = [];
- for each(var conflict in conflicts) {
- var item = Zotero.Sync.Storage.getItemFromRequestName(conflict.name);
- var item1 = yield item.clone(false, false, true);
- item1.setField('dateModified',
- Zotero.Date.dateToSQL(new Date(conflict.localData.modTime), true));
- var item2 = yield item.clone(false, false, true);
- item2.setField('dateModified',
- Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime), true));
- objectPairs.push([item1, item2]);
- }
-
- var io = {
- dataIn: {
- type: 'storagefile',
- captions: [
- Zotero.getString('sync.storage.localFile'),
- Zotero.getString('sync.storage.remoteFile'),
- Zotero.getString('sync.storage.savedFile')
- ],
- objects: objectPairs
- }
- };
-
- var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
- .getService(Components.interfaces.nsIWindowMediator);
- var lastWin = wm.getMostRecentWindow("navigator:browser");
- lastWin.openDialog('chrome://zotero/content/merge.xul', '', 'chrome,modal,centerscreen', io);
-
- if (!io.dataOut) {
- return false;
- }
-
- // Since we're only putting cloned items into the merge window,
- // we have to manually set the ids
- for (var i=0; i<conflicts.length; i++) {
- io.dataOut[i].id = Zotero.Sync.Storage.getItemFromRequestName(conflicts[i].name).id;
- }
-
- return io.dataOut;
- });
-
-
- function _processMergeData(data) {
- if (!data.length) {
- return false;
- }
-
- for each(var mergeItem in data) {
- var itemID = mergeItem.id;
- var dateModified = mergeItem.ref.getField('dateModified');
- // Local
- if (dateModified == mergeItem.left.getField('dateModified')) {
- Zotero.Sync.Storage.setSyncState(
- itemID, Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD
- );
- }
- // Remote
- else {
- Zotero.Sync.Storage.setSyncState(
- itemID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD
- );
- }
- }
- }
-}
diff --git a/chrome/content/zotero/xpcom/storage/storageEngine.js b/chrome/content/zotero/xpcom/storage/storageEngine.js
new file mode 100644
index 000000000..70c5dc217
--- /dev/null
+++ b/chrome/content/zotero/xpcom/storage/storageEngine.js
@@ -0,0 +1,307 @@
+/*
+ ***** BEGIN LICENSE BLOCK *****
+
+ Copyright © 2015 Center for History and New Media
+ George Mason University, Fairfax, Virginia, USA
+ http://zotero.org
+
+ This file is part of Zotero.
+
+ Zotero is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ Zotero is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with Zotero. If not, see <http://www.gnu.org/licenses/>.
+
+ ***** END LICENSE BLOCK *****
+*/
+
+
+if (!Zotero.Sync.Storage) {
+ Zotero.Sync.Storage = {};
+}
+
+/**
+ * An Engine manages file sync processes for a given library
+ *
+ * @param {Object} options
+ * @param {Zotero.Sync.APIClient} options.apiClient
+ * @param {Integer} options.libraryID
+ * @param {Function} [onError] - Function to run on error
+ * @param {Boolean} [stopOnError]
+ */
+Zotero.Sync.Storage.Engine = function (options) {
+ if (options.apiClient == undefined) {
+ throw new Error("options.apiClient not set");
+ }
+ if (options.libraryID == undefined) {
+ throw new Error("options.libraryID not set");
+ }
+
+ this.apiClient = options.apiClient;
+ this.background = options.background;
+ this.firstInSession = options.firstInSession;
+ this.lastFullFileCheck = options.lastFullFileCheck;
+ this.libraryID = options.libraryID;
+ this.library = Zotero.Libraries.get(options.libraryID);
+
+ this.local = Zotero.Sync.Storage.Local;
+ this.utils = Zotero.Sync.Storage.Utilities;
+ this.mode = this.local.getModeForLibrary(this.libraryID);
+ var modeClass = this.utils.getClassForMode(this.mode);
+ this.controller = new modeClass(options);
+ this.setStatus = options.setStatus || function () {};
+ this.onError = options.onError || function (e) {};
+ this.stopOnError = options.stopOnError || false;
+
+ this.queues = [];
+ ['download', 'upload'].forEach(function (type) {
+ this.queues[type] = new ConcurrentCaller({
+ id: `${this.libraryID}/${type}`,
+ numConcurrent: Zotero.Prefs.get(
+ 'sync.storage.max' + Zotero.Utilities.capitalize(type) + 's'
+ ),
+ onError: this.onError,
+ stopOnError: this.stopOnError,
+ logger: Zotero.debug
+ });
+ }.bind(this))
+
+ this.maxCheckAge = 10800; // maximum age in seconds for upload modification check (3 hours)
+}
+
+Zotero.Sync.Storage.Engine.prototype.start = Zotero.Promise.coroutine(function* () {
+ var libraryID = this.libraryID;
+ if (!Zotero.Prefs.get("sync.storage.enabled")) {
+ Zotero.debug("File sync is not enabled for " + this.library.name);
+ return false;
+ }
+
+ Zotero.debug("Starting file sync for " + this.library.name);
+
+ if (!this.controller.verified) {
+ Zotero.debug(`${this.mode} file sync is not active`);
+
+ throw new Error("Storage mode verification not implemented");
+
+ // TODO: Check server
+ }
+ if (this.controller.cacheCredentials) {
+ yield this.controller.cacheCredentials();
+ }
+
+ // Get library last-sync time for download-on-sync libraries.
+ var lastSyncTime = null;
+ var downloadAll = this.local.downloadOnSync(libraryID);
+ if (downloadAll) {
+ lastSyncTime = yield this.controller.getLastSyncTime(libraryID);
+ }
+
+ // Check for updated files to upload
+ if (!Zotero.Libraries.isFilesEditable(libraryID)) {
+ Zotero.debug("No file editing access -- skipping file modification check for "
+ + this.library.name);
+ }
+ // If this is a background sync, it's not the first sync of the session, the library has had
+ // at least one full check this session, and it's been less than maxCheckAge since the last
+ // full check of this library, check only files that were previously modified or opened
+ // recently
+ else if (this.background
+ && !this.firstInSession
+ && this.local.lastFullFileCheck[libraryID]
+ && (this.local.lastFullFileCheck[libraryID]
+ + (this.maxCheckAge * 1000)) > new Date().getTime()) {
+ let itemIDs = this.local.getFilesToCheck(libraryID, this.maxCheckAge);
+ yield this.local.checkForUpdatedFiles(libraryID, itemIDs);
+ }
+ // Otherwise check all files in library
+ else {
+ this.local.lastFullFileCheck[libraryID] = new Date().getTime();
+ yield this.local.checkForUpdatedFiles(libraryID);
+ }
+
+ yield this.local.resolveConflicts(libraryID);
+
+ var downloadForced = yield this.local.checkForForcedDownloads(libraryID);
+
+ // If we don't have any forced downloads, we can skip downloads if the last sync time hasn't
+ // changed or doesn't exist on the server (meaning there are no files)
+ if (downloadAll && !downloadForced) {
+ if (lastSyncTime) {
+ if (this.library.lastStorageSync == lastSyncTime) {
+ Zotero.debug("Last " + this.mode.toUpperCase() + " sync id hasn't changed for "
+ + this.library.name + " -- skipping file downloads");
+ downloadAll = false;
+ }
+ }
+ else {
+ Zotero.debug(`No last ${this.mode} sync time for ${this.library.name}`
+ + " -- skipping file downloads");
+ downloadAll = false;
+ }
+ }
+
+ // Get files to download
+ if (downloadAll || downloadForced) {
+ let itemIDs = yield this.local.getFilesToDownload(libraryID, !downloadAll);
+ if (itemIDs.length) {
+ Zotero.debug(itemIDs.length + " file" + (itemIDs.length == 1 ? '' : 's') + " to "
+ + "download for " + this.library.name);
+ for (let itemID of itemIDs) {
+ let item = yield Zotero.Items.getAsync(itemID);
+ yield this.queueItem(item);
+ }
+ }
+ else {
+ Zotero.debug("No files to download for " + this.library.name);
+ }
+ }
+
+ // Get files to upload
+ if (Zotero.Libraries.isFilesEditable(libraryID)) {
+ let itemIDs = yield this.local.getFilesToUpload(libraryID);
+ if (itemIDs.length) {
+ Zotero.debug(itemIDs.length + " file" + (itemIDs.length == 1 ? '' : 's') + " to "
+ + "upload for " + this.library.name);
+ for (let itemID of itemIDs) {
+ let item = yield Zotero.Items.getAsync(itemID, { noCache: true });
+ yield this.queueItem(item);
+ }
+ }
+ else {
+ Zotero.debug("No files to upload for " + this.library.name);
+ }
+ }
+ else {
+ Zotero.debug("No file editing access -- skipping file uploads for " + this.library.name);
+ }
+
+ var promises = {
+ download: this.queues.download.runAll(),
+ upload: this.queues.upload.runAll()
+ }
+
+ // Process the results
+ var changes = new Zotero.Sync.Storage.Result;
+ for (let type of ['download', 'upload']) {
+ let results = yield promises[type];
+
+ if (this.stopOnError) {
+ for (let p of results) {
+ if (p.isRejected()) {
+ let e = p.reason();
+ Zotero.debug(`File ${type} sync failed for ${this.library.name}`);
+ throw e;
+ }
+ }
+ }
+
+ Zotero.debug(`File ${type} sync finished for ${this.library.name}`);
+
+ changes.updateFromResults(results.filter(p => p.isFulfilled()).map(p => p.value()));
+ }
+
+ // If files were uploaded, update the remote last-sync time
+ if (changes.remoteChanges) {
+ lastSyncTime = yield this.controller.setLastSyncTime(libraryID);
+ if (!lastSyncTime) {
+ throw new Error("Last sync time not set after sync");
+ }
+ }
+
+ // If there's a remote last-sync time from either the check before downloads or when it
+ // was changed after uploads, store that locally so we know we can skip download checks
+ // next time
+ if (lastSyncTime) {
+ this.library.lastStorageSync = lastSyncTime;
+ yield this.library.saveTx();
+ }
+
+ // If WebDAV sync, purge deleted and orphaned files
+ if (this.mode == 'webdav') {
+ try {
+ yield this.controller.purgeDeletedStorageFiles(libraryID);
+ yield this.controller.purgeOrphanedStorageFiles(libraryID);
+ }
+ catch (e) {
+ Zotero.logError(e);
+ }
+ }
+
+ if (!changes.localChanges) {
+ Zotero.debug("No local changes made during file sync");
+ }
+
+ Zotero.debug("Done with file sync for " + this.library.name);
+
+ return changes;
+})
+
+
+Zotero.Sync.Storage.Engine.prototype.stop = function () {
+ for (let type in this.queues) {
+ this.queues[type].stop();
+ }
+}
+
+Zotero.Sync.Storage.Engine.prototype.queueItem = Zotero.Promise.coroutine(function* (item) {
+ switch (yield this.local.getSyncState(item.id)) {
+ case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD:
+ case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD:
+ var type = 'download';
+ var onStart = Zotero.Promise.method(function (request) {
+ return this.controller.downloadFile(request);
+ }.bind(this));
+ break;
+
+ case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD:
+ case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD:
+ var type = 'upload';
+ var onStart = Zotero.Promise.method(function (request) {
+ return this.controller.uploadFile(request);
+ }.bind(this));
+ break;
+
+ case false:
+ Zotero.debug("Sync state for item " + item.id + " not found", 2);
+ return;
+
+ default:
+ throw new Error("Invalid sync state " + (yield this.local.getSyncState(item.id)));
+ }
+
+ var request = new Zotero.Sync.Storage.Request({
+ type,
+ libraryID: this.libraryID,
+ name: item.libraryKey,
+ onStart,
+ onProgress: this.onProgress
+ });
+ if (type == 'upload') {
+ try {
+ request.setMaxSize(yield Zotero.Attachments.getTotalFileSize(item));
+ }
+ // If this fails, ignore it, though we might fail later
+ catch (e) {
+ // But if the file doesn't exist yet, don't try to upload it
+ //
+ // This isn't a perfect test, because the file could still be in the process of being
+ // downloaded (e.g., from the web). It'd be better to download files to a temp
+ // directory and move them into place.
+ if (!(yield item.getFilePathAsync())) {
+ Zotero.debug("File " + item.libraryKey + " not yet available to upload -- skipping");
+ return;
+ }
+
+ Zotero.logError(e);
+ }
+ }
+ this.queues[type].add(request.start.bind(request));
+})
diff --git a/chrome/content/zotero/xpcom/storage/storageLocal.js b/chrome/content/zotero/xpcom/storage/storageLocal.js
new file mode 100644
index 000000000..ea8a13c0e
--- /dev/null
+++ b/chrome/content/zotero/xpcom/storage/storageLocal.js
@@ -0,0 +1,1088 @@
+Zotero.Sync.Storage.Local = {
+ lastFullFileCheck: {},
+ uploadCheckFiles: [],
+
+ getClassForLibrary: function (libraryID) {
+ return Zotero.Sync.Storage.Utilities.getClassForMode(this.getModeForLibrary(libraryID));
+ },
+
+ getModeForLibrary: function (libraryID) {
+ var libraryType = Zotero.Libraries.getType(libraryID);
+ switch (libraryType) {
+ case 'user':
+ case 'publications':
+ return Zotero.Prefs.get("sync.storage.protocol") == 'webdav' ? 'webdav' : 'zfs';
+
+ case 'group':
+ return 'zfs';
+
+ default:
+ throw new Error(`Unexpected library type '${libraryType}'`);
+ }
+ },
+
+ setModeForLibrary: function (libraryID, mode) {
+ var libraryType = Zotero.Libraries.getType(libraryID);
+
+ if (libraryType != 'user') {
+ throw new Error(`Cannot set storage mode for ${libraryType} library`);
+ }
+
+ switch (mode) {
+ case 'webdav':
+ case 'zfs':
+ Zotero.Prefs.set("sync.storage.protocol", mode);
+ break;
+
+ default:
+ throw new Error(`Unexpected storage mode '${mode}'`);
+ }
+ },
+
+ /**
+ * Check or enable download-as-needed mode
+ *
+ * @param {Integer} [libraryID]
+ * @param {Boolean} [enable] - If true, enable download-as-needed mode for the given library
+ * @return {Boolean|undefined} - If 'enable' isn't set to true, return true if
+ * download-as-needed mode enabled and false if not
+ */
+ downloadAsNeeded: function (libraryID, enable) {
+ var pref = this._getDownloadPrefFromLibrary(libraryID);
+ var val = 'on-demand';
+ if (enable) {
+ Zotero.Prefs.set(pref, val);
+ return;
+ }
+ return Zotero.Prefs.get(pref) == val;
+ },
+
+ /**
+ * Check or enable download-on-sync mode
+ *
+ * @param {Integer} [libraryID]
+ * @param {Boolean} [enable] - If true, enable download-on-demand mode for the given library
+ * @return {Boolean|undefined} - If 'enable' isn't set to true, return true if
+ * download-as-needed mode enabled and false if not
+ */
+ downloadOnSync: function (libraryID, enable) {
+ var pref = this._getDownloadPrefFromLibrary(libraryID);
+ var val = 'on-demand';
+ if (enable) {
+ Zotero.Prefs.set(pref, val);
+ return;
+ }
+ return Zotero.Prefs.get(pref) == val;
+ },
+
+ _getDownloadPrefFromLibrary: function (libraryID) {
+ if (libraryID == Zotero.Libraries.userLibraryID) {
+ return 'sync.storage.downloadMode.personal';
+ }
+ // TODO: Library-specific settings
+
+ // Group library
+ return 'sync.storage.downloadMode.groups';
+ },
+
+ /**
+ * Get files to check for local modifications for uploading
+ *
+ * This includes files previously modified or opened externally via Zotero within maxCheckAge
+ */
+ getFilesToCheck: Zotero.Promise.coroutine(function* (libraryID, maxCheckAge) {
+ var minTime = new Date().getTime() - (maxCheckAge * 1000);
+
+ // Get files modified and synced since maxCheckAge
+ var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
+ + "WHERE libraryID=? AND linkMode IN (?,?) AND syncState IN (?) AND "
+ + "storageModTime>=?";
+ var params = [
+ libraryID,
+ Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
+ Zotero.Attachments.LINK_MODE_IMPORTED_URL,
+ Zotero.Sync.Storage.SYNC_STATE_IN_SYNC,
+ minTime
+ ];
+ var itemIDs = yield Zotero.DB.columnQueryAsync(sql, params);
+
+ // Get files opened since maxCheckAge
+ itemIDs = itemIDs.concat(
+ this.uploadCheckFiles.filter(x => x.timestamp >= minTime).map(x => x.itemID)
+ );
+
+ return Zotero.Utilities.arrayUnique(itemIDs);
+ }),
+
+
+ /**
+ * Scans local files and marks any that have changed for uploading
+ * and any that are missing for downloading
+ *
+ * @param {Integer} libraryID
+ * @param {Integer[]} [itemIDs]
+ * @param {Object} [itemModTimes] Item mod times indexed by item ids;
+ * items with stored mod times
+ * that differ from the provided
+ * time but file mod times
+ * matching the stored time will
+ * be marked for download
+ * @return {Promise} Promise resolving to TRUE if any items changed state,
+ * FALSE otherwise
+ */
+ checkForUpdatedFiles: Zotero.Promise.coroutine(function* (libraryID, itemIDs, itemModTimes) {
+ var libraryName = Zotero.Libraries.getName(libraryID);
+ var msg = "Checking for locally changed attachment files in " + libraryName;
+
+ var memmgr = Components.classes["@mozilla.org/memory-reporter-manager;1"]
+ .getService(Components.interfaces.nsIMemoryReporterManager);
+ memmgr.init();
+ //Zotero.debug("Memory usage: " + memmgr.resident);
+
+ if (itemIDs) {
+ if (!itemIDs.length) {
+ Zotero.debug("No files to check for local changes");
+ return false;
+ }
+ }
+ if (itemModTimes) {
+ if (!Object.keys(itemModTimes).length) {
+ return false;
+ }
+ msg += " in download-marking mode";
+ }
+
+ Zotero.debug(msg);
+
+ var changed = false;
+
+ if (!itemIDs) {
+ itemIDs = Object.keys(itemModTimes ? itemModTimes : {});
+ }
+
+ // Can only handle a certain number of bound parameters at a time
+ var numIDs = itemIDs.length;
+ var maxIDs = Zotero.DB.MAX_BOUND_PARAMETERS - 10;
+ var done = 0;
+ var rows = [];
+
+ do {
+ let chunk = itemIDs.splice(0, maxIDs);
+ let sql = "SELECT itemID, linkMode, path, storageModTime, storageHash, syncState "
+ + "FROM itemAttachments JOIN items USING (itemID) "
+ + "WHERE linkMode IN (?,?) AND syncState IN (?,?)";
+ let params = [
+ Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
+ Zotero.Attachments.LINK_MODE_IMPORTED_URL,
+ Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD,
+ Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ ];
+ if (libraryID !== false) {
+ sql += " AND libraryID=?";
+ params.push(libraryID);
+ }
+ if (chunk.length) {
+ sql += " AND itemID IN (" + chunk.map(() => '?').join() + ")";
+ params = params.concat(chunk);
+ }
+ let chunkRows = yield Zotero.DB.queryAsync(sql, params);
+ if (chunkRows) {
+ rows = rows.concat(chunkRows);
+ }
+ done += chunk.length;
+ }
+ while (done < numIDs);
+
+ // If no files, or everything is already marked for download,
+ // we don't need to do anything
+ if (!rows.length) {
+ Zotero.debug("No in-sync or to-upload files found in " + libraryName);
+ return false;
+ }
+
+ // Index attachment data by item id
+ itemIDs = [];
+ var attachmentData = {};
+ for (let row of rows) {
+ var id = row.itemID;
+ itemIDs.push(id);
+ attachmentData[id] = {
+ linkMode: row.linkMode,
+ path: row.path,
+ mtime: row.storageModTime,
+ hash: row.storageHash,
+ state: row.syncState
+ };
+ }
+ rows = null;
+
+ var t = new Date();
+ var items = yield Zotero.Items.getAsync(itemIDs, { noCache: true });
+ var numItems = items.length;
+ var updatedStates = {};
+
+ //Zotero.debug("Memory usage: " + memmgr.resident);
+
+ var changed = false;
+ for (let i = 0; i < items.length; i++) {
+ let item = items[i];
+ // TODO: Catch error?
+ let state = yield this._checkForUpdatedFile(item, attachmentData[item.id]);
+ if (state !== false) {
+ yield Zotero.Sync.Storage.Local.setSyncState(item.id, state);
+ changed = true;
+ }
+ }
+
+ if (!items.length) {
+ Zotero.debug("No synced files have changed locally");
+ }
+
+ Zotero.debug(`Checked ${numItems} files in ${libraryName} in ` + (new Date() - t) + " ms");
+
+ return changed;
+ }),
+
+
+ _checkForUpdatedFile: Zotero.Promise.coroutine(function* (item, attachmentData, remoteModTime) {
+ var lk = item.libraryKey;
+ Zotero.debug("Checking attachment file for item " + lk, 4);
+
+ var path = item.getFilePath();
+ if (!path) {
+ Zotero.debug("Marking pathless attachment " + lk + " as in-sync");
+ return Zotero.Sync.Storage.SYNC_STATE_IN_SYNC;
+ }
+ var fileName = OS.Path.basename(path);
+ var file;
+
+ try {
+ file = yield OS.File.open(path);
+ let info = yield file.stat();
+ //Zotero.debug("Memory usage: " + memmgr.resident);
+
+ let fmtime = info.lastModificationDate.getTime();
+ //Zotero.debug("File modification time for item " + lk + " is " + fmtime);
+
+ if (fmtime < 0) {
+ Zotero.debug("File mod time " + fmtime + " is less than 0 -- interpreting as 0", 2);
+ fmtime = 0;
+ }
+
+ // If file is already marked for upload, skip check. Even if the file was changed
+ // both locally and remotely, conflicts are checked at upload time, so we don't need
+ // to worry about it here.
+ if ((yield this.getSyncState(item.id)) == Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD) {
+ Zotero.debug("File is already marked for upload");
+ return false;
+ }
+
+ //Zotero.debug("Stored mtime is " + attachmentData.mtime);
+ //Zotero.debug("File mtime is " + fmtime);
+
+ //BAIL AFTER DOWNLOAD MARKING MODE, OR CHECK LOCAL?
+ let mtime = attachmentData ? attachmentData.mtime : false;
+
+ // Download-marking mode
+ if (remoteModTime) {
+ Zotero.debug(`Remote mod time for item ${lk} is ${remoteModTime}`);
+
+ // Ignore attachments whose stored mod times haven't changed
+ mtime = mtime !== false ? mtime : (yield this.getSyncedModificationTime(item.id));
+ if (mtime == remoteModTime) {
+ Zotero.debug(`Synced mod time (${mtime}) hasn't changed for item ${lk}`);
+ return false;
+ }
+
+ Zotero.debug(`Marking attachment ${lk} for download (stored mtime: ${mtime})`);
+ // DEBUG: Always set here, or allow further steps?
+ return Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD;
+ }
+
+ var same = !this.checkFileModTime(item, fmtime, mtime);
+ if (same) {
+ Zotero.debug("File has not changed");
+ return false;
+ }
+
+ // If file hash matches stored hash, only the mod time changed, so skip
+ let fileHash = yield Zotero.Utilities.Internal.md5Async(file);
+
+ var hash = attachmentData ? attachmentData.hash : (yield this.getSyncedHash(item.id));
+ if (hash && hash == fileHash) {
+ // We have to close the file before modifying it from the main
+ // thread (at least on Windows, where assigning lastModifiedTime
+ // throws an NS_ERROR_FILE_IS_LOCKED otherwise)
+ yield file.close();
+
+ Zotero.debug("Mod time didn't match (" + fmtime + " != " + mtime + ") "
+ + "but hash did for " + fileName + " for item " + lk
+ + " -- updating file mod time");
+ try {
+ yield OS.File.setDates(path, null, mtime);
+ }
+ catch (e) {
+ Zotero.File.checkPathAccessError(e, path, 'update');
+ }
+ return false;
+ }
+
+ // Mark file for upload
+ Zotero.debug("Marking attachment " + lk + " as changed "
+ + "(" + mtime + " != " + fmtime + ")");
+ return Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD;
+ }
+ catch (e) {
+ if (e instanceof OS.File.Error &&
+ (e.becauseNoSuchFile
+ // This can happen if a path is too long on Windows,
+ // e.g. a file is being accessed on a VM through a share
+ // (and probably in other cases).
+ || (e.winLastError && e.winLastError == 3)
+ // Handle long filenames on OS X/Linux
+ || (e.unixErrno && e.unixErrno == 63))) {
+ Zotero.debug("Marking attachment " + lk + " as missing");
+ return Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD;
+ }
+
+ if (e instanceof OS.File.Error) {
+ if (e.becauseClosed) {
+ Zotero.debug("File was closed", 2);
+ }
+ Zotero.debug(e);
+ Zotero.debug(e.toString());
+ throw new Error(`Error for operation '${e.operation}' for ${path}`);
+ }
+
+ throw e;
+ }
+ finally {
+ if (file) {
+ //Zotero.debug("Closing file for item " + lk);
+ file.close();
+ }
+ }
+ }),
+
+ /**
+ *
+ * @param {Zotero.Item} item
+ * @param {Integer} fmtime - File modification time in milliseconds
+ * @param {Integer} mtime - Remote modification time in milliseconds
+ * @return {Boolean} - True if file modification time differs from remote mod time,
+ * false otherwise
+ */
+ checkFileModTime(item, fmtime, mtime) {
+ var libraryKey = item.libraryKey;
+
+ if (fmtime == mtime) {
+ Zotero.debug(`Mod time for ${libraryKey} matches remote file -- skipping`);
+ }
+ // Compare floored timestamps for filesystems that don't support millisecond
+ // precision (e.g., HFS+)
+ else if (Math.floor(mtime / 1000) * 1000 == fmtime
+ || Math.floor(fmtime / 1000) * 1000 == mtime) {
+ Zotero.debug(`File mod times for ${libraryKey} are within one-second precision `
+ + "(" + fmtime + " ≅ " + mtime + ") -- skipping");
+ }
+ // Allow timestamp to be exactly one hour off to get around time zone issues
+ // -- there may be a proper way to fix this
+ else if (Math.abs(fmtime - mtime) == 3600000
+ // And check with one-second precision as well
+ || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
+ || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
+ Zotero.debug(`File mod time (${fmtime}) for {$libraryKey} is exactly one hour off `
+ + `remote file (${mtime}) -- assuming time zone issue and skipping`);
+ }
+ else {
+ return true;
+ }
+
+ return false;
+ },
+
+ checkForForcedDownloads: Zotero.Promise.coroutine(function* (libraryID) {
+ // Forced downloads happen even in on-demand mode
+ var sql = "SELECT COUNT(*) FROM items JOIN itemAttachments USING (itemID) "
+ + "WHERE libraryID=? AND syncState=?";
+ return !!(yield Zotero.DB.valueQueryAsync(
+ sql, [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD]
+ ));
+ }),
+
+
+ /**
+ * Get files marked as ready to download
+ *
+ * @param {Integer} libraryID
+ * @return {Promise<Number[]>} - Promise for an array of attachment itemIDs
+ */
+ getFilesToDownload: function (libraryID, forcedOnly) {
+ var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
+ + "WHERE libraryID=? AND syncState IN (?";
+ var params = [libraryID, Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD];
+ if (!forcedOnly) {
+ sql += ",?";
+ params.push(Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD);
+ }
+ sql += ") "
+ // Skip attachments with empty path, which can't be saved, and files with .zotero*
+ // paths, which have somehow ended up in some users' libraries
+ + "AND path!='' AND path NOT LIKE 'storage:.zotero%'";
+ return Zotero.DB.columnQueryAsync(sql, params);
+ },
+
+
+ /**
+ * Get files marked as ready to upload
+ *
+ * @param {Integer} libraryID
+ * @return {Promise<Number[]>} - Promise for an array of attachment itemIDs
+ */
+ getFilesToUpload: function (libraryID) {
+ var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
+ + "WHERE libraryID=? AND syncState IN (?,?) AND linkMode IN (?,?)";
+ var params = [
+ libraryID,
+ Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD,
+ Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD,
+ Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
+ Zotero.Attachments.LINK_MODE_IMPORTED_URL
+ ];
+ return Zotero.DB.columnQueryAsync(sql, params);
+ },
+
+
+ /**
+ * @param {Integer} libraryID
+ * @return {Promise<String[]>} - Promise for an array of item keys
+ */
+ getDeletedFiles: function (libraryID) {
+ var sql = "SELECT key FROM storageDeleteLog WHERE libraryID=?";
+ return Zotero.DB.columnQueryAsync(sql, libraryID);
+ },
+
+
+ /**
+ * @param {Integer} itemID
+ */
+ getSyncState: function (itemID) {
+ var sql = "SELECT syncState FROM itemAttachments WHERE itemID=?";
+ return Zotero.DB.valueQueryAsync(sql, itemID);
+ },
+
+
+ /**
+ * @param {Integer} itemID
+ * @param {Integer} syncState Constant from Zotero.Sync.Storage
+ */
+ setSyncState: Zotero.Promise.method(function (itemID, syncState) {
+ switch (syncState) {
+ case Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD:
+ case Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD:
+ case Zotero.Sync.Storage.SYNC_STATE_IN_SYNC:
+ case Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD:
+ case Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD:
+ case Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT:
+ break;
+
+ default:
+ throw new Error("Invalid sync state " + syncState);
+ }
+
+ var sql = "UPDATE itemAttachments SET syncState=? WHERE itemID=?";
+ return Zotero.DB.valueQueryAsync(sql, [syncState, itemID]);
+ }),
+
+
+ /**
+ * @param {Integer} itemID
+ * @return {Integer|NULL} Mod time as timestamp in ms,
+ * or NULL if never synced
+ */
+ getSyncedModificationTime: Zotero.Promise.coroutine(function* (itemID) {
+ var sql = "SELECT storageModTime FROM itemAttachments WHERE itemID=?";
+ var mtime = yield Zotero.DB.valueQueryAsync(sql, itemID);
+ if (mtime === false) {
+ throw new Error("Item " + itemID + " not found")
+ }
+ return mtime;
+ }),
+
+
+ /**
+ * @param {Integer} itemID
+ * @param {Integer} mtime - File modification time as timestamp in ms
+ * @param {Boolean} [updateItem=FALSE] - Update clientDateModified field of attachment item
+ */
+ setSyncedModificationTime: Zotero.Promise.coroutine(function* (itemID, mtime, updateItem) {
+ if (mtime < 0) {
+ Components.utils.reportError("Invalid file mod time " + mtime
+ + " in Zotero.Storage.setSyncedModificationTime()");
+ mtime = 0;
+ }
+
+ Zotero.DB.requireTransaction();
+
+ var sql = "UPDATE itemAttachments SET storageModTime=? WHERE itemID=?";
+ yield Zotero.DB.queryAsync(sql, [mtime, itemID]);
+
+ if (updateItem) {
+ // Update item date modified so the new mod time will be synced
+ let sql = "UPDATE items SET clientDateModified=? WHERE itemID=?";
+ yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]);
+ }
+ }),
+
+
+ /**
+ * @param {Integer} itemID
+ * @return {Promise<String|null|false>} - File hash, null if never synced, if false if
+ * file doesn't exist
+ */
+ getSyncedHash: Zotero.Promise.coroutine(function* (itemID) {
+ var sql = "SELECT storageHash FROM itemAttachments WHERE itemID=?";
+ var hash = yield Zotero.DB.valueQueryAsync(sql, itemID);
+ if (hash === false) {
+ throw new Error("Item " + itemID + " not found");
+ }
+ return hash;
+ }),
+
+
+ /**
+ * @param {Integer} itemID
+ * @param {String} hash File hash
+ * @param {Boolean} [updateItem=FALSE] Update dateModified field of
+ * attachment item
+ */
+ setSyncedHash: Zotero.Promise.coroutine(function* (itemID, hash, updateItem) {
+ if (hash !== null && hash.length != 32) {
+ throw new Error("Invalid file hash '" + hash + "'");
+ }
+
+ Zotero.DB.requireTransaction();
+
+ var sql = "UPDATE itemAttachments SET storageHash=? WHERE itemID=?";
+ yield Zotero.DB.queryAsync(sql, [hash, itemID]);
+
+ if (updateItem) {
+ // Update item date modified so the new mod time will be synced
+ var sql = "UPDATE items SET clientDateModified=? WHERE itemID=?";
+ yield Zotero.DB.queryAsync(sql, [Zotero.DB.transactionDateTime, itemID]);
+ }
+ }),
+
+
+ /**
+ * Extract a downloaded file and update the database metadata
+ *
+ * @param {Zotero.Item} data.item
+ * @param {Integer} data.mtime
+ * @param {String} data.md5
+ * @param {Boolean} data.compressed
+ * @return {Promise}
+ */
+ processDownload: Zotero.Promise.coroutine(function* (data) {
+ if (!data) {
+ throw new Error("'data' not set");
+ }
+ if (!data.item) {
+ throw new Error("'data.item' not set");
+ }
+ if (!data.mtime) {
+ throw new Error("'data.mtime' not set");
+ }
+ if (data.mtime != parseInt(data.mtime)) {
+ throw new Error("Invalid mod time '" + data.mtime + "'");
+ }
+ if (!data.compressed && !data.md5) {
+ throw new Error("'data.md5' is required if 'data.compressed'");
+ }
+
+ var item = data.item;
+ var mtime = parseInt(data.mtime);
+ var md5 = data.md5;
+
+ // TODO: Test file hash
+
+ if (data.compressed) {
+ var newPath = yield this._processZipDownload(item);
+ }
+ else {
+ var newPath = yield this._processSingleFileDownload(item);
+ }
+
+ // If newPath is set, the file was renamed, so set item filename to that
+ // and mark for updated
+ var path = yield item.getFilePathAsync();
+ if (newPath && path != newPath) {
+ // If library isn't editable but filename was changed, update
+ // database without updating the item's mod time, which would result
+ // in a library access error
+ if (!Zotero.Items.isEditable(item)) {
+ Zotero.debug("File renamed without library access -- "
+ + "updating itemAttachments path", 3);
+ yield item.relinkAttachmentFile(newPath, true);
+ }
+ else {
+ yield item.relinkAttachmentFile(newPath);
+ }
+
+ path = newPath;
+ }
+
+ if (!path) {
+ // This can happen if an HTML snapshot filename was changed and synced
+ // elsewhere but the renamed file wasn't synced, so the ZIP doesn't
+ // contain a file with the known name
+ Components.utils.reportError("File '" + item.attachmentFilename
+ + "' not found after processing download " + item.libraryKey);
+ return new Zotero.Sync.Storage.Result({
+ localChanges: false
+ });
+ }
+
+ try {
+ // If hash not provided (e.g., WebDAV), calculate it now
+ if (!md5) {
+ md5 = yield item.attachmentHash;
+ }
+ }
+ catch (e) {
+ Zotero.File.checkFileAccessError(e, path, 'update');
+ }
+
+ // Set the file mtime to the time from the server
+ yield OS.File.setDates(path, null, new Date(parseInt(mtime)));
+
+ yield Zotero.DB.executeTransaction(function* () {
+ yield this.setSyncedHash(item.id, md5);
+ yield this.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
+ yield this.setSyncedModificationTime(item.id, mtime);
+ }.bind(this));
+
+ return new Zotero.Sync.Storage.Result({
+ localChanges: true
+ });
+ }),
+
+
+ _processSingleFileDownload: Zotero.Promise.coroutine(function* (item) {
+ var tempFilePath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.tmp');
+
+ if (!(yield OS.File.exists(tempFilePath))) {
+ Zotero.debug(tempFilePath, 1);
+ throw new Error("Downloaded file not found");
+ }
+
+ var parentDirPath = Zotero.Attachments.getStorageDirectory(item).path;
+ if (!(yield OS.File.exists(parentDirPath))) {
+ yield Zotero.Attachments.createDirectoryForItem(item);
+ }
+
+ yield this._deleteExistingAttachmentFiles(item);
+
+ var path = item.getFilePath();
+ if (!path) {
+ throw new Error("Empty path for item " + item.key);
+ }
+ // Don't save Windows aliases
+ if (path.endsWith('.lnk')) {
+ return false;
+ }
+
+ var fileName = OS.Path.basename(path);
+ var renamed = false;
+
+ // Make sure the new filename is valid, in case an invalid character made it over
+ // (e.g., from before we checked for them)
+ var filteredName = Zotero.File.getValidFileName(fileName);
+ if (filteredName != fileName) {
+ Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
+ fileName = filteredName;
+ path = OS.Path.dirname(path, fileName);
+ renamed = true;
+ }
+
+ Zotero.debug("Moving download file " + OS.Path.basename(tempFilePath)
+ + " into attachment directory as '" + fileName + "'");
+ try {
+ var finalFileName = Zotero.File.createShortened(
+ path, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644
+ );
+ }
+ catch (e) {
+ Zotero.File.checkFileAccessError(e, path, 'create');
+ }
+
+ if (finalFileName != fileName) {
+ Zotero.debug("Changed filename '" + fileName + "' to '" + finalFileName + "'");
+
+ fileName = finalFileName;
+ path = OS.Path.dirname(path, fileName);
+
+ // Abort if Windows path limitation would cause filenames to be overly truncated
+ if (Zotero.isWin && fileName.length < 40) {
+ try {
+ yield OS.File.remove(path);
+ }
+ catch (e) {}
+ // TODO: localize
+ var msg = "Due to a Windows path length limitation, your Zotero data directory "
+ + "is too deep in the filesystem for syncing to work reliably. "
+ + "Please relocate your Zotero data to a higher directory.";
+ Zotero.debug(msg, 1);
+ throw new Error(msg);
+ }
+
+ renamed = true;
+ }
+
+ try {
+ yield OS.File.move(tempFilePath, path);
+ }
+ catch (e) {
+ try {
+ yield OS.File.remove(tempFilePath);
+ }
+ catch (e) {}
+
+ Zotero.File.checkFileAccessError(e, path, 'create');
+ }
+
+ // processDownload() needs to know that we're renaming the file
+ return renamed ? path : null;
+ }),
+
+
+ _processZipDownload: Zotero.Promise.coroutine(function* (item) {
+ var zipFile = Zotero.getTempDirectory();
+ zipFile.append(item.key + '.tmp');
+
+ if (!zipFile.exists()) {
+ Zotero.debug(zipFile.path);
+ throw new Error(`Downloaded ZIP file not found for item ${item.libraryKey}`);
+ }
+
+ var zipReader = Components.classes["@mozilla.org/libjar/zip-reader;1"].
+ createInstance(Components.interfaces.nsIZipReader);
+ try {
+ zipReader.open(zipFile);
+ zipReader.test(null);
+
+ Zotero.debug("ZIP file is OK");
+ }
+ catch (e) {
+ Zotero.debug(zipFile.leafName + " is not a valid ZIP file", 2);
+ zipReader.close();
+
+ try {
+ zipFile.remove(false);
+ }
+ catch (e) {
+ Zotero.File.checkFileAccessError(e, zipFile, 'delete');
+ }
+
+ // TODO: Remove prop file to trigger reuploading, in case it was an upload error?
+
+ return false;
+ }
+
+ var parentDir = Zotero.Attachments.getStorageDirectory(item);
+ if (!parentDir.exists()) {
+ yield Zotero.Attachments.createDirectoryForItem(item);
+ }
+
+ try {
+ yield this._deleteExistingAttachmentFiles(item);
+ }
+ catch (e) {
+ zipReader.close();
+ throw (e);
+ }
+
+ var returnFile = null;
+ var count = 0;
+
+ var itemFileName = item.attachmentFilename;
+
+ var entries = zipReader.findEntries(null);
+ while (entries.hasMore()) {
+ count++;
+ var entryName = entries.getNext();
+ var b64re = /%ZB64$/;
+ if (entryName.match(b64re)) {
+ var fileName = Zotero.Utilities.Internal.Base64.decode(
+ entryName.replace(b64re, '')
+ );
+ }
+ else {
+ var fileName = entryName;
+ }
+
+ if (fileName.startsWith('.zotero')) {
+ Zotero.debug("Skipping " + fileName);
+ continue;
+ }
+
+ Zotero.debug("Extracting " + fileName);
+
+ var primaryFile = false;
+ var filtered = false;
+ var renamed = false;
+
+ // Make sure the new filename is valid, in case an invalid character
+ // somehow make it into the ZIP (e.g., from before we checked for them)
+ //
+ // Do this before trying to use the relative descriptor, since otherwise
+ // it might fail silently and select the parent directory
+ var filteredName = Zotero.File.getValidFileName(fileName);
+ if (filteredName != fileName) {
+ Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
+ fileName = filteredName;
+ filtered = true;
+ }
+
+ // Name in ZIP is a relative descriptor, so file has to be reconstructed
+ // using setRelativeDescriptor()
+ var destFile = parentDir.clone();
+ destFile.QueryInterface(Components.interfaces.nsILocalFile);
+ destFile.setRelativeDescriptor(parentDir, fileName);
+
+ fileName = destFile.leafName;
+
+ // If only one file in zip and it doesn't match the known filename,
+ // take our chances and use that name
+ if (count == 1 && !entries.hasMore() && itemFileName) {
+ // May not be necessary, but let's be safe
+ itemFileName = Zotero.File.getValidFileName(itemFileName);
+ if (itemFileName != fileName) {
+ Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'", 2);
+ Components.utils.reportError("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'");
+ fileName = itemFileName;
+ destFile.leafName = fileName;
+ renamed = true;
+ }
+ }
+
+ var primaryFile = itemFileName == fileName;
+ if (primaryFile && filtered) {
+ renamed = true;
+ }
+
+ if (destFile.exists()) {
+ var msg = "ZIP entry '" + fileName + "' " + "already exists";
+ Zotero.debug(msg, 2);
+ Components.utils.reportError(msg + " in " + funcName);
+ Zotero.debug(destFile.path);
+ continue;
+ }
+
+ try {
+ Zotero.File.createShortened(destFile, Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
+ }
+ catch (e) {
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e);
+
+ zipReader.close();
+
+ Zotero.File.checkFileAccessError(e, destFile, 'create');
+ }
+
+ if (destFile.leafName != fileName) {
+ Zotero.debug("Changed filename '" + fileName + "' to '" + destFile.leafName + "'");
+
+ // Abort if Windows path limitation would cause filenames to be overly truncated
+ if (Zotero.isWin && destFile.leafName.length < 40) {
+ try {
+ destFile.remove(false);
+ }
+ catch (e) {}
+ zipReader.close();
+ // TODO: localize
+ var msg = "Due to a Windows path length limitation, your Zotero data directory "
+ + "is too deep in the filesystem for syncing to work reliably. "
+ + "Please relocate your Zotero data to a higher directory.";
+ Zotero.debug(msg, 1);
+ throw new Error(msg);
+ }
+
+ if (primaryFile) {
+ renamed = true;
+ }
+ }
+
+ try {
+ zipReader.extract(entryName, destFile);
+ }
+ catch (e) {
+ try {
+ destFile.remove(false);
+ }
+ catch (e) {}
+
+ // For advertising junk files, ignore a bug on Windows where
+ // destFile.create() works but zipReader.extract() doesn't
+ // when the path length is close to 255.
+ if (destFile.leafName.match(/[a-zA-Z0-9+=]{130,}/)) {
+ var msg = "Ignoring error extracting '" + destFile.path + "'";
+ Zotero.debug(msg, 2);
+ Zotero.debug(e, 2);
+ Components.utils.reportError(msg + " in " + funcName);
+ continue;
+ }
+
+ zipReader.close();
+
+ Zotero.File.checkFileAccessError(e, destFile, 'create');
+ }
+
+ destFile.permissions = 0644;
+
+ // If we're renaming the main file, processDownload() needs to know
+ if (renamed) {
+ returnFile = destFile.path;
+ }
+ }
+ zipReader.close();
+ zipFile.remove(false);
+
+ return returnFile;
+ }),
+
+
+ _deleteExistingAttachmentFiles: Zotero.Promise.coroutine(function* (item) {
+ var parentDir = Zotero.Attachments.getStorageDirectory(item).path;
+ return this._deleteExistingFilesInDirectory(parentDir);
+ }),
+
+
+ _deleteExistingFilesInDirectory: Zotero.Promise.coroutine(function* (dir) {
+ var dirsToDelete = [];
+ var iterator = new OS.File.DirectoryIterator(dir);
+ try {
+ yield iterator.forEach(function (entry) {
+ return Zotero.Promise.coroutine(function* () {
+ if (entry.isDir) {
+ dirsToDelete.push(entry.path);
+ }
+ else {
+ try {
+ yield OS.File.remove(entry.path);
+ }
+ catch (e) {
+ Zotero.File.checkFileAccessError(e, entry.path, 'delete');
+ }
+ }
+ })();
+ });
+ }
+ finally {
+ iterator.close();
+ }
+ for (let path of dirsToDelete) {
+ yield this._deleteExistingFilesInDirectory(path);
+ }
+ }),
+
+
+ /**
+ * @return {Promise<Object[]>} - A promise for an array of conflict objects
+ */
+ getConflicts: Zotero.Promise.coroutine(function* (libraryID) {
+ var sql = "SELECT itemID, version FROM items JOIN itemAttachments USING (itemID) "
+ + "WHERE libraryID=? AND syncState=?";
+ var rows = yield Zotero.DB.queryAsync(
+ sql,
+ [
+ { int: libraryID },
+ Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ ]
+ );
+ var keyVersionPairs = rows.map(function (row) {
+ var { libraryID, key } = Zotero.Items.getLibraryAndKeyFromID(row.itemID);
+ return [key, row.version];
+ });
+ var cacheObjects = yield Zotero.Sync.Data.Local.getCacheObjects(
+ 'item', libraryID, keyVersionPairs
+ );
+ if (!cacheObjects.length) return [];
+
+ var cacheObjectsByKey = {};
+ cacheObjects.forEach(obj => cacheObjectsByKey[obj.key] = obj);
+
+ var items = [];
+ var localItems = yield Zotero.Items.getAsync(rows.map(row => row.itemID));
+ for (let localItem of localItems) {
+ // Use the mtime for the dateModified field, since that's all that's shown in the
+ // CR window at the moment
+ let localItemJSON = yield localItem.toJSON();
+ localItemJSON.dateModified = Zotero.Date.dateToISO(
+ new Date(yield localItem.attachmentModificationTime)
+ );
+
+ let remoteItemJSON = cacheObjectsByKey[localItem.key];
+ if (!remoteItemJSON) {
+ Zotero.logError("Cached object not found for item " + localItem.libraryKey);
+ continue;
+ }
+ remoteItemJSON = remoteItemJSON.data;
+ remoteItemJSON.dateModified = Zotero.Date.dateToISO(new Date(remoteItemJSON.mtime));
+ items.push({
+ left: localItemJSON,
+ right: remoteItemJSON,
+ changes: [],
+ conflicts: []
+ })
+ }
+ return items;
+ }),
+
+
+ resolveConflicts: Zotero.Promise.coroutine(function* (libraryID) {
+ var conflicts = yield this.getConflicts(libraryID);
+ if (!conflicts.length) return false;
+
+ Zotero.debug("Reconciling conflicts for " + Zotero.Libraries.get(libraryID).name);
+
+ var io = {
+ dataIn: {
+ type: 'file',
+ captions: [
+ Zotero.getString('sync.storage.localFile'),
+ Zotero.getString('sync.storage.remoteFile'),
+ Zotero.getString('sync.storage.savedFile')
+ ],
+ conflicts
+ }
+ };
+
+ var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
+ .getService(Components.interfaces.nsIWindowMediator);
+ var lastWin = wm.getMostRecentWindow("navigator:browser");
+ lastWin.openDialog('chrome://zotero/content/merge.xul', '', 'chrome,modal,centerscreen', io);
+
+ if (!io.dataOut) {
+ return false;
+ }
+ yield Zotero.DB.executeTransaction(function* () {
+ for (let i = 0; i < conflicts.length; i++) {
+ let conflict = conflicts[i];
+ let mtime = io.dataOut[i].dateModified;
+ // Local
+ if (mtime == conflict.left.dateModified) {
+ syncState = Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD;
+ }
+ // Remote
+ else {
+ syncState = Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD;
+ }
+ let itemID = Zotero.Items.getIDFromLibraryAndKey(libraryID, conflict.left.key);
+ yield Zotero.Sync.Storage.Local.setSyncState(itemID, syncState);
+ }
+ });
+ return true;
+ })
+}
diff --git a/chrome/content/zotero/xpcom/storage/request.js b/chrome/content/zotero/xpcom/storage/storageRequest.js
similarity index 60%
rename from chrome/content/zotero/xpcom/storage/request.js
rename to chrome/content/zotero/xpcom/storage/storageRequest.js
index bf0d5baa0..ca6b496e3 100644
--- a/chrome/content/zotero/xpcom/storage/request.js
+++ b/chrome/content/zotero/xpcom/storage/storageRequest.js
@@ -27,15 +27,25 @@
/**
* Transfer request for storage sync
*
- * @param {String} name Identifier for request (e.g., "[libraryID]/[key]")
- * @param {Function} onStart Callback to run when request starts
+ * @param {Object} options
+ * @param {String} options.type
+ * @param {Integer} options.libraryID
+ * @param {String} options.name - Identifier for request (e.g., "[libraryID]/[key]")
+ * @param {Function|Function[]} [options.onStart]
+ * @param {Function|Function[]} [options.onProgress]
+ * @param {Function|Function[]} [options.onStop]
*/
-Zotero.Sync.Storage.Request = function (name, callbacks) {
- Zotero.debug("Initializing request '" + name + "'");
+Zotero.Sync.Storage.Request = function (options) {
+ if (!options.type) throw new Error("type must be provided");
+ if (!options.libraryID) throw new Error("libraryID must be provided");
+ if (!options.name) throw new Error("name must be provided");
+ ['type', 'libraryID', 'name'].forEach(x => this[x] = options[x]);
- this.callbacks = ['onStart', 'onProgress'];
+ Zotero.debug(`Initializing ${this.type} request ${this.name}`);
- this.name = name;
+ this.callbacks = ['onStart', 'onProgress', 'onStop'];
+
+ this.Type = Zotero.Utilities.capitalize(this.type);
this.channel = null;
this.queue = null;
this.progress = 0;
@@ -48,17 +58,10 @@ Zotero.Sync.Storage.Request = function (name, callbacks) {
this._remaining = null;
this._maxSize = null;
this._finished = false;
- this._forceFinish = false;
- this._changesMade = false;
- for (var func in callbacks) {
- if (this.callbacks.indexOf(func) !== -1) {
- // Stuff all single functions into arrays
- this['_' + func] = typeof callbacks[func] === 'function' ? [callbacks[func]] : callbacks[func];
- }
- else {
- throw new Error("Invalid handler '" + func + "'");
- }
+ for (let name of this.callbacks) {
+ if (!options[name]) continue;
+ this['_' + name] = Array.isArray(options[name]) ? options[name] : [options[name]];
}
}
@@ -99,11 +102,6 @@ Zotero.Sync.Storage.Request.prototype.importCallbacks = function (request) {
}
-Zotero.Sync.Storage.Request.prototype.__defineGetter__('promise', function () {
- return this._deferred.promise;
-});
-
-
Zotero.Sync.Storage.Request.prototype.__defineGetter__('percentage', function () {
if (this._finished) {
return 100;
@@ -142,7 +140,7 @@ Zotero.Sync.Storage.Request.prototype.__defineGetter__('remaining', function ()
}
if (!this.progressMax) {
- if (this.queue.type == 'upload' && this._maxSize) {
+ if (this.type == 'upload' && this._maxSize) {
return Math.round(Zotero.Sync.Storage.compressionTracker.ratio * this._maxSize);
}
@@ -175,72 +173,47 @@ Zotero.Sync.Storage.Request.prototype.setChannel = function (channel) {
}
-Zotero.Sync.Storage.Request.prototype.start = function () {
- if (!this.queue) {
- throw ("Request " + this.name + " must be added to a queue before starting");
- }
-
- Zotero.debug("Starting " + this.queue.name + " request " + this.name);
+Zotero.Sync.Storage.Request.prototype.start = Zotero.Promise.coroutine(function* () {
+ Zotero.debug("Starting " + this.type + " request " + this.name);
if (this._running) {
- throw new Error("Request " + this.name + " already running");
+ throw new Error(this.type + " request " + this.name + " already running");
+ }
+
+ if (!this._onStart) {
+ throw new Error("onStart not provided -- nothing to do!");
}
this._running = true;
- this.queue.activeRequests++;
- if (this.queue.type == 'download') {
- Zotero.Sync.Storage.setItemDownloadPercentage(this.name, 0);
- }
-
- var self = this;
-
- // this._onStart is an array of promises returning changesMade.
+ // this._onStart is an array of promises for objects of result flags, which are combined
+ // into a single object here
//
// The main sync logic is triggered here.
-
- Zotero.Promise.all([f(this) for each(f in this._onStart)])
- .then(function (results) {
- return {
- localChanges: results.some(function (val) val && val.localChanges == true),
- remoteChanges: results.some(function (val) val && val.remoteChanges == true),
- conflict: results.reduce(function (prev, cur) {
- return prev.conflict ? prev : cur;
- }).conflict
- };
- })
- .then(function (results) {
- Zotero.debug(results);
+ try {
+ var results = yield Zotero.Promise.all(this._onStart.map(f => f(this)));
- if (results.localChanges) {
- Zotero.debug("Changes were made by " + self.queue.name
- + " request " + self.name);
- }
- else {
- Zotero.debug("No changes were made by " + self.queue.name
- + " request " + self.name);
- }
+ var result = new Zotero.Sync.Storage.Result;
+ result.updateFromResults(results);
- // This promise updates localChanges/remoteChanges on the queue
- self._deferred.resolve(results);
- })
- .catch(function (e) {
- if (self._stopping) {
- Zotero.debug("Skipping error for stopping request " + self.name);
- return;
+ Zotero.debug(this.Type + " request " + this.name + " finished");
+ Zotero.debug(result + "");
+
+ return result;
+ }
+ catch (e) {
+ Zotero.logError(this.Type + " request " + this.name + " failed");
+ throw e;
+ }
+ finally {
+ this._finished = true;
+ this._running = false;
+
+ if (this._onStop) {
+ this._onStop.forEach(x => x());
}
- Zotero.debug(self.queue.Type + " request " + self.name + " failed");
- self._deferred.reject(e);
- })
- // Finish the request (and in turn the queue, if this is the last request)
- .finally(function () {
- if (!self._finished) {
- self._finish();
- }
- });
-
- return this._deferred.promise;
-}
+ }
+});
Zotero.Sync.Storage.Request.prototype.isRunning = function () {
@@ -263,7 +236,7 @@ Zotero.Sync.Storage.Request.prototype.isFinished = function () {
* @param {Integer} progressMax Max progress value for this request
* (usually total bytes)
*/
-Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress, progressMax) {
+Zotero.Sync.Storage.Request.prototype.onProgress = function (progress, progressMax) {
//Zotero.debug(progress + "/" + progressMax + " for request " + this.name);
if (!this._running) {
@@ -273,10 +246,6 @@ Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress,
return;
}
- if (!this.channel) {
- this.channel = channel;
- }
-
// Workaround for invalid progress values (possibly related to
// https://bugzilla.mozilla.org/show_bug.cgi?id=451991 and fixed in 3.1)
if (progress < this.progress) {
@@ -292,9 +261,8 @@ Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress,
this.progress = progress;
this.progressMax = progressMax;
- this.queue.updateProgress();
- if (this.queue.type == 'download') {
+ if (this.type == 'download') {
Zotero.Sync.Storage.setItemDownloadPercentage(this.name, this.percentage);
}
@@ -310,59 +278,15 @@ Zotero.Sync.Storage.Request.prototype.onProgress = function (channel, progress,
* Stop the request's underlying network request, if there is one
*/
Zotero.Sync.Storage.Request.prototype.stop = function (force) {
- if (force) {
- this._forceFinish = true;
- }
-
if (this.channel && this.channel.isPending()) {
this._stopping = true;
try {
- Zotero.debug("Stopping request '" + this.name + "'");
+ Zotero.debug(`Stopping ${this.type} request '${this.name} '`);
this.channel.cancel(0x804b0002); // NS_BINDING_ABORTED
}
catch (e) {
- Zotero.debug(e);
+ Zotero.debug(e, 1);
}
}
- else {
- this._finish();
- }
-}
-
-
-/**
- * Mark request as finished and notify queue that it's done
- */
-Zotero.Sync.Storage.Request.prototype._finish = function () {
- // If an error occurred, we wait to finish the request, since doing
- // so might end the queue before the error flag has been set on the queue.
- // When the queue's error handler stops the queue, it stops the request
- // with stop(true) to force the finish to occur, allowing the queue's
- // promise to be rejected with the error.
- if (!this._forceFinish && this._deferred.promise.isRejected()) {
- return;
- }
-
- Zotero.debug("Finishing " + this.queue.name + " request '" + this.name + "'");
- this._finished = true;
- var active = this._running;
- this._running = false;
-
- Zotero.Sync.Storage.setItemDownloadPercentage(this.name, false);
-
- if (active) {
- this.queue.activeRequests--;
- }
- // TEMP: mechanism for failures?
- try {
- this.queue.finishedRequests++;
- this.queue.updateProgress();
- }
- catch (e) {
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
- this._deferred.reject(e);
- throw e;
- }
}
diff --git a/chrome/content/zotero/xpcom/storage/storageResult.js b/chrome/content/zotero/xpcom/storage/storageResult.js
new file mode 100644
index 000000000..eaa1f38c1
--- /dev/null
+++ b/chrome/content/zotero/xpcom/storage/storageResult.js
@@ -0,0 +1,47 @@
+"use strict";
+
+/**
+ * @property {Boolean} localChanges - Changes were made locally. For logging purposes only.
+ * @property {Boolean} remoteChanges - Changes were made on the server. This causes the
+ * last-sync time to be updated on the server (WebDAV) or retrieved (ZFS) and stored locally
+ * to skip additional file syncs until further server changes are made.
+ * @property {Boolean} syncRequired - A data sync is required to upload local changes
+ * @propretty {Boolean} fileSyncRequired - Another file sync is required to handle files left in
+ * conflict
+ */
+Zotero.Sync.Storage.Result = function (options = {}) {
+ this._props = ['localChanges', 'remoteChanges', 'syncRequired', 'fileSyncRequired'];
+ for (let prop of this._props) {
+ this[prop] = options[prop] || false;
+ }
+}
+
+/**
+ * Update the properties on this object from multiple Result objects
+ *
+ * @param {Zotero.Sync.Storage.Result[]} results
+ */
+Zotero.Sync.Storage.Result.prototype.updateFromResults = function (results) {
+ for (let prop of this._props) {
+ if (!this[prop]) {
+ for (let result of results) {
+ if (!(result instanceof Zotero.Sync.Storage.Result)) {
+ Zotero.debug(result, 1);
+ throw new Error("'result' is not a storage result");
+ }
+ if (result[prop]) {
+ this[prop] = true;
+ }
+ }
+ }
+ }
+}
+
+
+Zotero.Sync.Storage.Result.prototype.toString = function () {
+ var obj = {};
+ for (let prop of this._props) {
+ obj[prop] = this[prop] || false;
+ }
+ return JSON.stringify(obj, null, " ");
+}
diff --git a/chrome/content/zotero/xpcom/storage/storageUtilities.js b/chrome/content/zotero/xpcom/storage/storageUtilities.js
new file mode 100644
index 000000000..7df99f1a6
--- /dev/null
+++ b/chrome/content/zotero/xpcom/storage/storageUtilities.js
@@ -0,0 +1,67 @@
+Zotero.Sync.Storage.Utilities = {
+ getClassForMode: function (mode) {
+ switch (mode) {
+ case 'zfs':
+ return Zotero.Sync.Storage.ZFS_Module;
+
+ case 'webdav':
+ return Zotero.Sync.Storage.WebDAV_Module;
+
+ default:
+ throw new Error("Invalid storage mode '" + mode + "'");
+ }
+ },
+
+ getItemFromRequest: function (request) {
+ var [libraryID, key] = request.name.split('/');
+ return Zotero.Items.getByLibraryAndKey(libraryID, key);
+ },
+
+
+ /**
+ * Create zip file of attachment directory in the temp directory
+ *
+ * @param {Zotero.Sync.Storage.Request} request
+ * @return {Promise<Boolean>} - True if the zip file was created, false otherwise
+ */
+ createUploadFile: Zotero.Promise.coroutine(function* (request) {
+ var item = this.getItemFromRequest(request);
+ Zotero.debug("Creating ZIP file for item " + item.libraryKey);
+
+ switch (item.attachmentLinkMode) {
+ case Zotero.Attachments.LINK_MODE_LINKED_FILE:
+ case Zotero.Attachments.LINK_MODE_LINKED_URL:
+ throw new Error("Upload file must be an imported snapshot or file");
+ }
+
+ var zipFile = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.zip');
+
+ return Zotero.File.zipDirectory(
+ Zotero.Attachments.getStorageDirectory(item).path,
+ zipFile,
+ {
+ onStopRequest: function (req, context, status) {
+ var zipFileName = OS.Path.basename(zipFile);
+
+ var originalSize = 0;
+ for (let entry of context.entries) {
+ let zipEntry = context.zipWriter.getEntry(entry.name);
+ if (!zipEntry) {
+ Zotero.logError("ZIP entry '" + entry.name + "' not found for "
+ + "request '" + request.name + "'")
+ continue;
+ }
+ originalSize += zipEntry.realSize;
+ }
+
+ Zotero.debug("Zip of " + zipFileName + " finished with status " + status
+ + " (original " + Math.round(originalSize / 1024) + "KB, "
+ + "compressed " + Math.round(context.zipWriter.file.fileSize / 1024) + "KB, "
+ + Math.round(
+ ((originalSize - context.zipWriter.file.fileSize) / originalSize) * 100
+ ) + "% reduction)");
+ }
+ }
+ );
+ })
+}
diff --git a/chrome/content/zotero/xpcom/storage/streamListener.js b/chrome/content/zotero/xpcom/storage/streamListener.js
index 8a8e28295..136a8f895 100644
--- a/chrome/content/zotero/xpcom/storage/streamListener.js
+++ b/chrome/content/zotero/xpcom/storage/streamListener.js
@@ -30,10 +30,9 @@
* Possible properties of data object:
* - onStart: f(request)
* - onProgress: f(request, progress, progressMax)
- * - onStop: f(request, status, response, data)
- * - onCancel: f(request, status, data)
+ * - onStop: f(request, status, response)
+ * - onCancel: f(request, status)
* - streams: array of streams to close on completion
- * - Other values to pass to onStop()
*/
Zotero.Sync.Storage.StreamListener = function (data) {
this._data = data;
@@ -110,17 +109,15 @@ Zotero.Sync.Storage.StreamListener.prototype = {
},
onStateChange: function (wp, request, stateFlags, status) {
- Zotero.debug("onStateChange");
- Zotero.debug(stateFlags);
- Zotero.debug(status);
+ Zotero.debug("onStateChange with " + stateFlags);
- if ((stateFlags & Components.interfaces.nsIWebProgressListener.STATE_START)
- && (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_NETWORK)) {
- this._onStart(request);
- }
- else if ((stateFlags & Components.interfaces.nsIWebProgressListener.STATE_STOP)
- && (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_NETWORK)) {
- this._onStop(request, status);
+ if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_IS_REQUEST) {
+ if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_START) {
+ this._onStart(request);
+ }
+ else if (stateFlags & Components.interfaces.nsIWebProgressListener.STATE_STOP) {
+ this._onStop(request, status);
+ }
}
},
@@ -148,18 +145,38 @@ Zotero.Sync.Storage.StreamListener.prototype = {
},
// nsIChannelEventSink
- onChannelRedirect: function (oldChannel, newChannel, flags) {
+ //
+ // If this._data.onChannelRedirect exists, it should return a promise resolving to true to
+ // follow the redirect or false to cancel it
+ onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) {
Zotero.debug('onChannelRedirect');
+ if (this._data && this._data.onChannelRedirect) {
+ let result = yield this._data.onChannelRedirect(oldChannel, newChannel, flags);
+ if (!result) {
+ oldChannel.cancel(Components.results.NS_BINDING_ABORTED);
+ newChannel.cancel(Components.results.NS_BINDING_ABORTED);
+ return false;
+ }
+ }
+
// if redirecting, store the new channel
this._channel = newChannel;
- },
+ }),
asyncOnChannelRedirect: function (oldChan, newChan, flags, redirectCallback) {
Zotero.debug('asyncOnRedirect');
- this.onChannelRedirect(oldChan, newChan, flags);
- redirectCallback.onRedirectVerifyCallback(0);
+ this.onChannelRedirect(oldChan, newChan, flags)
+ .then(function (result) {
+ redirectCallback.onRedirectVerifyCallback(
+ result ? Components.results.NS_SUCCEEDED : Components.results.NS_FAILED
+ );
+ })
+ .catch(function (e) {
+ Zotero.logError(e);
+ redirectCallback.onRedirectVerifyCallback(Components.results.NS_FAILED);
+ });
},
// nsIHttpEventSink
@@ -177,8 +194,7 @@ Zotero.Sync.Storage.StreamListener.prototype = {
_onStart: function (request) {
Zotero.debug('Starting request');
if (this._data && this._data.onStart) {
- var data = this._getPassData();
- this._data.onStart(request, data);
+ this._data.onStart(request);
}
},
@@ -189,7 +205,6 @@ Zotero.Sync.Storage.StreamListener.prototype = {
},
_onStop: function (request, status) {
- Zotero.debug('Request ended with status ' + status);
var cancelled = status == 0x804b0002; // NS_BINDING_ABORTED
if (!cancelled && status == 0 && request instanceof Components.interfaces.nsIHttpChannel) {
@@ -201,9 +216,11 @@ Zotero.Sync.Storage.StreamListener.prototype = {
Zotero.debug("Request responseStatus not available", 1);
status = 0;
}
+ Zotero.debug('Request ended with status code ' + status);
request.QueryInterface(Components.interfaces.nsIRequest);
}
else {
+ Zotero.debug('Request ended with status ' + status);
status = 0;
}
@@ -213,38 +230,20 @@ Zotero.Sync.Storage.StreamListener.prototype = {
}
}
- var data = this._getPassData();
-
if (cancelled) {
if (this._data.onCancel) {
- this._data.onCancel(request, status, data);
+ this._data.onCancel(request, status);
}
}
else {
if (this._data.onStop) {
- this._data.onStop(request, status, this._response, data);
+ this._data.onStop(request, status, this._response);
}
}
this._channel = null;
},
- _getPassData: function () {
- // Make copy of data without callbacks to pass along
- var passData = {};
- for (var i in this._data) {
- switch (i) {
- case "onStart":
- case "onProgress":
- case "onStop":
- case "onCancel":
- continue;
- }
- passData[i] = this._data[i];
- }
- return passData;
- },
-
// nsIInterfaceRequestor
getInterface: function (iid) {
try {
diff --git a/chrome/content/zotero/xpcom/storage/webdav.js b/chrome/content/zotero/xpcom/storage/webdav.js
index 9ce0e9e6d..2b6c8c4a3 100644
--- a/chrome/content/zotero/xpcom/storage/webdav.js
+++ b/chrome/content/zotero/xpcom/storage/webdav.js
@@ -24,740 +24,105 @@
*/
-Zotero.Sync.Storage.WebDAV = (function () {
- var _initialized = false;
- var _parentURI;
- var _rootURI;
- var _cachedCredentials = false;
+Zotero.Sync.Storage.WebDAV_Module = {};
+Zotero.Sync.Storage.WebDAV_Module.prototype = {
+ name: "WebDAV",
+ get verified() {
+ return Zotero.Prefs.get("sync.storage.verified");
+ },
- var _loginManagerHost = 'chrome://zotero';
- var _loginManagerURL = 'Zotero Storage Server';
+ _initialized: false,
+ _parentURI: null,
+ _rootURI: null,
+ _cachedCredentials: false,
- var _lastSyncIDLength = 30;
+ _loginManagerHost: 'chrome://zotero',
+ _loginManagerURL: 'Zotero Storage Server',
- //
- // Private methods
- //
- /**
- * Get mod time of file on storage server
- *
- * @param {Zotero.Item} item
- * @param {Function} callback Callback f(item, mdate)
- */
- function getStorageModificationTime(item, request) {
- var uri = getItemPropertyURI(item);
+ _lastSyncIDLength: 30,
+
+
+ get defaultError() {
+ return Zotero.getString('sync.storage.error.webdav.default');
+ },
+
+ get defaultErrorRestart() {
+ return Zotero.getString('sync.storage.error.webdav.defaultRestart', Zotero.appName);
+ },
+
+ get _username() {
+ return Zotero.Prefs.get('sync.storage.username');
+ },
+
+ get _password() {
+ var username = this._username;
- return Zotero.HTTP.promise("GET", uri,
- {
- debug: true,
- successCodes: [200, 300, 404],
- requestObserver: function (xmlhttp) {
- request.setChannel(xmlhttp.channel);
- }
- })
- .then(function (req) {
- checkResponse(req);
-
- // mod_speling can return 300s for 404s with base name matches
- if (req.status == 404 || req.status == 300) {
- return false;
- }
-
- // No modification time set
- if (!req.responseText) {
- return false;
- }
-
- var seconds = false;
- var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
- .createInstance(Components.interfaces.nsIDOMParser);
- try {
- var xml = parser.parseFromString(req.responseText, "text/xml");
- var mtime = xml.getElementsByTagName('mtime')[0].textContent;
- }
- catch (e) {
- Zotero.debug(e);
- var mtime = false;
- }
-
- // TEMP
- if (!mtime) {
- mtime = req.responseText;
- seconds = true;
- }
-
- var invalid = false;
-
- // Unix timestamps need to be converted to ms-based timestamps
- if (seconds) {
- if (mtime.match(/^[0-9]{1,10}$/)) {
- Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds");
- mtime = mtime * 1000;
- }
- else {
- invalid = true;
- }
- }
- else if (!mtime.match(/^[0-9]{1,13}$/)) {
- invalid = true;
- }
-
- // Delete invalid .prop files
- if (invalid) {
- var msg = "Invalid mod date '" + Zotero.Utilities.ellipsize(mtime, 20)
- + "' for item " + Zotero.Items.getLibraryKeyHash(item);
- Zotero.debug(msg, 1);
- Components.utils.reportError(msg);
- return deleteStorageFiles([item.key + ".prop"])
- .then(function (results) {
- throw new Error(Zotero.Sync.Storage.WebDAV.defaultError);
- });
- }
-
- return new Date(parseInt(mtime));
- })
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- throw new Error("HTTP " + e.status + " error from WebDAV "
- + "server for GET request");
- }
- throw e;
- });
- }
-
-
- /**
- * Set mod time of file on storage server
- *
- * @param {Zotero.Item} item
- */
- function setStorageModificationTime(item) {
- var uri = getItemPropertyURI(item);
-
- var mtime = item.attachmentModificationTime;
- var hash = item.attachmentHash;
-
- var prop = '<properties version="1">'
- + '<mtime>' + mtime + '</mtime>'
- + '<hash>' + hash + '</hash>'
- + '</properties>';
-
- return Zotero.HTTP.promise("PUT", uri,
- { body: prop, debug: true, successCodes: [200, 201, 204] })
- .then(function (req) {
- return { mtime: mtime, hash: hash };
- })
- .catch(function (e) {
- throw new Error("HTTP " + e.xmlhttp.status
- + " from WebDAV server for HTTP PUT");
- });
- };
-
-
-
- /**
- * Upload the generated ZIP file to the server
- *
- * @param {Object} Object with 'request' property
- * @return {void}
- */
- function processUploadFile(data) {
- /*
- updateSizeMultiplier(
- (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100
- );
- */
- var request = data.request;
- var item = Zotero.Sync.Storage.getItemFromRequestName(request.name);
-
- return getStorageModificationTime(item, request)
- .then(function (mdate) {
- if (!request.isRunning()) {
- Zotero.debug("Upload request '" + request.name
- + "' is no longer running after getting mod time");
- return false;
- }
-
- // Check for conflict
- if (Zotero.Sync.Storage.getSyncState(item.id)
- != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) {
- if (mdate) {
- // Remote prop time
- var mtime = mdate.getTime();
-
- // Local file time
- var fmtime = item.attachmentModificationTime;
-
- var same = false;
- if (fmtime == mtime) {
- same = true;
- Zotero.debug("File mod time matches remote file -- skipping upload");
- }
- // Allow floored timestamps for filesystems that don't support
- // millisecond precision (e.g., HFS+)
- else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
- same = true;
- Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
- + "-- skipping upload");
- }
- // Allow timestamp to be exactly one hour off to get around
- // time zone issues -- there may be a proper way to fix this
- else if (Math.abs(fmtime - mtime) == 3600000
- // And check with one-second precision as well
- || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
- || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
- same = true;
- Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
- + "-- assuming time zone issue and skipping upload");
- }
-
- if (same) {
- Zotero.DB.beginTransaction();
- var syncState = Zotero.Sync.Storage.getSyncState(item.id);
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- Zotero.DB.commitTransaction();
- return true;
- }
-
- var smtime = Zotero.Sync.Storage.getSyncedModificationTime(item.id);
- if (smtime != mtime) {
- Zotero.debug("Conflict -- last synced file mod time "
- + "does not match time on storage server"
- + " (" + smtime + " != " + mtime + ")");
- return {
- localChanges: false,
- remoteChanges: false,
- conflict: {
- local: { modTime: fmtime },
- remote: { modTime: mtime }
- }
- };
- }
- }
- else {
- Zotero.debug("Remote file not found for item " + item.id);
- }
- }
-
- var file = Zotero.getTempDirectory();
- file.append(item.key + '.zip');
-
- var fis = Components.classes["@mozilla.org/network/file-input-stream;1"]
- .createInstance(Components.interfaces.nsIFileInputStream);
- fis.init(file, 0x01, 0, 0);
-
- var bis = Components.classes["@mozilla.org/network/buffered-input-stream;1"]
- .createInstance(Components.interfaces.nsIBufferedInputStream)
- bis.init(fis, 64 * 1024);
-
- var uri = getItemURI(item);
-
- var ios = Components.classes["@mozilla.org/network/io-service;1"].
- getService(Components.interfaces.nsIIOService);
- var channel = ios.newChannelFromURI(uri);
- channel.QueryInterface(Components.interfaces.nsIUploadChannel);
- channel.setUploadStream(bis, 'application/octet-stream', -1);
- channel.QueryInterface(Components.interfaces.nsIHttpChannel);
- channel.requestMethod = 'PUT';
- channel.allowPipelining = false;
-
- channel.setRequestHeader('Keep-Alive', '', false);
- channel.setRequestHeader('Connection', '', false);
-
- var deferred = Zotero.Promise.defer();
-
- var listener = new Zotero.Sync.Storage.StreamListener(
- {
- onProgress: function (a, b, c) {
- request.onProgress(a, b, c);
- },
- onStop: function (httpRequest, status, response, data) {
- data.request.setChannel(false);
-
- deferred.resolve(
- Zotero.Promise.try(function () {
- return onUploadComplete(httpRequest, status, response, data);
- })
- );
- },
- onCancel: function (httpRequest, status, data) {
- onUploadCancel(httpRequest, status, data);
- deferred.resolve(false);
- },
- request: request,
- item: item,
- streams: [fis, bis]
- }
- );
- channel.notificationCallbacks = listener;
-
- var dispURI = uri.clone();
- if (dispURI.password) {
- dispURI.password = '********';
- }
- Zotero.debug("HTTP PUT of " + file.leafName + " to " + dispURI.spec);
-
- channel.asyncOpen(listener, null);
-
- return deferred.promise;
- });
- }
-
-
- function onUploadComplete(httpRequest, status, response, data) {
- var request = data.request;
- var item = data.item;
- var url = httpRequest.name;
-
- Zotero.debug("Upload of attachment " + item.key
- + " finished with status code " + status);
-
- switch (status) {
- case 200:
- case 201:
- case 204:
- break;
-
- case 403:
- case 500:
- Zotero.debug(response);
- throw (Zotero.getString('sync.storage.error.fileUploadFailed') +
- " " + Zotero.getString('sync.storage.error.checkFileSyncSettings'));
-
- case 507:
- Zotero.debug(response);
- throw Zotero.getString('sync.storage.error.webdav.insufficientSpace');
-
- default:
- Zotero.debug(response);
- throw (Zotero.getString('sync.storage.error.fileUploadFailed') +
- " " + Zotero.getString('sync.storage.error.checkFileSyncSettings')
- + "\n\n" + "HTTP " + status);
+ if (!username) {
+ Zotero.debug('Username not set before getting Zotero.Sync.Storage.WebDAV.password');
+ return '';
}
- return setStorageModificationTime(item)
- .then(function (props) {
- if (!request.isRunning()) {
- Zotero.debug("Upload request '" + request.name
- + "' is no longer running after getting mod time");
- return false;
- }
-
- Zotero.DB.beginTransaction();
-
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, props.mtime, true);
- Zotero.Sync.Storage.setSyncedHash(item.id, props.hash);
-
- Zotero.DB.commitTransaction();
-
- try {
- var file = Zotero.getTempDirectory();
- file.append(item.key + '.zip');
- file.remove(false);
- }
- catch (e) {
- Components.utils.reportError(e);
- }
-
- return {
- localChanges: true,
- remoteChanges: true
- };
- });
- }
-
-
- function onUploadCancel(httpRequest, status, data) {
- var request = data.request;
- var item = data.item;
+ Zotero.debug('Getting WebDAV password');
+ var loginManager = Components.classes["@mozilla.org/login-manager;1"]
+ .getService(Components.interfaces.nsILoginManager);
+ var logins = loginManager.findLogins({}, this._loginManagerHost, this._loginManagerURL, null);
- Zotero.debug("Upload of attachment " + item.key + " cancelled with status code " + status);
-
- try {
- var file = Zotero.getTempDirectory();
- file.append(item.key + '.zip');
- file.remove(false);
- }
- catch (e) {
- Components.utils.reportError(e);
- }
- }
-
-
- /**
- * Create a Zotero directory on the storage server
- */
- function createServerDirectory(callback) {
- var uri = Zotero.Sync.Storage.WebDAV.rootURI;
- Zotero.HTTP.WebDAV.doMkCol(uri, function (req) {
- Zotero.debug(req.responseText);
- Zotero.debug(req.status);
-
- switch (req.status) {
- case 201:
- return [uri, Zotero.Sync.Storage.SUCCESS];
-
- case 401:
- return [uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED];
-
- case 403:
- return [uri, Zotero.Sync.Storage.ERROR_FORBIDDEN];
-
- case 405:
- return [uri, Zotero.Sync.Storage.ERROR_NOT_ALLOWED];
-
- case 500:
- return [uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR];
-
- default:
- return [uri, Zotero.Sync.Storage.ERROR_UNKNOWN];
+ // Find user from returned array of nsILoginInfo objects
+ for (var i = 0; i < logins.length; i++) {
+ if (logins[i].username == username) {
+ return logins[i].password;
}
- });
- }
-
-
- /**
- * Get the storage URI for an item
- *
- * @inner
- * @param {Zotero.Item}
- * @return {nsIURI} URI of file on storage server
- */
- function getItemURI(item) {
- var uri = Zotero.Sync.Storage.WebDAV.rootURI;
- uri.spec = uri.spec + item.key + '.zip';
- return uri;
- }
-
-
- /**
- * Get the storage property file URI for an item
- *
- * @inner
- * @param {Zotero.Item}
- * @return {nsIURI} URI of property file on storage server
- */
- function getItemPropertyURI(item) {
- var uri = Zotero.Sync.Storage.WebDAV.rootURI;
- uri.spec = uri.spec + item.key + '.prop';
- return uri;
- }
-
-
- /**
- * Get the storage property file URI corresponding to a given item storage URI
- *
- * @param {nsIURI} Item storage URI
- * @return {nsIURI|FALSE} Property file URI, or FALSE if not an item storage URI
- */
- function getPropertyURIFromItemURI(uri) {
- if (!uri.spec.match(/\.zip$/)) {
- return false;
- }
- var propURI = uri.clone();
- propURI.QueryInterface(Components.interfaces.nsIURL);
- propURI.fileName = uri.fileName.replace(/\.zip$/, '.prop');
- propURI.QueryInterface(Components.interfaces.nsIURI);
- return propURI;
- }
-
-
- /**
- * @inner
- * @param {String[]} files Remote filenames to delete (e.g., ZIPs)
- * @param {Function} callback Passed object containing three arrays:
- * 'deleted', 'missing', and 'error',
- * each containing filenames
- */
- function deleteStorageFiles(files) {
- var results = {
- deleted: [],
- missing: [],
- error: []
- };
-
- if (files.length == 0) {
- return Zotero.Promise.resolve(results);
}
- let deleteURI = _rootURI.clone();
- // This should never happen, but let's be safe
- if (!deleteURI.spec.match(/\/$/)) {
- return Zotero.Promise.reject("Root URI does not end in slash in "
- + "Zotero.Sync.Storage.WebDAV.deleteStorageFiles()");
- }
-
- var funcs = [];
- for (let i=0; i<files.length; i++) {
- let fileName = files[i];
- let baseName = fileName.match(/^([^\.]+)/)[1];
- funcs.push(function () {
- let deleteURI = _rootURI.clone();
- deleteURI.QueryInterface(Components.interfaces.nsIURL);
- deleteURI.fileName = fileName;
- deleteURI.QueryInterface(Components.interfaces.nsIURI);
- return Zotero.HTTP.promise("DELETE", deleteURI, { successCodes: [200, 204, 404] })
- .then(function (req) {
- switch (req.status) {
- case 204:
- // IIS 5.1 and Sakai return 200
- case 200:
- var fileDeleted = true;
- break;
-
- case 404:
- var fileDeleted = true;
- break;
- }
-
- // If an item file URI, get the property URI
- var deletePropURI = getPropertyURIFromItemURI(deleteURI);
-
- // If we already deleted the prop file, skip it
- if (!deletePropURI || results.deleted.indexOf(deletePropURI.fileName) != -1) {
- if (fileDeleted) {
- results.deleted.push(baseName);
- }
- else {
- results.missing.push(baseName);
- }
- return;
- }
-
- let propFileName = deletePropURI.fileName;
-
- // Delete property file
- return Zotero.HTTP.promise("DELETE", deletePropURI, { successCodes: [200, 204, 404] })
- .then(function (req) {
- switch (req.status) {
- case 204:
- // IIS 5.1 and Sakai return 200
- case 200:
- results.deleted.push(baseName);
- break;
-
- case 404:
- if (fileDeleted) {
- results.deleted.push(baseName);
- }
- else {
- results.missing.push(baseName);
- }
- break;
- }
- });
- })
- .catch(function (e) {
- results.error.push(baseName);
- throw e;
- });
- });
- }
-
- Components.utils.import("resource://zotero/concurrent-caller.js");
- var caller = new ConcurrentCaller(4);
- caller.stopOnError = true;
- caller.setLogger(function (msg) Zotero.debug(msg));
- caller.onError(function (e) Components.utils.reportError(e));
- return caller.fcall(funcs)
- .then(function () {
- return results;
- });
- }
+ return '';
+ },
-
- /**
- * Checks for an invalid SSL certificate and throws a nice error
- */
- function checkResponse(req) {
- var channel = req.channel;
- if (!channel instanceof Ci.nsIChannel) {
- Zotero.Sync.Storage.EventManager.error('No HTTPS channel available');
- }
-
- // Check if the error we encountered is really an SSL error
- // Logic borrowed from https://developer.mozilla.org/en-US/docs/How_to_check_the_security_state_of_an_XMLHTTPRequest_over_SSL
- // http://mxr.mozilla.org/mozilla-central/source/security/nss/lib/ssl/sslerr.h
- // http://mxr.mozilla.org/mozilla-central/source/security/nss/lib/util/secerr.h
- var secErrLimit = Ci.nsINSSErrorsService.NSS_SEC_ERROR_LIMIT - Ci.nsINSSErrorsService.NSS_SEC_ERROR_BASE;
- var secErr = Math.abs(Ci.nsINSSErrorsService.NSS_SEC_ERROR_BASE) - (channel.status & 0xffff);
- var sslErrLimit = Ci.nsINSSErrorsService.NSS_SSL_ERROR_LIMIT - Ci.nsINSSErrorsService.NSS_SSL_ERROR_BASE;
- var sslErr = Math.abs(Ci.nsINSSErrorsService.NSS_SSL_ERROR_BASE) - (channel.status & 0xffff);
- if( (secErr < 0 || secErr > secErrLimit) && (sslErr < 0 || sslErr > sslErrLimit) ) {
+ set _password(password) {
+ var username = this._username;
+ if (!username) {
+ Zotero.debug('Username not set before setting Zotero.Sync.Server.Mode.WebDAV.password');
return;
}
- var secInfo = channel.securityInfo;
- if (secInfo instanceof Ci.nsITransportSecurityInfo) {
- secInfo.QueryInterface(Ci.nsITransportSecurityInfo);
- if ((secInfo.securityState & Ci.nsIWebProgressListener.STATE_IS_INSECURE) == Ci.nsIWebProgressListener.STATE_IS_INSECURE) {
- var host = 'host';
- try {
- host = channel.URI.host;
- }
- catch (e) {
- Zotero.debug(e);
- }
-
- var msg = Zotero.getString('sync.storage.error.webdav.sslCertificateError', host);
- // In Standalone, provide cert_override.txt instructions and a
- // button to open the Zotero profile directory
- if (Zotero.isStandalone) {
- msg += "\n\n" + Zotero.getString('sync.storage.error.webdav.seeCertOverrideDocumentation');
- var buttonText = Zotero.getString('general.openDocumentation');
- var func = function () {
- var zp = Zotero.getActiveZoteroPane();
- zp.loadURI("https://www.zotero.org/support/kb/cert_override", { shiftKey: true });
- };
- }
- // In Firefox display a button to load the WebDAV URL
- else {
- msg += "\n\n" + Zotero.getString('sync.storage.error.webdav.loadURLForMoreInfo');
- var buttonText = Zotero.getString('sync.storage.error.webdav.loadURL');
- var func = function () {
- var zp = Zotero.getActiveZoteroPane();
- zp.loadURI(channel.URI.spec, { shiftKey: true });
- };
- }
-
- var e = new Zotero.Error(
- msg,
- 0,
- {
- dialogText: msg,
- dialogButtonText: buttonText,
- dialogButtonCallback: func
- }
- );
- throw e;
- }
- else if ((secInfo.securityState & Ci.nsIWebProgressListener.STATE_IS_BROKEN) == Ci.nsIWebProgressListener.STATE_IS_BROKEN) {
- var msg = Zotero.getString('sync.storage.error.webdav.sslConnectionError', host) +
- Zotero.getString('sync.storage.error.webdav.loadURLForMoreInfo');
- var e = new Zotero.Error(
- msg,
- 0,
- {
- dialogText: msg,
- dialogButtonText: Zotero.getString('sync.storage.error.webdav.loadURL'),
- dialogButtonCallback: function () {
- var zp = Zotero.getActiveZoteroPane();
- zp.loadURI(channel.URI.spec, { shiftKey: true });
- }
- }
- );
- throw e;
- }
- }
- }
-
-
- //
- // Public methods (called via Zotero.Sync.Storage.WebDAV)
- //
- var obj = new Zotero.Sync.Storage.Mode;
- obj.name = "WebDAV";
-
- Object.defineProperty(obj, "defaultError", {
- get: function () Zotero.getString('sync.storage.error.webdav.default')
- });
-
- Object.defineProperty(obj, "defaultErrorRestart", {
- get: function () Zotero.getString('sync.storage.error.webdav.defaultRestart', Zotero.appName)
- });
-
- Object.defineProperty(obj, "includeUserFiles", {
- get: function () {
- return Zotero.Prefs.get("sync.storage.enabled") && Zotero.Prefs.get("sync.storage.protocol") == 'webdav';
- }
- });
- obj.includeGroupItems = false;
-
- Object.defineProperty(obj, "_verified", {
- get: function () Zotero.Prefs.get("sync.storage.verified")
- });
-
- Object.defineProperty(obj, "_username", {
- get: function () Zotero.Prefs.get('sync.storage.username')
- });
-
- Object.defineProperty(obj, "_password", {
- get: function () {
- var username = this._username;
-
- if (!username) {
- Zotero.debug('Username not set before getting Zotero.Sync.Storage.WebDAV.password');
- return '';
- }
-
- Zotero.debug('Getting WebDAV password');
- var loginManager = Components.classes["@mozilla.org/login-manager;1"]
- .getService(Components.interfaces.nsILoginManager);
- var logins = loginManager.findLogins({}, _loginManagerHost, _loginManagerURL, null);
-
- // Find user from returned array of nsILoginInfo objects
- for (var i = 0; i < logins.length; i++) {
- if (logins[i].username == username) {
- return logins[i].password;
- }
- }
-
- return '';
- },
+ _cachedCredentials = false;
- set: function (password) {
- var username = this._username;
- if (!username) {
- Zotero.debug('Username not set before setting Zotero.Sync.Server.Mode.WebDAV.password');
- return;
- }
-
- _cachedCredentials = false;
-
- var loginManager = Components.classes["@mozilla.org/login-manager;1"]
- .getService(Components.interfaces.nsILoginManager);
- var logins = loginManager.findLogins({}, _loginManagerHost, _loginManagerURL, null);
-
- for (var i = 0; i < logins.length; i++) {
- Zotero.debug('Clearing WebDAV passwords');
- loginManager.removeLogin(logins[i]);
- break;
- }
-
- if (password) {
- Zotero.debug(_loginManagerURL);
- var nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1",
- Components.interfaces.nsILoginInfo, "init");
- var loginInfo = new nsLoginInfo(_loginManagerHost, _loginManagerURL,
- null, username, password, "", "");
- loginManager.addLogin(loginInfo);
- }
+ var loginManager = Components.classes["@mozilla.org/login-manager;1"]
+ .getService(Components.interfaces.nsILoginManager);
+ var logins = loginManager.findLogins({}, this._loginManagerHost, this._loginManagerURL, null);
+
+ for (var i = 0; i < logins.length; i++) {
+ Zotero.debug('Clearing WebDAV passwords');
+ loginManager.removeLogin(logins[i]);
+ break;
}
- });
-
- Object.defineProperty(obj, "rootURI", {
- get: function () {
- if (!_rootURI) {
- this._init();
- }
- return _rootURI.clone();
+
+ if (password) {
+ Zotero.debug(this._loginManagerURL);
+ var nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1",
+ Components.interfaces.nsILoginInfo, "init");
+ var loginInfo = new nsLoginInfo(this._loginManagerHost, this._loginManagerURL,
+ null, username, password, "", "");
+ loginManager.addLogin(loginInfo);
}
- });
+ },
- Object.defineProperty(obj, "parentURI", {
- get: function () {
- if (!_parentURI) {
- this._init();
- }
- return _parentURI.clone();
+ get rootURI() {
+ if (!this._rootURI) {
+ this._init();
}
- });
+ return this._rootURI.clone();
+ },
- obj._init = function () {
- _rootURI = false;
- _parentURI = false;
+ get parentURI() {
+ if (!this._parentURI) {
+ this._init();
+ }
+ return this._parentURI.clone();
+ },
+
+ init: function () {
+ this._rootURI = false;
+ this._parentURI = false;
var scheme = Zotero.Prefs.get('sync.storage.scheme');
switch (scheme) {
@@ -816,26 +181,52 @@ Zotero.Sync.Storage.WebDAV = (function () {
if (!uri.spec.match(/\/$/)) {
uri.spec += "/";
}
- _parentURI = uri;
+ this._parentURI = uri;
var uri = uri.clone();
uri.spec += "zotero/";
- _rootURI = uri;
- };
+ this._rootURI = uri;
+ },
+
+ cacheCredentials: Zotero.Promise.coroutine(function* () {
+ if (this._cachedCredentials) {
+ Zotero.debug("WebDAV credentials are already cached");
+ return;
+ }
- obj.clearCachedCredentials = function() {
- _rootURI = _parentURI = undefined;
- _cachedCredentials = false;
- };
+ try {
+ var req = Zotero.HTTP.request("OPTIONS", this.rootURI);
+ checkResponse(req);
+
+ Zotero.debug("Credentials are cached");
+ this._cachedCredentials = true;
+ }
+ catch (e) {
+ if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
+ let msg = "HTTP " + e.status + " error from WebDAV server "
+ + "for OPTIONS request";
+ Zotero.debug(msg, 1);
+ Components.utils.reportError(msg);
+ throw new Error(Zotero.Sync.Storage.WebDAV.defaultErrorRestart);
+ }
+ throw e;
+ }
+ }),
+
+
+ clearCachedCredentials: function() {
+ this._rootURI = this._parentURI = undefined;
+ this._cachedCredentials = false;
+ },
/**
* Begin download process for individual file
*
* @param {Zotero.Sync.Storage.Request} [request]
*/
- obj._downloadFile = function (request) {
- var item = Zotero.Sync.Storage.getItemFromRequestName(request.name);
+ downloadFile: function (request, requeueCallback) {
+ var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
if (!item) {
throw new Error("Item '" + request.name + "' not found");
}
@@ -868,7 +259,9 @@ Zotero.Sync.Storage.WebDAV = (function () {
Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
Zotero.DB.commitTransaction();
return {
- localChanges: true
+ localChanges: true, // ?
+ remoteChanges: false,
+ syncRequired: false
};
}
@@ -930,7 +323,11 @@ Zotero.Sync.Storage.WebDAV = (function () {
Zotero.debug("Finished download of " + destFile.path);
try {
- deferred.resolve(Zotero.Sync.Storage.processDownload(data));
+ deferred.resolve(
+ Zotero.Sync.Storage.processDownload(
+ data, requeueCallback
+ )
+ );
}
catch (e) {
deferred.reject(e);
@@ -963,10 +360,10 @@ Zotero.Sync.Storage.WebDAV = (function () {
return deferred.promise;
});
- };
+ },
- obj._uploadFile = function (request) {
+ uploadFile: function (request) {
var deferred = Zotero.Promise.defer();
var created = Zotero.Sync.Storage.createUploadFile(
request,
@@ -986,10 +383,10 @@ Zotero.Sync.Storage.WebDAV = (function () {
return Zotero.Promise.resolve(false);
}
return deferred.promise;
- };
+ },
- obj._getLastSyncTime = function () {
+ getLastSyncTime: function () {
var lastSyncURI = this.rootURI;
lastSyncURI.spec += "lastsync.txt";
@@ -1063,10 +460,10 @@ Zotero.Sync.Storage.WebDAV = (function () {
throw (e);
}
});
- };
+ },
- obj._setLastSyncTime = function (libraryID, localLastSyncID) {
+ setLastSyncTime: function (libraryID, localLastSyncID) {
if (libraryID != Zotero.Libraries.userLibraryID) {
throw new Error("libraryID must be user library");
}
@@ -1102,36 +499,11 @@ Zotero.Sync.Storage.WebDAV = (function () {
Components.utils.reportError(msg);
throw Zotero.Sync.Storage.WebDAV.defaultError;
});
- };
+ },
- obj._cacheCredentials = function () {
- if (_cachedCredentials) {
- Zotero.debug("WebDAV credentials are already cached");
- return;
- }
-
- return Zotero.HTTP.promise("OPTIONS", this.rootURI)
- .then(function (req) {
- checkResponse(req);
-
- Zotero.debug("Credentials are cached");
- _cachedCredentials = true;
- })
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- var msg = "HTTP " + e.status + " error from WebDAV server "
- + "for OPTIONS request";
- Zotero.debug(msg, 1);
- Components.utils.reportError(msg);
- throw new Error(Zotero.Sync.Storage.WebDAV.defaultErrorRestart);
- }
- throw e;
- });
- };
-
- obj._checkServer = function () {
+ checkServer: function () {
var deferred = Zotero.Promise.defer();
try {
@@ -1399,7 +771,7 @@ Zotero.Sync.Storage.WebDAV = (function () {
}, 0);
return deferred.promise;
- };
+ },
/**
@@ -1407,7 +779,7 @@ Zotero.Sync.Storage.WebDAV = (function () {
*
* @return bool True if the verification succeeded, false otherwise
*/
- obj._checkServerCallback = function (uri, status, window, skipSuccessMessage) {
+ checkServerCallback: function (uri, status, window, skipSuccessMessage) {
var promptService =
Components.classes["@mozilla.org/embedcomp/prompt-service;1"].
createInstance(Components.interfaces.nsIPromptService);
@@ -1545,72 +917,58 @@ Zotero.Sync.Storage.WebDAV = (function () {
promptService.alert(window, errorTitle, errorMessage);
}
return false;
- };
+ },
/**
* Remove files on storage server that were deleted locally
*
- * @param {Function} callback Passed number of files deleted
+ * @param {Integer} libraryID
*/
- obj._purgeDeletedStorageFiles = function () {
- return Zotero.Promise.try(function () {
- if (!this.includeUserFiles) {
- return false;
- }
-
- Zotero.debug("Purging deleted storage files");
- var files = Zotero.Sync.Storage.getDeletedFiles();
- if (!files) {
- Zotero.debug("No files to delete remotely");
- return false;
- }
-
- // Add .zip extension
- var files = files.map(function (file) file + ".zip");
-
- return deleteStorageFiles(files)
- .then(function (results) {
- // Remove deleted and nonexistent files from storage delete log
- var toPurge = results.deleted.concat(results.missing);
- if (toPurge.length > 0) {
- var done = 0;
- var maxFiles = 999;
- var numFiles = toPurge.length;
-
- Zotero.DB.beginTransaction();
-
- do {
- var chunk = toPurge.splice(0, maxFiles);
- var sql = "DELETE FROM storageDeleteLog WHERE key IN ("
- + chunk.map(function () '?').join() + ")";
- Zotero.DB.query(sql, chunk);
- done += chunk.length;
+ purgeDeletedStorageFiles: Zotero.Promise.coroutine(function* (libraryID) {
+ Zotero.debug("Purging deleted storage files");
+ var files = yield Zotero.Sync.Storage.Local.getDeletedFiles(libraryID);
+ if (!files.length) {
+ Zotero.debug("No files to delete remotely");
+ return false;
+ }
+
+ // Add .zip extension
+ var files = files.map(file => file + ".zip");
+
+ var results = yield deleteStorageFiles(files)
+
+ // Remove deleted and nonexistent files from storage delete log
+ var toPurge = results.deleted.concat(results.missing);
+ if (toPurge.length > 0) {
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Utilities.Internal.forEachChunkAsync(
+ toPurge,
+ Zotero.DB.MAX_BOUND_PARAMETERS,
+ function (chunk) {
+ var sql = "DELETE FROM storageDeleteLog WHERE libraryID=? AND key IN ("
+ + chunk.map(() => '?').join() + ")";
+ return Zotero.DB.queryAsync(sql, [libraryID].concat(chunk));
}
- while (done < numFiles);
-
- Zotero.DB.commitTransaction();
- }
-
- Zotero.debug(results);
-
- return results.deleted.length;
+ );
});
- }.bind(this));
- };
+ }
+
+ Zotero.debug(results);
+
+ return results.deleted.length;
+ }),
/**
* Delete orphaned storage files older than a day before last sync time
*/
- obj._purgeOrphanedStorageFiles = function () {
+ purgeOrphanedStorageFiles: function (libraryID) {
+ // Note: libraryID not currently used
+
return Zotero.Promise.try(function () {
const daysBeforeSyncTime = 1;
- if (!this.includeUserFiles) {
- return false;
- }
-
// If recently purged, skip
var lastpurge = Zotero.Prefs.get('lastWebDAVOrphanPurge');
var days = 10;
@@ -1742,7 +1100,602 @@ Zotero.Sync.Storage.WebDAV = (function () {
return deferred.promise;
}.bind(this));
- };
+ },
- return obj;
-}());
+
+ //
+ // Private methods
+ //
+ /**
+ * Get mod time of file on storage server
+ *
+ * @param {Zotero.Item} item
+ * @param {Function} callback Callback f(item, mdate)
+ */
+ _getStorageModificationTime: function (item, request) {
+ var uri = getItemPropertyURI(item);
+
+ return Zotero.HTTP.promise("GET", uri,
+ {
+ debug: true,
+ successCodes: [200, 300, 404],
+ requestObserver: function (xmlhttp) {
+ request.setChannel(xmlhttp.channel);
+ }
+ })
+ .then(function (req) {
+ checkResponse(req);
+
+ // mod_speling can return 300s for 404s with base name matches
+ if (req.status == 404 || req.status == 300) {
+ return false;
+ }
+
+ // No modification time set
+ if (!req.responseText) {
+ return false;
+ }
+
+ var seconds = false;
+ var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
+ .createInstance(Components.interfaces.nsIDOMParser);
+ try {
+ var xml = parser.parseFromString(req.responseText, "text/xml");
+ var mtime = xml.getElementsByTagName('mtime')[0].textContent;
+ }
+ catch (e) {
+ Zotero.debug(e);
+ var mtime = false;
+ }
+
+ // TEMP
+ if (!mtime) {
+ mtime = req.responseText;
+ seconds = true;
+ }
+
+ var invalid = false;
+
+ // Unix timestamps need to be converted to ms-based timestamps
+ if (seconds) {
+ if (mtime.match(/^[0-9]{1,10}$/)) {
+ Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds");
+ mtime = mtime * 1000;
+ }
+ else {
+ invalid = true;
+ }
+ }
+ else if (!mtime.match(/^[0-9]{1,13}$/)) {
+ invalid = true;
+ }
+
+ // Delete invalid .prop files
+ if (invalid) {
+ var msg = "Invalid mod date '" + Zotero.Utilities.ellipsize(mtime, 20)
+ + "' for item " + Zotero.Items.getLibraryKeyHash(item);
+ Zotero.debug(msg, 1);
+ Components.utils.reportError(msg);
+ return deleteStorageFiles([item.key + ".prop"])
+ .then(function (results) {
+ throw new Error(Zotero.Sync.Storage.WebDAV.defaultError);
+ });
+ }
+
+ return new Date(parseInt(mtime));
+ })
+ .catch(function (e) {
+ if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
+ throw new Error("HTTP " + e.status + " error from WebDAV "
+ + "server for GET request");
+ }
+ throw e;
+ });
+ },
+
+
+ /**
+ * Set mod time of file on storage server
+ *
+ * @param {Zotero.Item} item
+ */
+ _setStorageModificationTime: Zotero.Promise.coroutine(function* (item) {
+ var uri = getItemPropertyURI(item);
+
+ var mtime = item.attachmentModificationTime;
+ var hash = yield item.attachmentHash;
+
+ var prop = '<properties version="1">'
+ + '<mtime>' + mtime + '</mtime>'
+ + '<hash>' + hash + '</hash>'
+ + '</properties>';
+
+ return Zotero.HTTP.promise("PUT", uri,
+ { body: prop, debug: true, successCodes: [200, 201, 204] })
+ .then(function (req) {
+ return { mtime: mtime, hash: hash };
+ })
+ .catch(function (e) {
+ throw new Error("HTTP " + e.xmlhttp.status
+ + " from WebDAV server for HTTP PUT");
+ })
+ }),
+
+
+
+ /**
+ * Upload the generated ZIP file to the server
+ *
+ * @param {Object} Object with 'request' property
+ * @return {void}
+ */
+ _processUploadFile: Zotero.Promise.coroutine(function* (data) {
+ /*
+ updateSizeMultiplier(
+ (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100
+ );
+ */
+ var request = data.request;
+ var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
+
+ var mdate = getStorageModificationTime(item, request);
+
+ if (!request.isRunning()) {
+ Zotero.debug("Upload request '" + request.name
+ + "' is no longer running after getting mod time");
+ return false;
+ }
+
+ // Check for conflict
+ if (Zotero.Sync.Storage.getSyncState(item.id)
+ != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) {
+ if (mdate) {
+ // Local file time
+ var fmtime = yield item.attachmentModificationTime;
+ // Remote prop time
+ var mtime = mdate.getTime();
+
+ var same = !(yield Zotero.Sync.Storage.checkFileModTime(item, fmtime, mtime));
+ if (same) {
+ Zotero.DB.beginTransaction();
+ var syncState = Zotero.Sync.Storage.getSyncState(item.id);
+ Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true);
+ Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
+ Zotero.DB.commitTransaction();
+ return true;
+ }
+
+ let smtime = Zotero.Sync.Storage.getSyncedModificationTime(item.id);
+ if (smtime != mtime) {
+ Zotero.debug("Conflict -- last synced file mod time "
+ + "does not match time on storage server"
+ + " (" + smtime + " != " + mtime + ")");
+ return {
+ localChanges: false,
+ remoteChanges: false,
+ syncRequired: false,
+ conflict: {
+ local: { modTime: fmtime },
+ remote: { modTime: mtime }
+ }
+ };
+ }
+ }
+ else {
+ Zotero.debug("Remote file not found for item " + item.id);
+ }
+ }
+
+ var file = Zotero.getTempDirectory();
+ file.append(item.key + '.zip');
+
+ var fis = Components.classes["@mozilla.org/network/file-input-stream;1"]
+ .createInstance(Components.interfaces.nsIFileInputStream);
+ fis.init(file, 0x01, 0, 0);
+
+ var bis = Components.classes["@mozilla.org/network/buffered-input-stream;1"]
+ .createInstance(Components.interfaces.nsIBufferedInputStream)
+ bis.init(fis, 64 * 1024);
+
+ var uri = getItemURI(item);
+
+ var ios = Components.classes["@mozilla.org/network/io-service;1"].
+ getService(Components.interfaces.nsIIOService);
+ var channel = ios.newChannelFromURI(uri);
+ channel.QueryInterface(Components.interfaces.nsIUploadChannel);
+ channel.setUploadStream(bis, 'application/octet-stream', -1);
+ channel.QueryInterface(Components.interfaces.nsIHttpChannel);
+ channel.requestMethod = 'PUT';
+ channel.allowPipelining = false;
+
+ channel.setRequestHeader('Keep-Alive', '', false);
+ channel.setRequestHeader('Connection', '', false);
+
+ var deferred = Zotero.Promise.defer();
+
+ var listener = new Zotero.Sync.Storage.StreamListener(
+ {
+ onProgress: function (a, b, c) {
+ request.onProgress(a, b, c);
+ },
+ onStop: function (httpRequest, status, response, data) {
+ data.request.setChannel(false);
+
+ deferred.resolve(
+ Zotero.Promise.try(function () {
+ return onUploadComplete(httpRequest, status, response, data);
+ })
+ );
+ },
+ onCancel: function (httpRequest, status, data) {
+ onUploadCancel(httpRequest, status, data);
+ deferred.resolve(false);
+ },
+ request: request,
+ item: item,
+ streams: [fis, bis]
+ }
+ );
+ channel.notificationCallbacks = listener;
+
+ var dispURI = uri.clone();
+ if (dispURI.password) {
+ dispURI.password = '********';
+ }
+ Zotero.debug("HTTP PUT of " + file.leafName + " to " + dispURI.spec);
+
+ channel.asyncOpen(listener, null);
+
+ return deferred.promise;
+ }),
+
+
+ _onUploadComplete: function (httpRequest, status, response, data) {
+ var request = data.request;
+ var item = data.item;
+ var url = httpRequest.name;
+
+ Zotero.debug("Upload of attachment " + item.key
+ + " finished with status code " + status);
+
+ switch (status) {
+ case 200:
+ case 201:
+ case 204:
+ break;
+
+ case 403:
+ case 500:
+ Zotero.debug(response);
+ throw (Zotero.getString('sync.storage.error.fileUploadFailed') +
+ " " + Zotero.getString('sync.storage.error.checkFileSyncSettings'));
+
+ case 507:
+ Zotero.debug(response);
+ throw Zotero.getString('sync.storage.error.webdav.insufficientSpace');
+
+ default:
+ Zotero.debug(response);
+ throw (Zotero.getString('sync.storage.error.fileUploadFailed') +
+ " " + Zotero.getString('sync.storage.error.checkFileSyncSettings')
+ + "\n\n" + "HTTP " + status);
+ }
+
+ return setStorageModificationTime(item)
+ .then(function (props) {
+ if (!request.isRunning()) {
+ Zotero.debug("Upload request '" + request.name
+ + "' is no longer running after getting mod time");
+ return false;
+ }
+
+ Zotero.DB.beginTransaction();
+
+ Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
+ Zotero.Sync.Storage.setSyncedModificationTime(item.id, props.mtime, true);
+ Zotero.Sync.Storage.setSyncedHash(item.id, props.hash);
+
+ Zotero.DB.commitTransaction();
+
+ try {
+ var file = Zotero.getTempDirectory();
+ file.append(item.key + '.zip');
+ file.remove(false);
+ }
+ catch (e) {
+ Components.utils.reportError(e);
+ }
+
+ return {
+ localChanges: true,
+ remoteChanges: true,
+ syncRequired: true
+ };
+ });
+ },
+
+
+ _onUploadCancel: function (httpRequest, status, data) {
+ var request = data.request;
+ var item = data.item;
+
+ Zotero.debug("Upload of attachment " + item.key + " cancelled with status code " + status);
+
+ try {
+ var file = Zotero.getTempDirectory();
+ file.append(item.key + '.zip');
+ file.remove(false);
+ }
+ catch (e) {
+ Components.utils.reportError(e);
+ }
+ },
+
+
+ /**
+ * Create a Zotero directory on the storage server
+ */
+ _createServerDirectory: function (callback) {
+ var uri = Zotero.Sync.Storage.WebDAV.rootURI;
+ Zotero.HTTP.WebDAV.doMkCol(uri, function (req) {
+ Zotero.debug(req.responseText);
+ Zotero.debug(req.status);
+
+ switch (req.status) {
+ case 201:
+ return [uri, Zotero.Sync.Storage.SUCCESS];
+
+ case 401:
+ return [uri, Zotero.Sync.Storage.ERROR_AUTH_FAILED];
+
+ case 403:
+ return [uri, Zotero.Sync.Storage.ERROR_FORBIDDEN];
+
+ case 405:
+ return [uri, Zotero.Sync.Storage.ERROR_NOT_ALLOWED];
+
+ case 500:
+ return [uri, Zotero.Sync.Storage.ERROR_SERVER_ERROR];
+
+ default:
+ return [uri, Zotero.Sync.Storage.ERROR_UNKNOWN];
+ }
+ });
+ },
+
+
+ /**
+ * Get the storage URI for an item
+ *
+ * @inner
+ * @param {Zotero.Item}
+ * @return {nsIURI} URI of file on storage server
+ */
+ _getItemURI: function (item) {
+ var uri = Zotero.Sync.Storage.WebDAV.rootURI;
+ uri.spec = uri.spec + item.key + '.zip';
+ return uri;
+ },
+
+
+ /**
+ * Get the storage property file URI for an item
+ *
+ * @inner
+ * @param {Zotero.Item}
+ * @return {nsIURI} URI of property file on storage server
+ */
+ _getItemPropertyURI: function (item) {
+ var uri = Zotero.Sync.Storage.WebDAV.rootURI;
+ uri.spec = uri.spec + item.key + '.prop';
+ return uri;
+ },
+
+
+ /**
+ * Get the storage property file URI corresponding to a given item storage URI
+ *
+ * @param {nsIURI} Item storage URI
+ * @return {nsIURI|FALSE} Property file URI, or FALSE if not an item storage URI
+ */
+ _getPropertyURIFromItemURI: function (uri) {
+ if (!uri.spec.match(/\.zip$/)) {
+ return false;
+ }
+ var propURI = uri.clone();
+ propURI.QueryInterface(Components.interfaces.nsIURL);
+ propURI.fileName = uri.fileName.replace(/\.zip$/, '.prop');
+ propURI.QueryInterface(Components.interfaces.nsIURI);
+ return propURI;
+ },
+
+
+ /**
+ * @inner
+ * @param {String[]} files Remote filenames to delete (e.g., ZIPs)
+ * @param {Function} callback Passed object containing three arrays:
+ * 'deleted', 'missing', and 'error',
+ * each containing filenames
+ */
+ _deleteStorageFiles: function (files) {
+ var results = {
+ deleted: [],
+ missing: [],
+ error: []
+ };
+
+ if (files.length == 0) {
+ return Zotero.Promise.resolve(results);
+ }
+
+ let deleteURI = _rootURI.clone();
+ // This should never happen, but let's be safe
+ if (!deleteURI.spec.match(/\/$/)) {
+ return Zotero.Promise.reject("Root URI does not end in slash in "
+ + "Zotero.Sync.Storage.WebDAV.deleteStorageFiles()");
+ }
+
+ var funcs = [];
+ for (let i=0; i<files.length; i++) {
+ let fileName = files[i];
+ let baseName = fileName.match(/^([^\.]+)/)[1];
+ funcs.push(function () {
+ let deleteURI = _rootURI.clone();
+ deleteURI.QueryInterface(Components.interfaces.nsIURL);
+ deleteURI.fileName = fileName;
+ deleteURI.QueryInterface(Components.interfaces.nsIURI);
+ return Zotero.HTTP.promise("DELETE", deleteURI, { successCodes: [200, 204, 404] })
+ .then(function (req) {
+ switch (req.status) {
+ case 204:
+ // IIS 5.1 and Sakai return 200
+ case 200:
+ var fileDeleted = true;
+ break;
+
+ case 404:
+ var fileDeleted = true;
+ break;
+ }
+
+ // If an item file URI, get the property URI
+ var deletePropURI = getPropertyURIFromItemURI(deleteURI);
+
+ // If we already deleted the prop file, skip it
+ if (!deletePropURI || results.deleted.indexOf(deletePropURI.fileName) != -1) {
+ if (fileDeleted) {
+ results.deleted.push(baseName);
+ }
+ else {
+ results.missing.push(baseName);
+ }
+ return;
+ }
+
+ let propFileName = deletePropURI.fileName;
+
+ // Delete property file
+ return Zotero.HTTP.promise("DELETE", deletePropURI, { successCodes: [200, 204, 404] })
+ .then(function (req) {
+ switch (req.status) {
+ case 204:
+ // IIS 5.1 and Sakai return 200
+ case 200:
+ results.deleted.push(baseName);
+ break;
+
+ case 404:
+ if (fileDeleted) {
+ results.deleted.push(baseName);
+ }
+ else {
+ results.missing.push(baseName);
+ }
+ break;
+ }
+ });
+ })
+ .catch(function (e) {
+ results.error.push(baseName);
+ throw e;
+ });
+ });
+ }
+
+ Components.utils.import("resource://zotero/concurrentCaller.js");
+ var caller = new ConcurrentCaller(4);
+ caller.stopOnError = true;
+ caller.setLogger(function (msg) Zotero.debug(msg));
+ caller.onError(function (e) Components.utils.reportError(e));
+ return caller.fcall(funcs)
+ .then(function () {
+ return results;
+ });
+ },
+
+
+ /**
+ * Checks for an invalid SSL certificate and throws a nice error
+ */
+ _checkResponse: function (req) {
+ var channel = req.channel;
+ if (!channel instanceof Ci.nsIChannel) {
+ Zotero.Sync.Storage.EventManager.error('No HTTPS channel available');
+ }
+
+ // Check if the error we encountered is really an SSL error
+ // Logic borrowed from https://developer.mozilla.org/en-US/docs/How_to_check_the_security_state_of_an_XMLHTTPRequest_over_SSL
+ // http://mxr.mozilla.org/mozilla-central/source/security/nss/lib/ssl/sslerr.h
+ // http://mxr.mozilla.org/mozilla-central/source/security/nss/lib/util/secerr.h
+ var secErrLimit = Ci.nsINSSErrorsService.NSS_SEC_ERROR_LIMIT - Ci.nsINSSErrorsService.NSS_SEC_ERROR_BASE;
+ var secErr = Math.abs(Ci.nsINSSErrorsService.NSS_SEC_ERROR_BASE) - (channel.status & 0xffff);
+ var sslErrLimit = Ci.nsINSSErrorsService.NSS_SSL_ERROR_LIMIT - Ci.nsINSSErrorsService.NSS_SSL_ERROR_BASE;
+ var sslErr = Math.abs(Ci.nsINSSErrorsService.NSS_SSL_ERROR_BASE) - (channel.status & 0xffff);
+ if( (secErr < 0 || secErr > secErrLimit) && (sslErr < 0 || sslErr > sslErrLimit) ) {
+ return;
+ }
+
+ var secInfo = channel.securityInfo;
+ if (secInfo instanceof Ci.nsITransportSecurityInfo) {
+ secInfo.QueryInterface(Ci.nsITransportSecurityInfo);
+ if ((secInfo.securityState & Ci.nsIWebProgressListener.STATE_IS_INSECURE) == Ci.nsIWebProgressListener.STATE_IS_INSECURE) {
+ var host = 'host';
+ try {
+ host = channel.URI.host;
+ }
+ catch (e) {
+ Zotero.debug(e);
+ }
+
+ var msg = Zotero.getString('sync.storage.error.webdav.sslCertificateError', host);
+ // In Standalone, provide cert_override.txt instructions and a
+ // button to open the Zotero profile directory
+ if (Zotero.isStandalone) {
+ msg += "\n\n" + Zotero.getString('sync.storage.error.webdav.seeCertOverrideDocumentation');
+ var buttonText = Zotero.getString('general.openDocumentation');
+ var func = function () {
+ var zp = Zotero.getActiveZoteroPane();
+ zp.loadURI("https://www.zotero.org/support/kb/cert_override", { shiftKey: true });
+ };
+ }
+ // In Firefox display a button to load the WebDAV URL
+ else {
+ msg += "\n\n" + Zotero.getString('sync.storage.error.webdav.loadURLForMoreInfo');
+ var buttonText = Zotero.getString('sync.storage.error.webdav.loadURL');
+ var func = function () {
+ var zp = Zotero.getActiveZoteroPane();
+ zp.loadURI(channel.URI.spec, { shiftKey: true });
+ };
+ }
+
+ var e = new Zotero.Error(
+ msg,
+ 0,
+ {
+ dialogText: msg,
+ dialogButtonText: buttonText,
+ dialogButtonCallback: func
+ }
+ );
+ throw e;
+ }
+ else if ((secInfo.securityState & Ci.nsIWebProgressListener.STATE_IS_BROKEN) == Ci.nsIWebProgressListener.STATE_IS_BROKEN) {
+ var msg = Zotero.getString('sync.storage.error.webdav.sslConnectionError', host) +
+ Zotero.getString('sync.storage.error.webdav.loadURLForMoreInfo');
+ var e = new Zotero.Error(
+ msg,
+ 0,
+ {
+ dialogText: msg,
+ dialogButtonText: Zotero.getString('sync.storage.error.webdav.loadURL'),
+ dialogButtonCallback: function () {
+ var zp = Zotero.getActiveZoteroPane();
+ zp.loadURI(channel.URI.spec, { shiftKey: true });
+ }
+ }
+ );
+ throw e;
+ }
+ }
+ }
+}
diff --git a/chrome/content/zotero/xpcom/storage/zfs.js b/chrome/content/zotero/xpcom/storage/zfs.js
index df52f547a..5c6e0ecc9 100644
--- a/chrome/content/zotero/xpcom/storage/zfs.js
+++ b/chrome/content/zotero/xpcom/storage/zfs.js
@@ -24,605 +24,874 @@
*/
-Zotero.Sync.Storage.ZFS = (function () {
- var _rootURI;
- var _userURI;
- var _headers = {
- "Zotero-API-Version" : ZOTERO_CONFIG.API_VERSION
- };
- var _cachedCredentials = false;
- var _s3Backoff = 1;
- var _s3ConsecutiveFailures = 0;
- var _maxS3Backoff = 60;
- var _maxS3ConsecutiveFailures = 5;
+Zotero.Sync.Storage.ZFS_Module = function (options) {
+ this.options = options;
+ this.apiClient = options.apiClient;
+
+ this._s3Backoff = 1;
+ this._s3ConsecutiveFailures = 0;
+ this._maxS3Backoff = 60;
+ this._maxS3ConsecutiveFailures = 5;
+};
+Zotero.Sync.Storage.ZFS_Module.prototype = {
+ name: "ZFS",
+ verified: true,
/**
- * Get file metadata on storage server
- *
- * @param {Zotero.Item} item
- * @param {Function} callback Callback f(item, etag)
+ * @return {Promise} A promise for the last sync time
*/
- function getStorageFileInfo(item, request) {
- var funcName = "Zotero.Sync.Storage.ZFS.getStorageFileInfo()";
+ getLastSyncTime: Zotero.Promise.coroutine(function* (libraryID) {
+ var params = this._getRequestParams(libraryID, "laststoragesync");
+ var uri = this.apiClient.buildRequestURI(params);
- return Zotero.HTTP.promise("GET", getItemInfoURI(item),
- {
- successCodes: [200, 404],
- headers: _headers,
- requestObserver: function (xmlhttp) {
- request.setChannel(xmlhttp.channel);
- }
- })
- .then(function (req) {
- if (req.status == 404) {
- return false;
- }
-
- var info = {};
- info.hash = req.getResponseHeader('ETag');
- if (!info.hash) {
- var msg = "Hash not found in info response in " + funcName
- + " (" + Zotero.Items.getLibraryKeyHash(item) + ")";
- Zotero.debug(msg, 1);
- Zotero.debug(req.status);
- Zotero.debug(req.responseText);
- Components.utils.reportError(msg);
- try {
- Zotero.debug(req.getAllResponseHeaders());
- }
- catch (e) {
- Zotero.debug("Response headers unavailable");
- }
- var msg = Zotero.getString('sync.storage.error.zfs.restart', Zotero.appName);
- throw msg;
- }
- info.filename = req.getResponseHeader('X-Zotero-Filename');
- var mtime = req.getResponseHeader('X-Zotero-Modification-Time');
- info.mtime = parseInt(mtime);
- info.compressed = req.getResponseHeader('X-Zotero-Compressed') == 'Yes';
- Zotero.debug(info);
-
- return info;
- })
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- if (e.xmlhttp.status == 0) {
- var msg = "Request cancelled getting storage file info";
- }
- else {
- var msg = "Unexpected status code " + e.xmlhttp.status
- + " getting storage file info for item " + item.libraryKey;
- }
- Zotero.debug(msg, 1);
- Zotero.debug(e.xmlhttp.responseText);
- Components.utils.reportError(msg);
- throw new Error(Zotero.Sync.Storage.defaultError);
- }
-
- throw e;
- });
- }
+ try {
+ let req = yield this.apiClient.makeRequest(
+ "GET", uri, { successCodes: [200, 404], debug: true }
+ );
+
+ // Not yet synced
+ if (req.status == 404) {
+ Zotero.debug("No last sync time for library " + libraryID);
+ return null;
+ }
+
+ let ts = req.responseText;
+ let date = new Date(ts * 1000);
+ Zotero.debug("Last successful ZFS sync for library " + libraryID + " was " + date);
+ return ts;
+ }
+ catch (e) {
+ Zotero.logError(e);
+ throw e;
+ }
+ }),
+
+
+ setLastSyncTime: Zotero.Promise.coroutine(function* (libraryID) {
+ var params = this._getRequestParams(libraryID, "laststoragesync");
+ var uri = this.apiClient.buildRequestURI(params);
+
+ try {
+ var req = yield this.apiClient.makeRequest(
+ "POST", uri, { successCodes: [200, 404], debug: true }
+ );
+ }
+ catch (e) {
+ var msg = "Unexpected status code " + e.xmlhttp.status + " setting last file sync time";
+ Zotero.logError(e);
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+
+ // Not yet synced
+ //
+ // TODO: Don't call this at all if no files uploaded
+ if (req.status == 404) {
+ return;
+ }
+
+ var time = req.responseText;
+ if (parseInt(time) != time) {
+ Zotero.logError(`Unexpected response ${time} setting last file sync time`);
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+ return parseInt(time);
+ }),
/**
- * Upload the file to the server
+ * Begin download process for individual file
*
- * @param {Object} Object with 'request' property
- * @return {void}
+ * @param {Zotero.Sync.Storage.Request} request
+ * @return {Promise<Boolean>} - True if file download, false if not
*/
- function processUploadFile(data) {
- /*
- updateSizeMultiplier(
- (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100
- );
- */
+ downloadFile: Zotero.Promise.coroutine(function* (request) {
+ var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
+ if (!item) {
+ throw new Error("Item '" + request.name + "' not found");
+ }
- var request = data.request;
- var item = Zotero.Sync.Storage.getItemFromRequestName(request.name);
- return getStorageFileInfo(item, request)
- .then(function (info) {
- if (request.isFinished()) {
- Zotero.debug("Upload request '" + request.name
- + "' is no longer running after getting file info");
- return false;
- }
-
- // Check for conflict
- if (Zotero.Sync.Storage.getSyncState(item.id)
- != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) {
- if (info) {
- // Remote mod time
- var mtime = info.mtime;
- // Local file time
- var fmtime = item.attachmentModificationTime;
-
- var same = false;
- var useLocal = false;
- if (fmtime == mtime) {
- same = true;
- Zotero.debug("File mod time matches remote file -- skipping upload");
- }
- // Allow floored timestamps for filesystems that don't support
- // millisecond precision (e.g., HFS+)
- else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
- same = true;
- Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
- + "-- skipping upload");
- }
- // Allow timestamp to be exactly one hour off to get around
- // time zone issues -- there may be a proper way to fix this
- else if (Math.abs(fmtime - mtime) == 3600000
- // And check with one-second precision as well
- || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
- || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
- same = true;
- Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
- + "-- assuming time zone issue and skipping upload");
- }
- // Ignore maxed-out 32-bit ints, from brief problem after switch to 32-bit servers
- else if (mtime == 2147483647) {
- Zotero.debug("Remote mod time is invalid -- uploading local file version");
- useLocal = true;
- }
-
- if (same) {
- Zotero.debug(Zotero.Sync.Storage.getSyncedModificationTime(item.id));
-
- Zotero.DB.beginTransaction();
- var syncState = Zotero.Sync.Storage.getSyncState(item.id);
- //Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true);
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- Zotero.DB.commitTransaction();
- return {
- localChanges: true,
- remoteChanges: false
- };
- }
-
- var smtime = Zotero.Sync.Storage.getSyncedModificationTime(item.id);
- if (!useLocal && smtime != mtime) {
- Zotero.debug("Conflict -- last synced file mod time "
- + "does not match time on storage server"
- + " (" + smtime + " != " + mtime + ")");
- return {
- localChanges: false,
- remoteChanges: false,
- conflict: {
- local: { modTime: fmtime },
- remote: { modTime: mtime }
- }
- };
- }
- }
- else {
- Zotero.debug("Remote file not found for item " + item.libraryID + "/" + item.key);
- }
- }
-
- return getFileUploadParameters(
- item,
- function (item, target, uploadKey, params) {
- return postFile(request, item, target, uploadKey, params);
- },
- function () {
- updateItemFileInfo(item);
- return {
- localChanges: true,
- remoteChanges: false
- };
- }
- );
+ var path = item.getFilePath();
+ if (!path) {
+ Zotero.debug(`Cannot download file for attachment ${item.libraryKey} with no path`);
+ return new Zotero.Sync.Storage.Result;
+ }
+
+ var destPath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.tmp');
+
+ // saveURI() below appears not to create empty files for Content-Length: 0,
+ // so we create one here just in case, which also lets us check file access
+ try {
+ let file = yield OS.File.open(destPath, {
+ truncate: true
});
- }
-
-
- /**
- * Get mod time of file on storage server
- *
- * @param {Zotero.Item} item
- * @param {Function} uploadCallback Callback f(request, item, target, params)
- * @param {Function} existsCallback Callback f() to call when file already exists
- * on server and uploading isn't necessary
- */
- function getFileUploadParameters(item, uploadCallback, existsCallback) {
- var funcName = "Zotero.Sync.Storage.ZFS.getFileUploadParameters()";
-
- var uri = getItemURI(item);
-
- if (Zotero.Attachments.getNumFiles(item) > 1) {
- var file = Zotero.getTempDirectory();
- var filename = item.key + '.zip';
- file.append(filename);
- uri.spec = uri.spec;
- var zip = true;
+ file.close();
}
- else {
- var file = item.getFile();
- var filename = file.leafName;
- var zip = false;
+ catch (e) {
+ Zotero.File.checkFileAccessError(e, destPath, 'create');
}
- var mtime = item.attachmentModificationTime;
- var hash = Zotero.Utilities.Internal.md5(file);
-
- var body = "md5=" + hash + "&filename=" + encodeURIComponent(filename)
- + "&filesize=" + file.fileSize + "&mtime=" + mtime;
- if (zip) {
- body += "&zip=1";
- }
-
- return Zotero.HTTP.promise("POST", uri, { body: body, headers: _headers, debug: true })
- .then(function (req) {
- if (!req.responseXML) {
- throw new Error("Invalid response retrieving file upload parameters");
- }
-
- var rootTag = req.responseXML.documentElement.tagName;
-
- if (rootTag != 'upload' && rootTag != 'exists') {
- throw new Error("Invalid response retrieving file upload parameters");
- }
-
- // File was already available, so uploading isn't required
- if (rootTag == 'exists') {
- return existsCallback();
- }
-
- var url = req.responseXML.getElementsByTagName('url')[0].textContent;
- var uploadKey = req.responseXML.getElementsByTagName('key')[0].textContent;
- var params = {}, p = '';
- var paramNodes = req.responseXML.getElementsByTagName('params')[0].childNodes;
- for (var i = 0; i < paramNodes.length; i++) {
- params[paramNodes[i].tagName] = paramNodes[i].textContent;
- }
- return uploadCallback(item, url, uploadKey, params);
- })
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- if (e.status == 413) {
- var retry = e.xmlhttp.getResponseHeader('Retry-After');
- if (retry) {
- var minutes = Math.round(retry / 60);
- var e = new Zotero.Error(
- Zotero.getString('sync.storage.error.zfs.tooManyQueuedUploads', minutes),
- "ZFS_UPLOAD_QUEUE_LIMIT"
- );
- throw e;
- }
-
- var text, buttonText = null, buttonCallback;
-
- // Group file
- if (item.libraryID) {
- var group = Zotero.Groups.getByLibraryID(item.libraryID);
- text = Zotero.getString('sync.storage.error.zfs.groupQuotaReached1', group.name) + "\n\n"
- + Zotero.getString('sync.storage.error.zfs.groupQuotaReached2');
- }
- // Personal file
- else {
- text = Zotero.getString('sync.storage.error.zfs.personalQuotaReached1') + "\n\n"
- + Zotero.getString('sync.storage.error.zfs.personalQuotaReached2');
- buttonText = Zotero.getString('sync.storage.openAccountSettings');
- buttonCallback = function () {
- var url = "https://www.zotero.org/settings/storage";
-
- var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
- .getService(Components.interfaces.nsIWindowMediator);
- var win = wm.getMostRecentWindow("navigator:browser");
- win.ZoteroPane.loadURI(url);
- }
- }
-
- text += "\n\n" + filename + " (" + Math.round(file.fileSize / 1024) + "KB)";
-
- var e = new Zotero.Error(
- Zotero.getString('sync.storage.error.zfs.fileWouldExceedQuota', filename),
- "ZFS_OVER_QUOTA",
- {
- dialogText: text,
- dialogButtonText: buttonText,
- dialogButtonCallback: buttonCallback
- }
- );
- e.errorType = 'warning';
- Zotero.debug(e, 2);
- Components.utils.reportError(e);
- throw e;
- }
- else if (e.status == 403) {
- var groupID = Zotero.Groups.getGroupIDFromLibraryID(item.libraryID);
- var e = new Zotero.Error(
- "File editing denied for group",
- "ZFS_FILE_EDITING_DENIED",
- {
- groupID: groupID
- }
- );
- throw e;
- }
- else if (e.status == 404) {
- Components.utils.reportError("Unexpected status code 404 in " + funcName
- + " (" + Zotero.Items.getLibraryKeyHash(item) + ")");
- if (Zotero.Prefs.get('sync.debugNoAutoResetClient')) {
- Components.utils.reportError("Skipping automatic client reset due to debug pref");
- return;
- }
- if (!Zotero.Sync.Server.canAutoResetClient) {
- Components.utils.reportError("Client has already been auto-reset -- manual sync required");
- return;
- }
- Zotero.Sync.Server.resetClient();
- Zotero.Sync.Server.canAutoResetClient = false;
- throw new Error(Zotero.Sync.Storage.defaultError);
- }
-
- var msg = "Unexpected status code " + e.status + " in " + funcName
- + " (" + Zotero.Items.getLibraryKeyHash(item) + ")";
- Zotero.debug(msg, 1);
- Zotero.debug(e.xmlhttp.getAllResponseHeaders());
- Components.utils.reportError(msg);
- throw new Error(Zotero.Sync.Storage.defaultError);
- }
-
- throw e;
- });
- }
-
-
- function postFile(request, item, url, uploadKey, params) {
- if (request.isFinished()) {
- Zotero.debug("Upload request " + request.name + " is no longer running after getting upload parameters");
- return false;
- }
-
- var file = getUploadFile(item);
-
- // TODO: make sure this doesn't appear in file
- var boundary = "---------------------------" + Math.random().toString().substr(2);
-
- var mis = Components.classes["@mozilla.org/io/multiplex-input-stream;1"]
- .createInstance(Components.interfaces.nsIMultiplexInputStream);
-
- // Add parameters
- for (var key in params) {
- var storage = Components.classes["@mozilla.org/storagestream;1"]
- .createInstance(Components.interfaces.nsIStorageStream);
- storage.init(4096, 4294967295, null); // PR_UINT32_MAX
- var out = storage.getOutputStream(0);
-
- var conv = Components.classes["@mozilla.org/intl/converter-output-stream;1"]
- .createInstance(Components.interfaces.nsIConverterOutputStream);
- conv.init(out, null, 4096, "?");
-
- var str = "--" + boundary + '\r\nContent-Disposition: form-data; name="' + key + '"'
- + '\r\n\r\n' + params[key] + '\r\n';
- conv.writeString(str);
- conv.close();
-
- var instr = storage.newInputStream(0);
- mis.appendStream(instr);
- }
-
- // Add file
- var sis = Components.classes["@mozilla.org/io/string-input-stream;1"]
- .createInstance(Components.interfaces.nsIStringInputStream);
- var str = "--" + boundary + '\r\nContent-Disposition: form-data; name="file"\r\n\r\n';
- sis.setData(str, -1);
- mis.appendStream(sis);
-
- var fis = Components.classes["@mozilla.org/network/file-input-stream;1"]
- .createInstance(Components.interfaces.nsIFileInputStream);
- fis.init(file, 0x01, 0, Components.interfaces.nsIFileInputStream.CLOSE_ON_EOF
- | Components.interfaces.nsIFileInputStream.REOPEN_ON_REWIND);
-
- var bis = Components.classes["@mozilla.org/network/buffered-input-stream;1"]
- .createInstance(Components.interfaces.nsIBufferedInputStream)
- bis.init(fis, 64 * 1024);
- mis.appendStream(bis);
-
- // End request
- var sis = Components.classes["@mozilla.org/io/string-input-stream;1"]
- .createInstance(Components.interfaces.nsIStringInputStream);
- var str = "\r\n--" + boundary + "--";
- sis.setData(str, -1);
- mis.appendStream(sis);
-
-
- /* var cstream = Components.classes["@mozilla.org/intl/converter-input-stream;1"].
- createInstance(Components.interfaces.nsIConverterInputStream);
- cstream.init(mis, "UTF-8", 0, 0); // you can use another encoding here if you wish
-
- let (str = {}) {
- cstream.readString(-1, str); // read the whole file and put it in str.value
- data = str.value;
- }
- cstream.close(); // this closes fstream
- alert(data);
- */
-
- var ios = Components.classes["@mozilla.org/network/io-service;1"].
- getService(Components.interfaces.nsIIOService);
- var uri = ios.newURI(url, null, null);
- var channel = ios.newChannelFromURI(uri);
-
- channel.QueryInterface(Components.interfaces.nsIUploadChannel);
- channel.setUploadStream(mis, "multipart/form-data", -1);
- channel.QueryInterface(Components.interfaces.nsIHttpChannel);
- channel.requestMethod = 'POST';
- channel.allowPipelining = false;
- channel.setRequestHeader('Keep-Alive', '', false);
- channel.setRequestHeader('Connection', '', false);
- channel.setRequestHeader("Content-Type", "multipart/form-data; boundary=" + boundary, false);
- //channel.setRequestHeader('Date', date, false);
-
- request.setChannel(channel);
-
var deferred = Zotero.Promise.defer();
+ var requestData = {item};
var listener = new Zotero.Sync.Storage.StreamListener(
{
- onProgress: function (a, b, c) {
- request.onProgress(a, b, c);
+ onStart: function (req) {
+ if (request.isFinished()) {
+ Zotero.debug("Download request " + request.name
+ + " stopped before download started -- closing channel");
+ req.cancel(Components.results.NS_BINDING_ABORTED);
+ deferred.resolve(false);
+ }
},
- onStop: function (httpRequest, status, response, data) {
- data.request.setChannel(false);
+ onChannelRedirect: Zotero.Promise.coroutine(function* (oldChannel, newChannel, flags) {
+ // These will be used in processDownload() if the download succeeds
+ oldChannel.QueryInterface(Components.interfaces.nsIHttpChannel);
- // For timeouts and failures from S3, which happen intermittently,
- // wait a little and try again
- let timeoutMessage = "Your socket connection to the server was not read from or "
- + "written to within the timeout period.";
- if (status == 0 || (status == 400 && response.indexOf(timeoutMessage) != -1)) {
- if (_s3ConsecutiveFailures >= _maxS3ConsecutiveFailures) {
- Zotero.debug(_s3ConsecutiveFailures
- + " consecutive S3 failures -- aborting", 1);
- _s3ConsecutiveFailures = 0;
+ Zotero.debug("CHANNEL HERE FOR " + item.libraryKey + " WITH " + oldChannel.status);
+ Zotero.debug(oldChannel.URI.spec);
+ Zotero.debug(newChannel.URI.spec);
+
+ var header;
+ try {
+ header = "Zotero-File-Modification-Time";
+ requestData.mtime = oldChannel.getResponseHeader(header);
+ header = "Zotero-File-MD5";
+ requestData.md5 = oldChannel.getResponseHeader(header);
+ header = "Zotero-File-Compressed";
+ requestData.compressed = oldChannel.getResponseHeader(header) == 'Yes';
+ }
+ catch (e) {
+ deferred.reject(new Error(`${header} header not set in file request for ${item.libraryKey}`));
+ return false;
+ }
+
+ if (!(yield OS.File.exists(path))) {
+ return true;
+ }
+
+ var updateHash = false;
+ var fileModTime = yield item.attachmentModificationTime;
+ if (requestData.mtime == fileModTime) {
+ Zotero.debug("File mod time matches remote file -- skipping download of "
+ + item.libraryKey);
+ }
+ // If not compressed, check hash, in case only timestamp changed
+ else if (!requestData.compressed && (yield item.attachmentHash) == requestData.md5) {
+ Zotero.debug("File hash matches remote file -- skipping download of "
+ + item.libraryKey);
+ updateHash = true;
+ }
+ else {
+ return true;
+ }
+
+ // Update local metadata and stop request, skipping file download
+ yield Zotero.DB.executeTransaction(function* () {
+ if (updateHash) {
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, requestData.md5);
}
- else {
- let libraryKey = Zotero.Items.getLibraryKeyHash(item);
- let msg = "S3 returned " + status
- + " (" + libraryKey + ") -- retrying upload"
- Components.utils.reportError(msg);
- Zotero.debug(msg, 1);
- Zotero.debug(response, 1);
- if (_s3Backoff < _maxS3Backoff) {
- _s3Backoff *= 2;
- }
- _s3ConsecutiveFailures++;
- Zotero.debug("Delaying " + libraryKey + " upload for "
- + _s3Backoff + " seconds", 2);
- Q.delay(_s3Backoff * 1000)
- .then(function () {
- deferred.resolve(postFile(request, item, url, uploadKey, params));
- });
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
+ item.id, requestData.mtime
+ );
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+ return false;
+ }),
+ onProgress: function (a, b, c) {
+ request.onProgress(a, b, c)
+ },
+ onStop: function (req, status, res) {
+ request.setChannel(false);
+
+ if (status != 200) {
+ if (status == 404) {
+ Zotero.debug("Remote file not found for item " + item.libraryKey);
+ deferred.resolve(new Zotero.Sync.Storage.Result);
return;
}
+
+ // If S3 connection is interrupted, delay and retry, or bail if too many
+ // consecutive failures
+ if (status == 0) {
+ if (this._s3ConsecutiveFailures < this._maxS3ConsecutiveFailures) {
+ let libraryKey = item.libraryKey;
+ let msg = "S3 returned 0 for " + libraryKey + " -- retrying download"
+ Components.utils.reportError(msg);
+ Zotero.debug(msg, 1);
+ if (this._s3Backoff < this._maxS3Backoff) {
+ this._s3Backoff *= 2;
+ }
+ this._s3ConsecutiveFailures++;
+ Zotero.debug("Delaying " + libraryKey + " download for "
+ + this._s3Backoff + " seconds", 2);
+ Zotero.Promise.delay(this._s3Backoff * 1000)
+ .then(function () {
+ deferred.resolve(this._downloadFile(request));
+ });
+ return;
+ }
+
+ Zotero.debug(this._s3ConsecutiveFailures
+ + " consecutive S3 failures -- aborting", 1);
+ this._s3ConsecutiveFailures = 0;
+ }
+
+ var msg = "Unexpected status code " + status + " for GET " + uri;
+ Zotero.debug(msg, 1);
+ Components.utils.reportError(msg);
+ // Output saved content, in case an error was captured
+ try {
+ let sample = Zotero.File.getContents(destPath, null, 4096);
+ if (sample) {
+ Zotero.debug(sample, 1);
+ }
+ }
+ catch (e) {
+ Zotero.debug(e, 1);
+ }
+ deferred.reject(new Error(Zotero.Sync.Storage.defaultError));
+ return;
}
- deferred.resolve(onUploadComplete(httpRequest, status, response, data));
- },
- onCancel: function (httpRequest, status, data) {
- onUploadCancel(httpRequest, status, data)
- deferred.resolve(false);
- },
- request: request,
- item: item,
- uploadKey: uploadKey,
- streams: [mis]
+ // Don't try to process if the request has been cancelled
+ if (request.isFinished()) {
+ Zotero.debug("Download request " + request.name
+ + " is no longer running after file download", 2);
+ deferred.resolve(false);
+ return;
+ }
+
+ Zotero.debug("Finished download of " + destPath);
+
+ try {
+ deferred.resolve(
+ Zotero.Sync.Storage.Local.processDownload(requestData)
+ );
+ }
+ catch (e) {
+ Zotero.debug("REJECTING");
+ deferred.reject(e);
+ }
+ }.bind(this),
+ onCancel: function (req, status) {
+ Zotero.debug("Request cancelled");
+ if (deferred.promise.isPending()) {
+ deferred.resolve(false);
+ }
+ }
}
);
- channel.notificationCallbacks = listener;
- var dispURI = uri.clone();
- if (dispURI.password) {
- dispURI.password = '********';
- }
- Zotero.debug("HTTP POST of " + file.leafName + " to " + dispURI.spec);
+ var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`);
+ var uri = this.apiClient.buildRequestURI(params);
+ var headers = this.apiClient.getHeaders();
- channel.asyncOpen(listener, null);
+ Zotero.debug('Saving ' + uri);
+ const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
+ var wbp = Components
+ .classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
+ .createInstance(nsIWBP);
+ wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE;
+ wbp.progressListener = listener;
+ Zotero.Utilities.Internal.saveURI(wbp, uri, destPath, headers);
return deferred.promise;
- }
+ }),
- function onUploadComplete(httpRequest, status, response, data) {
- return Q.try(function () {
- var request = data.request;
- var item = data.item;
- var uploadKey = data.uploadKey;
-
- Zotero.debug("Upload of attachment " + item.key
- + " finished with status code " + status);
-
- Zotero.debug(response);
-
- switch (status) {
- case 201:
- // Decrease backoff delay on successful upload
- if (_s3Backoff > 1) {
- _s3Backoff /= 2;
- }
- // And reset consecutive failures
- _s3ConsecutiveFailures = 0;
- break;
-
- case 500:
- throw new Error("File upload failed. Please try again.");
-
- default:
- var msg = "Unexpected file upload status " + status
- + " in Zotero.Sync.Storage.ZFS.onUploadComplete()"
- + " (" + Zotero.Items.getLibraryKeyHash(item) + ")";
- Zotero.debug(msg, 1);
- Components.utils.reportError(msg);
- Components.utils.reportError(response);
- throw new Error(Zotero.Sync.Storage.defaultError);
+ uploadFile: Zotero.Promise.coroutine(function* (request) {
+ var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
+ if (yield Zotero.Attachments.hasMultipleFiles(item)) {
+ let created = yield Zotero.Sync.Storage.Utilities.createUploadFile(request);
+ if (!created) {
+ return new Zotero.Sync.Storage.Result;
}
-
- var uri = getItemURI(item);
- var body = "update=" + uploadKey + "&mtime=" + item.attachmentModificationTime;
-
- // Register upload on server
- return Zotero.HTTP.promise("POST", uri, { body: body, headers: _headers, successCodes: [204] })
- .then(function (req) {
- updateItemFileInfo(item);
- return {
- localChanges: true,
- remoteChanges: true
- };
- })
- .catch(function (e) {
- var msg = "Unexpected file registration status " + e.status
- + " (" + Zotero.Items.getLibraryKeyHash(item) + ")";
- Zotero.debug(msg, 1);
- Zotero.debug(e.xmlhttp.responseText);
- Zotero.debug(e.xmlhttp.getAllResponseHeaders());
- Components.utils.reportError(msg);
- Components.utils.reportError(e.xmlhttp.responseText);
- throw new Error(Zotero.Sync.Storage.defaultError);
- });
- });
- }
+ return this._processUploadFile(request);
+ }
+ return this._processUploadFile(request);
+ }),
- function updateItemFileInfo(item) {
- // Mark as changed locally
- Zotero.DB.beginTransaction();
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
-
- // Store file mod time
- var mtime = item.attachmentModificationTime;
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, mtime, true);
-
- // Store file hash of individual files
- if (Zotero.Attachments.getNumFiles(item) == 1) {
- var hash = item.attachmentHash;
- Zotero.Sync.Storage.setSyncedHash(item.id, hash);
+ /**
+ * Remove all synced files from the server
+ */
+ purgeDeletedStorageFiles: Zotero.Promise.coroutine(function* () {
+ var sql = "SELECT value FROM settings WHERE setting=? AND key=?";
+ var values = yield Zotero.DB.columnQueryAsync(sql, ['storage', 'zfsPurge']);
+ if (!values) {
+ return false;
}
- Zotero.DB.commitTransaction();
+ Zotero.debug("Unlinking synced files on ZFS");
+
+ var uri = this.userURI;
+ uri.spec += "removestoragefiles?";
+ // Unused
+ for each(var value in values) {
+ switch (value) {
+ case 'user':
+ uri.spec += "user=1&";
+ break;
+
+ case 'group':
+ uri.spec += "group=1&";
+ break;
+
+ default:
+ throw new Error("Invalid zfsPurge value '" + value + "'");
+ }
+ }
+ uri.spec = uri.spec.substr(0, uri.spec.length - 1);
+
+ yield Zotero.HTTP.request("POST", uri, "");
+
+ var sql = "DELETE FROM settings WHERE setting=? AND key=?";
+ yield Zotero.DB.queryAsync(sql, ['storage', 'zfsPurge']);
+ }),
+
+
+ //
+ // Private methods
+ //
+ _getRequestParams: function (libraryID, target) {
+ var library = Zotero.Libraries.get(libraryID);
+ return {
+ libraryType: library.libraryType,
+ libraryTypeID: library.libraryTypeID,
+ target
+ };
+ },
+
+
+ /**
+ * Get authorization from API for uploading file
+ *
+ * @param {Zotero.Item} item
+ * @return {Object|String} - Object with upload params or 'exists'
+ */
+ _getFileUploadParameters: Zotero.Promise.coroutine(function* (item) {
+ var funcName = "Zotero.Sync.Storage.ZFS._getFileUploadParameters()";
+
+ var path = item.getFilePath();
+ var filename = OS.Path.basename(path);
+ var zip = yield Zotero.Attachments.hasMultipleFiles(item);
+ if (zip) {
+ var uploadPath = OS.Path.join(Zotero.getTempDirectory().path, item.key + '.zip');
+ }
+ else {
+ var uploadPath = path;
+ }
+
+ var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`);
+ var uri = this.apiClient.buildRequestURI(params);
+
+ // TODO: One-step uploads
+ /*var headers = {
+ "Content-Type": "application/json"
+ };
+ var storedHash = yield Zotero.Sync.Storage.Local.getSyncedHash(item.id);
+ //var storedModTime = yield Zotero.Sync.Storage.getSyncedModificationTime(item.id);
+ if (storedHash) {
+ headers["If-Match"] = storedHash;
+ }
+ else {
+ headers["If-None-Match"] = "*";
+ }
+ var mtime = yield item.attachmentModificationTime;
+ var hash = Zotero.Utilities.Internal.md5(file);
+ var json = {
+ md5: hash,
+ mtime,
+ filename,
+ size: file.fileSize
+ };
+ var charset = item.attachmentCharset;
+ var contentType = item.attachmentContentType;
+ if (charset) {
+ json.charset = charset;
+ }
+ if (contentType) {
+ json.contentType = contentType;
+ }
+ if (zip) {
+ json.zip = true;
+ }
try {
- if (Zotero.Attachments.getNumFiles(item) > 1) {
+ var req = yield this.apiClient.makeRequest(
+ "POST", uri, { body: JSON.stringify(json), headers, debug: true }
+ );
+ }*/
+
+ var headers = {
+ "Content-Type": "application/x-www-form-urlencoded"
+ };
+ var storedHash = yield Zotero.Sync.Storage.Local.getSyncedHash(item.id);
+ //var storedModTime = yield Zotero.Sync.Storage.getSyncedModificationTime(item.id);
+ if (storedHash) {
+ headers["If-Match"] = storedHash;
+ }
+ else {
+ headers["If-None-Match"] = "*";
+ }
+
+ // Build POST body
+ var mtime = yield item.attachmentModificationTime;
+ var params = {
+ md5: yield item.attachmentHash,
+ mtime,
+ filename,
+ filesize: (yield OS.File.stat(uploadPath)).size
+ };
+ var charset = item.attachmentCharset;
+ var contentType = item.attachmentContentType;
+ if (charset) {
+ params.charset = charset;
+ }
+ if (contentType) {
+ params.contentType = contentType;
+ }
+ if (zip) {
+ params.zipMD5 = yield Zotero.Utilities.Internal.md5Async(uploadPath);
+ params.zipFilename = OS.Path.basename(uploadPath);
+ }
+ var body = [];
+ for (let i in params) {
+ body.push(i + "=" + encodeURIComponent(params[i]));
+ }
+ body = body.join('&');
+
+ try {
+ var req = yield this.apiClient.makeRequest(
+ "POST",
+ uri,
+ {
+ body,
+ headers,
+ // This should include all errors in _handleUploadAuthorizationFailure()
+ successCodes: [200, 201, 204, 403, 404, 412, 413],
+ debug: true
+ }
+ );
+ }
+ catch (e) {
+ if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
+ let msg = "Unexpected status code " + e.status + " in " + funcName
+ + " (" + item.libraryKey + ")";
+ Zotero.logError(msg);
+ Zotero.debug(e.xmlhttp.getAllResponseHeaders());
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+ throw e;
+ }
+
+ var result = yield this._handleUploadAuthorizationFailure(req, item);
+ if (result instanceof Zotero.Sync.Storage.Result) {
+ return result;
+ }
+
+ try {
+ var json = JSON.parse(req.responseText);
+ }
+ catch (e) {
+ Zotero.logError(e);
+ Zotero.debug(req.responseText, 1);
+ }
+ if (!json) {
+ throw new Error("Invalid response retrieving file upload parameters");
+ }
+
+ if (!json.uploadKey && !json.exists) {
+ throw new Error("Invalid response retrieving file upload parameters");
+ }
+
+ if (json.exists) {
+ let version = req.getResponseHeader('Last-Modified-Version');
+ if (!version) {
+ throw new Error("Last-Modified-Version not provided");
+ }
+ json.version = version;
+ }
+
+ Zotero.debug('=-=-=--=');
+ Zotero.debug(json);
+
+ // TEMP
+ //
+ // Passed through to _updateItemFileInfo()
+ json.mtime = mtime;
+ json.md5 = params.md5;
+ if (storedHash) {
+ json.storedHash = storedHash;
+ }
+
+ return json;
+ }),
+
+
+ /**
+ * Handle known errors from upload authorization request
+ *
+ * These must be included in successCodes in _getFileUploadParameters()
+ */
+ _handleUploadAuthorizationFailure: Zotero.Promise.coroutine(function* (req, item) {
+ //
+ // These must be included in successCodes above.
+ // TODO: 429?
+ if (req.status == 403) {
+ let groupID = Zotero.Groups.getGroupIDFromLibraryID(item.libraryID);
+ let e = new Zotero.Error(
+ "File editing denied for group",
+ "ZFS_FILE_EDITING_DENIED",
+ {
+ groupID: groupID
+ }
+ );
+ throw e;
+ }
+ else if (req.status == 404) {
+ Components.utils.reportError("Unexpected status code 404 in upload authorization "
+ + "request (" + item.libraryKey + ")");
+ if (Zotero.Prefs.get('sync.debugNoAutoResetClient')) {
+ Components.utils.reportError("Skipping automatic client reset due to debug pref");
+ }
+ if (!Zotero.Sync.Server.canAutoResetClient) {
+ Components.utils.reportError("Client has already been auto-reset -- manual sync required");
+ }
+
+ // TODO: Make an API request to fix this
+
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+ else if (req.status == 412) {
+ Zotero.debug("412 BUT WE'RE COOL");
+ let version = req.getResponseHeader('Last-Modified-Version');
+ if (!version) {
+ throw new Error("Last-Modified-Version header not provided");
+ }
+ if (version > item.version) {
+ return new Zotero.Sync.Storage.Result({
+ syncRequired: true
+ });
+ }
+ if (version < item.version) {
+ throw new Error("Last-Modified-Version is lower than item version "
+ + `(${version} < ${item.version})`);
+ }
+
+ // Get updated item metadata
+ let library = Zotero.Libraries.get(item.libraryID);
+ let json = yield this.apiClient.downloadObjects(
+ library.libraryType,
+ library.libraryTypeID,
+ 'item',
+ [item.key]
+ )[0];
+ if (!Array.isArray(json)) {
+ Zotero.logError(json);
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+ if (json.length > 1) {
+ throw new Error("More than one result for item lookup");
+ }
+
+ yield Zotero.Sync.Data.Local.saveCacheObjects('item', item.libraryID, json);
+ json = json[0];
+
+ if (json.data.version > item.version) {
+ return new Zotero.Sync.Storage.Result({
+ syncRequired: true
+ });
+ }
+
+ let fileHash = yield item.attachmentHash;
+ let fileModTime = yield item.attachmentModificationTime;
+
+ Zotero.debug("MD5");
+ Zotero.debug(json.data.md5);
+ Zotero.debug(fileHash);
+
+ if (json.data.md5 == fileHash) {
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
+ item.id, fileModTime
+ );
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, fileHash);
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+ return new Zotero.Sync.Storage.Result;
+ }
+
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ );
+ return new Zotero.Sync.Storage.Result({
+ fileSyncRequired: true
+ });
+ }
+ else if (req.status == 413) {
+ let retry = req.getResponseHeader('Retry-After');
+ if (retry) {
+ let minutes = Math.round(retry / 60);
+ throw new Zotero.Error(
+ Zotero.getString('sync.storage.error.zfs.tooManyQueuedUploads', minutes),
+ "ZFS_UPLOAD_QUEUE_LIMIT"
+ );
+ }
+
+ let text, buttonText = null, buttonCallback;
+
+ // Group file
+ if (item.libraryID) {
+ var group = Zotero.Groups.getByLibraryID(item.libraryID);
+ text = Zotero.getString('sync.storage.error.zfs.groupQuotaReached1', group.name) + "\n\n"
+ + Zotero.getString('sync.storage.error.zfs.groupQuotaReached2');
+ }
+ // Personal file
+ else {
+ text = Zotero.getString('sync.storage.error.zfs.personalQuotaReached1') + "\n\n"
+ + Zotero.getString('sync.storage.error.zfs.personalQuotaReached2');
+ buttonText = Zotero.getString('sync.storage.openAccountSettings');
+ buttonCallback = function () {
+ var url = "https://www.zotero.org/settings/storage";
+
+ var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
+ .getService(Components.interfaces.nsIWindowMediator);
+ var win = wm.getMostRecentWindow("navigator:browser");
+ win.ZoteroPane.loadURI(url);
+ }
+ }
+
+ text += "\n\n" + filename + " (" + Math.round(file.fileSize / 1024) + "KB)";
+
+ let e = new Zotero.Error(
+ Zotero.getString('sync.storage.error.zfs.fileWouldExceedQuota', filename),
+ "ZFS_OVER_QUOTA",
+ {
+ dialogText: text,
+ dialogButtonText: buttonText,
+ dialogButtonCallback: buttonCallback
+ }
+ );
+ e.errorType = 'warning';
+ Zotero.debug(e, 2);
+ Components.utils.reportError(e);
+ throw e;
+ }
+ }),
+
+
+ /**
+ * Given parameters from authorization, upload file to S3
+ */
+ _uploadFile: Zotero.Promise.coroutine(function* (request, item, params) {
+ if (request.isFinished()) {
+ Zotero.debug("Upload request " + request.name + " is no longer running after getting "
+ + "upload parameters");
+ return new Zotero.Sync.Storage.Result;
+ }
+
+ var file = yield this._getUploadFile(item);
+
+ Components.utils.importGlobalProperties(["File"]);
+ file = File(file);
+
+ var blob = new Blob([params.prefix, file, params.suffix]);
+
+ try {
+ var req = yield Zotero.HTTP.request(
+ "POST",
+ params.url,
+ {
+ headers: {
+ "Content-Type": params.contentType
+ },
+ body: blob,
+ requestObserver: function (req) {
+ request.setChannel(req.channel);
+ req.upload.addEventListener("progress", function (event) {
+ if (event.lengthComputable) {
+ request.onProgress(event.loaded, event.total);
+ }
+ });
+ },
+ debug: true,
+ successCodes: [201]
+ }
+ );
+ }
+ catch (e) {
+ // For timeouts and failures from S3, which happen intermittently,
+ // wait a little and try again
+ let timeoutMessage = "Your socket connection to the server was not read from or "
+ + "written to within the timeout period.";
+ if (e.status == 0
+ || (e.status == 400 && e.xmlhttp.responseText.indexOf(timeoutMessage) != -1)) {
+ if (this._s3ConsecutiveFailures >= this._maxS3ConsecutiveFailures) {
+ Zotero.debug(this._s3ConsecutiveFailures
+ + " consecutive S3 failures -- aborting", 1);
+ this._s3ConsecutiveFailures = 0;
+ }
+ else {
+ let msg = "S3 returned " + e.status + " (" + item.libraryKey + ") "
+ + "-- retrying upload"
+ Zotero.logError(msg);
+ Zotero.debug(e.xmlhttp.responseText, 1);
+ if (this._s3Backoff < this._maxS3Backoff) {
+ this._s3Backoff *= 2;
+ }
+ this._s3ConsecutiveFailures++;
+ Zotero.debug("Delaying " + item.libraryKey + " upload for "
+ + this._s3Backoff + " seconds", 2);
+ yield Zotero.Promise.delay(this._s3Backoff * 1000);
+ return this._uploadFile(request, item, params);
+ }
+ }
+ else if (e.status == 500) {
+ // TODO: localize
+ throw new Error("File upload failed. Please try again.");
+ }
+ else {
+ Zotero.logError(`Unexpected file upload status ${e.status} (${item.libraryKey})`);
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e.xmlhttp.responseText);
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+
+ // TODO: Detect cancel?
+ //onUploadCancel(httpRequest, status, data)
+ //deferred.resolve(false);
+ }
+
+ request.setChannel(false);
+ return this._onUploadComplete(req, request, item, params);
+ }),
+
+
+ /**
+ * Post-upload file registration with API
+ */
+ _onUploadComplete: Zotero.Promise.coroutine(function* (req, request, item, params) {
+ var uploadKey = params.uploadKey;
+
+ Zotero.debug("Upload of attachment " + item.key + " finished with status code " + req.status);
+ Zotero.debug(req.responseText);
+
+ // Decrease backoff delay on successful upload
+ if (this._s3Backoff > 1) {
+ this._s3Backoff /= 2;
+ }
+ // And reset consecutive failures
+ this._s3ConsecutiveFailures = 0;
+
+ var requestParams = this._getRequestParams(item.libraryID, `items/${item.key}/file`);
+ var uri = this.apiClient.buildRequestURI(requestParams);
+ var headers = {
+ "Content-Type": "application/x-www-form-urlencoded"
+ };
+ if (params.storedHash) {
+ headers["If-Match"] = params.storedHash;
+ }
+ else {
+ headers["If-None-Match"] = "*";
+ }
+ var body = "upload=" + uploadKey;
+
+ // Register upload on server
+ try {
+ req = yield this.apiClient.makeRequest(
+ "POST",
+ uri,
+ {
+ body,
+ headers,
+ successCodes: [204],
+ requestObserver: function (xmlhttp) {
+ request.setChannel(xmlhttp.channel);
+ }
+ }
+ );
+ }
+ catch (e) {
+ let msg = `Unexpected file registration status ${e.status} (${item.libraryKey})`;
+ Zotero.logError(msg);
+ Zotero.logError(e.xmlhttp.responseText);
+ Zotero.debug(e.xmlhttp.getAllResponseHeaders());
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+
+ var version = req.getResponseHeader('Last-Modified-Version');
+ if (!version) {
+ throw new Error("Last-Modified-Version not provided");
+ }
+ params.version = version;
+
+ yield this._updateItemFileInfo(item, params);
+
+ return new Zotero.Sync.Storage.Result({
+ localChanges: true,
+ remoteChanges: true
+ });
+ }),
+
+
+ /**
+ * Update the local attachment item with the mtime and hash of the uploaded file and the
+ * library version returned by the upload request, and save a modified version of the item
+ * to the sync cache
+ */
+ _updateItemFileInfo: Zotero.Promise.coroutine(function* (item, params) {
+ // Mark as in-sync
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+
+ // Store file mod time and hash
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, params.mtime);
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, params.md5);
+ // Update sync cache with new file metadata and version from server
+ var json = yield Zotero.Sync.Data.Local.getCacheObject(
+ 'item', item.libraryID, item.key, item.version
+ );
+ if (json) {
+ json.version = params.version;
+ json.data.version = params.version;
+ json.data.mtime = params.mtime;
+ json.data.md5 = params.md5;
+ yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
+ }
+ // Update item with new version from server
+ yield Zotero.Items.updateVersion([item.id], params.version);
+
+ // TODO: Can filename, contentType, and charset change the attachment item?
+ });
+
+ try {
+ if (yield Zotero.Attachments.hasMultipleFiles(item)) {
var file = Zotero.getTempDirectory();
file.append(item.key + '.zip');
- file.remove(false);
+ yield OS.File.remove(file.path);
}
}
catch (e) {
Components.utils.reportError(e);
}
- }
+ }),
- function onUploadCancel(httpRequest, status, data) {
+ _onUploadCancel: Zotero.Promise.coroutine(function* (httpRequest, status, data) {
var request = data.request;
var item = data.item;
Zotero.debug("Upload of attachment " + item.key + " cancelled with status code " + status);
try {
- if (Zotero.Attachments.getNumFiles(item) > 1) {
+ if (yield Zotero.Attachments.hasMultipleFiles(item)) {
var file = Zotero.getTempDirectory();
file.append(item.key + '.zip');
file.remove(false);
@@ -631,40 +900,11 @@ Zotero.Sync.Storage.ZFS = (function () {
catch (e) {
Components.utils.reportError(e);
}
- }
+ }),
- /**
- * Get the storage URI for an item
- *
- * @inner
- * @param {Zotero.Item}
- * @return {nsIURI} URI of file on storage server
- */
- function getItemURI(item) {
- var uri = Zotero.Sync.Storage.ZFS.rootURI;
- // Be sure to mirror parameter changes to getItemInfoURI() below
- uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1&version=1';
- return uri;
- }
-
-
- /**
- * Get the storage info URI for an item
- *
- * @inner
- * @param {Zotero.Item}
- * @return {nsIURI} URI of file on storage server with info flag
- */
- function getItemInfoURI(item) {
- var uri = Zotero.Sync.Storage.ZFS.rootURI;
- uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1&version=1&info=1';
- return uri;
- }
-
-
- function getUploadFile(item) {
- if (Zotero.Attachments.getNumFiles(item) > 1) {
+ _getUploadFile: Zotero.Promise.coroutine(function* (item) {
+ if (yield Zotero.Attachments.hasMultipleFiles(item)) {
var file = Zotero.getTempDirectory();
var filename = item.key + '.zip';
file.append(filename);
@@ -673,500 +913,169 @@ Zotero.Sync.Storage.ZFS = (function () {
var file = item.getFile();
}
return file;
- }
+ }),
- //
- // Public methods (called via Zotero.Sync.Storage.ZFS)
- //
- var obj = new Zotero.Sync.Storage.Mode;
- obj.name = "ZFS";
-
- Object.defineProperty(obj, "includeUserFiles", {
- get: function () {
- return Zotero.Prefs.get("sync.storage.enabled") && Zotero.Prefs.get("sync.storage.protocol") == 'zotero';
- }
- });
-
- Object.defineProperty(obj, "includeGroupFiles", {
- get: function () {
- return Zotero.Prefs.get("sync.storage.groups.enabled");
- }
- });
-
- obj._verified = true;
-
- Object.defineProperty(obj, "rootURI", {
- get: function () {
- if (!_rootURI) {
- this._init();
- }
- return _rootURI.clone();
- }
- });
-
- Object.defineProperty(obj, "userURI", {
- get: function () {
- if (!_userURI) {
- this._init();
- }
- return _userURI.clone();
- }
- });
-
-
- obj._init = function () {
- _rootURI = false;
- _userURI = false;
-
- var url = ZOTERO_CONFIG.API_URL;
- var username = Zotero.Sync.Server.username;
- var password = Zotero.Sync.Server.password;
-
- var ios = Components.classes["@mozilla.org/network/io-service;1"].
- getService(Components.interfaces.nsIIOService);
- var uri = ios.newURI(url, null, null);
- uri.username = encodeURIComponent(username);
- uri.password = encodeURIComponent(password);
- _rootURI = uri;
-
- uri = uri.clone();
- uri.spec += 'users/' + Zotero.Users.getCurrentUserID() + '/';
- _userURI = uri;
- };
-
- obj.clearCachedCredentials = function() {
- _rootURI = _userURI = undefined;
- _cachedCredentials = false;
- };
-
/**
- * Begin download process for individual file
+ * Get attachment item metadata on storage server
*
- * @param {Zotero.Sync.Storage.Request} [request]
+ * @param {Zotero.Item} item
+ * @param {Zotero.Sync.Storage.Request} request
+ * @return {Promise<Object>|false} - Promise for object with 'hash', 'filename', 'mtime',
+ * 'compressed', or false if item not found
*/
- obj._downloadFile = function (request) {
- var item = Zotero.Sync.Storage.getItemFromRequestName(request.name);
- if (!item) {
- throw new Error("Item '" + request.name + "' not found");
- }
+ _getStorageFileInfo: Zotero.Promise.coroutine(function* (item, request) {
+ var funcName = "Zotero.Sync.Storage.ZFS._getStorageFileInfo()";
- var self = this;
+ var params = this._getRequestParams(item.libraryID, `items/${item.key}/file`);
+ var uri = this.apiClient.buildRequestURI(params);
- // Retrieve file info from server to store locally afterwards
- return getStorageFileInfo(item, request)
- .then(function (info) {
- if (!request.isRunning()) {
- Zotero.debug("Download request '" + request.name
- + "' is no longer running after getting remote file info");
- return false;
- }
-
- if (!info) {
- Zotero.debug("Remote file not found for item " + item.libraryID + "/" + item.key);
- return false;
- }
-
- var syncModTime = info.mtime;
- var syncHash = info.hash;
-
- var file = item.getFile();
- // Skip download if local file exists and matches mod time
- if (file && file.exists()) {
- if (syncModTime == file.lastModifiedTime) {
- Zotero.debug("File mod time matches remote file -- skipping download");
-
- Zotero.DB.beginTransaction();
- var syncState = Zotero.Sync.Storage.getSyncState(item.id);
- //var updateItem = syncState != 1;
- var updateItem = false;
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, syncModTime, updateItem);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- Zotero.DB.commitTransaction();
- return {
- localChanges: true,
- remoteChanges: false
- };
- }
- // If not compressed, check hash, in case only timestamp changed
- else if (!info.compressed && item.attachmentHash == syncHash) {
- Zotero.debug("File hash matches remote file -- skipping download");
-
- Zotero.DB.beginTransaction();
- var syncState = Zotero.Sync.Storage.getSyncState(item.id);
- //var updateItem = syncState != 1;
- var updateItem = false;
- if (!info.compressed) {
- Zotero.Sync.Storage.setSyncedHash(item.id, syncHash, false);
- }
- Zotero.Sync.Storage.setSyncedModificationTime(item.id, syncModTime, updateItem);
- Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
- Zotero.DB.commitTransaction();
- return {
- localChanges: true,
- remoteChanges: false
- };
+ try {
+ let req = yield this.apiClient.makeRequest(
+ "GET",
+ uri,
+ {
+ successCodes: [200, 404],
+ requestObserver: function (xmlhttp) {
+ request.setChannel(xmlhttp.channel);
}
}
-
- var destFile = Zotero.getTempDirectory();
- if (info.compressed) {
- destFile.append(item.key + '.zip.tmp');
- }
- else {
- destFile.append(item.key + '.tmp');
- }
-
- if (destFile.exists()) {
- try {
- destFile.remove(false);
- }
- catch (e) {
- Zotero.File.checkFileAccessError(e, destFile, 'delete');
- }
- }
-
- // saveURI() below appears not to create empty files for Content-Length: 0,
- // so we create one here just in case
+ );
+ if (req.status == 404) {
+ return new Zotero.Sync.Storage.Result;
+ }
+
+ let info = {};
+ info.hash = req.getResponseHeader('ETag');
+ if (!info.hash) {
+ let msg = `Hash not found in info response in ${funcName} (${item.libraryKey})`;
+ Zotero.debug(msg, 1);
+ Zotero.debug(req.status);
+ Zotero.debug(req.responseText);
+ Components.utils.reportError(msg);
try {
- destFile.create(Components.interfaces.nsIFile.NORMAL_FILE_TYPE, 0644);
+ Zotero.debug(req.getAllResponseHeaders());
}
catch (e) {
- Zotero.File.checkFileAccessError(e, destFile, 'create');
+ Zotero.debug("Response headers unavailable");
}
-
- var deferred = Zotero.Promise.defer();
-
- var listener = new Zotero.Sync.Storage.StreamListener(
- {
- onStart: function (request, data) {
- if (data.request.isFinished()) {
- Zotero.debug("Download request " + data.request.name
- + " stopped before download started -- closing channel");
- request.cancel(0x804b0002); // NS_BINDING_ABORTED
- deferred.resolve(false);
- }
- },
- onProgress: function (a, b, c) {
- request.onProgress(a, b, c)
- },
- onStop: function (request, status, response, data) {
- data.request.setChannel(false);
-
- if (status != 200) {
- if (status == 404) {
- deferred.resolve(false);
- return;
- }
-
- if (status == 0) {
- if (_s3ConsecutiveFailures >= _maxS3ConsecutiveFailures) {
- Zotero.debug(_s3ConsecutiveFailures
- + " consecutive S3 failures -- aborting", 1);
- _s3ConsecutiveFailures = 0;
- }
- else {
- let libraryKey = Zotero.Items.getLibraryKeyHash(item);
- let msg = "S3 returned " + status
- + " (" + libraryKey + ") -- retrying download"
- Components.utils.reportError(msg);
- Zotero.debug(msg, 1);
- if (_s3Backoff < _maxS3Backoff) {
- _s3Backoff *= 2;
- }
- _s3ConsecutiveFailures++;
- Zotero.debug("Delaying " + libraryKey + " download for "
- + _s3Backoff + " seconds", 2);
- Q.delay(_s3Backoff * 1000)
- .then(function () {
- deferred.resolve(self._downloadFile(data.request));
- });
- return;
- }
- }
-
- var msg = "Unexpected status code " + status
- + " for request " + data.request.name
- + " in Zotero.Sync.Storage.ZFS.downloadFile()";
- Zotero.debug(msg, 1);
- Components.utils.reportError(msg);
- // Ignore files not found in S3
- try {
- Zotero.debug(Zotero.File.getContents(destFile, null, 4096), 1);
- }
- catch (e) {
- Zotero.debug(e, 1);
- }
- deferred.reject(Zotero.Sync.Storage.defaultError);
- return;
- }
-
- // Don't try to process if the request has been cancelled
- if (data.request.isFinished()) {
- Zotero.debug("Download request " + data.request.name
- + " is no longer running after file download", 2);
- deferred.resolve(false);
- return;
- }
-
- Zotero.debug("Finished download of " + destFile.path);
-
- try {
- deferred.resolve(Zotero.Sync.Storage.processDownload(data));
- }
- catch (e) {
- deferred.reject(e);
- }
- },
- onCancel: function (request, status, data) {
- Zotero.debug("Request cancelled");
- deferred.resolve(false);
- },
- request: request,
- item: item,
- compressed: info.compressed,
- syncModTime: syncModTime,
- syncHash: syncHash
- }
- );
-
- var uri = getItemURI(item);
-
- // Don't display password in console
- var disp = uri.clone();
- if (disp.password) {
- disp.password = "********";
- }
- Zotero.debug('Saving ' + disp.spec + ' with saveURI()');
- const nsIWBP = Components.interfaces.nsIWebBrowserPersist;
- var wbp = Components
- .classes["@mozilla.org/embedding/browser/nsWebBrowserPersist;1"]
- .createInstance(nsIWBP);
- wbp.persistFlags = nsIWBP.PERSIST_FLAGS_BYPASS_CACHE;
- wbp.progressListener = listener;
- Zotero.Utilities.Internal.saveURI(wbp, uri, destFile);
-
- return deferred.promise;
- });
- };
-
-
- obj._uploadFile = function (request) {
- var item = Zotero.Sync.Storage.getItemFromRequestName(request.name);
- if (Zotero.Attachments.getNumFiles(item) > 1) {
- var deferred = Zotero.Promise.defer();
- var created = Zotero.Sync.Storage.createUploadFile(
- request,
- function (data) {
- if (!data) {
- deferred.resolve(false);
- return;
- }
- deferred.resolve(processUploadFile(data));
- }
- );
- if (!created) {
- return Zotero.Promise.resolve(false);
- }
- return deferred.promise;
- }
- else {
- return processUploadFile({ request: request });
- }
- };
-
-
- /**
- * @return {Promise} A promise for the last sync time
- */
- obj._getLastSyncTime = function (libraryID) {
- var lastSyncURI = this._getLastSyncURI(libraryID);
-
- var self = this;
- return Zotero.Promise.try(function () {
- // Cache the credentials at the root
- return self._cacheCredentials();
- })
- .then(function () {
- return Zotero.HTTP.promise("GET", lastSyncURI,
- { headers: _headers, successCodes: [200, 404], debug: true });
- })
- .then(function (req) {
- // Not yet synced
- if (req.status == 404) {
- Zotero.debug("No last sync time for library " + libraryID);
- return null;
+ let e = Zotero.getString('sync.storage.error.zfs.restart', Zotero.appName);
+ throw new Error(e);
}
+ info.filename = req.getResponseHeader('X-Zotero-Filename');
+ let mtime = req.getResponseHeader('X-Zotero-Modification-Time');
+ info.mtime = parseInt(mtime);
+ info.compressed = req.getResponseHeader('X-Zotero-Compressed') == 'Yes';
+ Zotero.debug(info);
- var ts = req.responseText;
- var date = new Date(ts * 1000);
- Zotero.debug("Last successful ZFS sync for library "
- + libraryID + " was " + date);
- return ts;
- })
- .catch(function (e) {
+ return info;
+ }
+ catch (e) {
if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- if (e.status == 401 || e.status == 403) {
- Zotero.debug("Clearing ZFS authentication credentials", 2);
- _cachedCredentials = false;
- }
-
- return Zotero.Promise.reject(e);
- }
- // TODO: handle browser offline exception
- else {
- throw e;
- }
- });
- };
-
-
- obj._setLastSyncTime = function (libraryID, localLastSyncTime) {
- if (localLastSyncTime) {
- var sql = "REPLACE INTO version VALUES (?, ?)";
- Zotero.DB.query(
- sql, ['storage_zfs_' + libraryID, { int: localLastSyncTime }]
- );
- return;
- }
-
- var lastSyncURI = this._getLastSyncURI(libraryID);
-
- return Zotero.HTTP.promise("POST", lastSyncURI, { headers: _headers, successCodes: [200, 404], debug: true })
- .then(function (req) {
- // Not yet synced
- //
- // TODO: Don't call this at all if no files uploaded
- if (req.status == 404) {
- return;
- }
-
- var ts = req.responseText;
-
- var sql = "REPLACE INTO version VALUES (?, ?)";
- Zotero.DB.query(
- sql, ['storage_zfs_' + libraryID, { int: ts }]
- );
- })
- .catch(function (e) {
- var msg = "Unexpected status code " + e.xmlhttp.status
- + " setting last file sync time";
- Zotero.debug(msg, 1);
- Components.utils.reportError(msg);
- throw new Error(Zotero.Sync.Storage.defaultError);
- });
- };
-
-
- obj._getLastSyncURI = function (libraryID) {
- if (libraryID === Zotero.Libraries.userLibraryID) {
- var lastSyncURI = this.userURI;
- }
- else if (libraryID) {
- var ios = Components.classes["@mozilla.org/network/io-service;1"].
- getService(Components.interfaces.nsIIOService);
- var uri = ios.newURI(Zotero.URI.getLibraryURI(libraryID), null, null);
- var path = uri.path;
- // We don't want the user URI, but it already has the right domain
- // and credentials, so just start with that and replace the path
- var lastSyncURI = this.userURI;
- lastSyncURI.path = path + "/";
- }
- else {
- throw new Error("libraryID not specified");
- }
- lastSyncURI.spec += "laststoragesync";
- return lastSyncURI;
- }
-
-
- obj._cacheCredentials = function () {
- if (_cachedCredentials) {
- Zotero.debug("ZFS credentials are already cached");
- return Zotero.Promise.resolve();
- }
-
- var uri = this.rootURI;
- // TODO: move to root uri
- uri.spec += "?auth=1";
-
- return Zotero.HTTP.promise("GET", uri, { headers: _headers }).
- then(function (req) {
- Zotero.debug("Credentials are cached");
- _cachedCredentials = true;
- })
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- if (e.status == 401) {
- var msg = "File sync login failed\n\n"
- + "Check your username and password in the Sync "
- + "pane of the Zotero preferences.";
- throw (msg);
- }
-
- var msg = "Unexpected status code " + e.status + " "
- + "caching ZFS credentials";
- Zotero.debug(msg, 1);
- throw (msg);
+ if (e.xmlhttp.status == 0) {
+ var msg = "Request cancelled getting storage file info";
}
else {
- throw (e);
+ var msg = "Unexpected status code " + e.xmlhttp.status
+ + " getting storage file info for item " + item.libraryKey;
}
- });
- };
+ Zotero.debug(msg, 1);
+ Zotero.debug(e.xmlhttp.responseText);
+ Components.utils.reportError(msg);
+ throw new Error(Zotero.Sync.Storage.defaultError);
+ }
+
+ throw e;
+ }
+ }),
/**
- * Remove all synced files from the server
+ * Upload the file to the server
+ *
+ * @param {Zotero.Sync.Storage.Request} request
+ * @return {Promise}
*/
- obj._purgeDeletedStorageFiles = function () {
- return Zotero.Promise.try(function () {
- // Cache the credentials at the root
- return this._cacheCredentials();
- }.bind(this))
- then(function () {
- // If we don't have a user id we've never synced and don't need to bother
- if (!Zotero.Users.getCurrentUserID()) {
- return false;
- }
-
- var sql = "SELECT value FROM settings WHERE setting=? AND key=?";
- var values = Zotero.DB.columnQuery(sql, ['storage', 'zfsPurge']);
- if (!values) {
- return false;
- }
-
- // TODO: promisify
-
- Zotero.debug("Unlinking synced files on ZFS");
-
- var uri = this.userURI;
- uri.spec += "removestoragefiles?";
- // Unused
- for each(var value in values) {
- switch (value) {
- case 'user':
- uri.spec += "user=1&";
- break;
-
- case 'group':
- uri.spec += "group=1&";
- break;
-
- default:
- throw "Invalid zfsPurge value '" + value
- + "' in ZFS purgeDeletedStorageFiles()";
+ _processUploadFile: Zotero.Promise.coroutine(function* (request) {
+ /*
+ updateSizeMultiplier(
+ (100 - Zotero.Sync.Storage.compressionTracker.ratio) / 100
+ );
+ */
+
+ var item = Zotero.Sync.Storage.Utilities.getItemFromRequest(request);
+
+
+ /*var info = yield this._getStorageFileInfo(item, request);
+
+ if (request.isFinished()) {
+ Zotero.debug("Upload request '" + request.name
+ + "' is no longer running after getting file info");
+ return false;
+ }
+
+ // Check for conflict
+ if ((yield Zotero.Sync.Storage.Local.getSyncState(item.id))
+ != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) {
+ if (info) {
+ // Local file time
+ var fmtime = yield item.attachmentModificationTime;
+ // Remote mod time
+ var mtime = info.mtime;
+
+ var useLocal = false;
+ var same = !(yield Zotero.Sync.Storage.checkFileModTime(item, fmtime, mtime));
+
+ // Ignore maxed-out 32-bit ints, from brief problem after switch to 32-bit servers
+ if (!same && mtime == 2147483647) {
+ Zotero.debug("Remote mod time is invalid -- uploading local file version");
+ useLocal = true;
+ }
+
+ if (same) {
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime);
+ yield Zotero.Sync.Storage.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+ return {
+ localChanges: true,
+ remoteChanges: false
+ };
+ }
+
+ let smtime = yield Zotero.Sync.Storage.getSyncedModificationTime(item.id);
+ if (!useLocal && smtime != mtime) {
+ Zotero.debug("Conflict -- last synced file mod time "
+ + "does not match time on storage server"
+ + " (" + smtime + " != " + mtime + ")");
+ return {
+ localChanges: false,
+ remoteChanges: false,
+ conflict: {
+ local: { modTime: fmtime },
+ remote: { modTime: mtime }
+ }
+ };
}
}
- uri.spec = uri.spec.substr(0, uri.spec.length - 1);
-
- return Zotero.HTTP.promise("POST", uri, "")
- .then(function (req) {
- var sql = "DELETE FROM settings WHERE setting=? AND key=?";
- Zotero.DB.query(sql, ['storage', 'zfsPurge']);
+ else {
+ Zotero.debug("Remote file not found for item " + item.libraryKey);
+ }
+ }*/
+
+ var result = yield this._getFileUploadParameters(item);
+ if (result.exists) {
+ yield this._updateItemFileInfo(item, result);
+ return new Zotero.Sync.Storage.Result({
+ localChanges: true,
+ remoteChanges: true
});
- }.bind(this));
- };
-
- return obj;
-}());
+ }
+ else if (result instanceof Zotero.Sync.Storage.Result) {
+ return result;
+ }
+ return this._uploadFile(request, item, result);
+ })
+}
diff --git a/chrome/content/zotero/xpcom/sync.js b/chrome/content/zotero/xpcom/sync.js
index a8779cf68..4f79121d1 100644
--- a/chrome/content/zotero/xpcom/sync.js
+++ b/chrome/content/zotero/xpcom/sync.js
@@ -1488,43 +1488,6 @@ Zotero.Sync.Server = new function () {
-Zotero.BufferedInputListener = function (callback) {
- this._callback = callback;
-}
-
-Zotero.BufferedInputListener.prototype = {
- binaryInputStream: null,
- size: 0,
- data: '',
-
- onStartRequest: function(request, context) {},
-
- onStopRequest: function(request, context, status) {
- this.binaryInputStream.close();
- delete this.binaryInputStream;
-
- this._callback(this.data);
- },
-
- onDataAvailable: function(request, context, inputStream, offset, count) {
- this.size += count;
-
- this.binaryInputStream = Components.classes["@mozilla.org/binaryinputstream;1"]
- .createInstance(Components.interfaces.nsIBinaryInputStream)
- this.binaryInputStream.setInputStream(inputStream);
- this.data += this.binaryInputStream.readBytes(this.binaryInputStream.available());
- },
-
- QueryInterface: function (iid) {
- if (iid.equals(Components.interfaces.nsISupports)
- || iid.equals(Components.interfaces.nsIStreamListener)) {
- return this;
- }
- throw Components.results.NS_ERROR_NO_INTERFACE;
- }
-}
-
-
Zotero.Sync.Server.Data = new function() {
var _noMergeTypes = ['search'];
diff --git a/chrome/content/zotero/xpcom/sync/syncAPIClient.js b/chrome/content/zotero/xpcom/sync/syncAPIClient.js
index 3c01b44bb..81d3eaf9a 100644
--- a/chrome/content/zotero/xpcom/sync/syncAPIClient.js
+++ b/chrome/content/zotero/xpcom/sync/syncAPIClient.js
@@ -28,14 +28,15 @@ if (!Zotero.Sync) {
}
Zotero.Sync.APIClient = function (options) {
- this.baseURL = options.baseURL;
- this.apiKey = options.apiKey;
- this.concurrentCaller = options.concurrentCaller;
+ if (!options.baseURL) throw new Error("baseURL not set");
+ if (!options.apiVersion) throw new Error("apiVersion not set");
+ if (!options.apiKey) throw new Error("apiKey not set");
+ if (!options.caller) throw new Error("caller not set");
- if (options.apiVersion == undefined) {
- throw new Error("options.apiVersion not set");
- }
+ this.baseURL = options.baseURL;
this.apiVersion = options.apiVersion;
+ this.apiKey = options.apiKey;
+ this.caller = options.caller;
}
Zotero.Sync.APIClient.prototype = {
@@ -44,7 +45,7 @@ Zotero.Sync.APIClient.prototype = {
getKeyInfo: Zotero.Promise.coroutine(function* () {
var uri = this.baseURL + "keys/" + this.apiKey;
- var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 404] });
+ var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 404] });
if (xmlhttp.status == 404) {
return false;
}
@@ -63,7 +64,7 @@ Zotero.Sync.APIClient.prototype = {
if (!userID) throw new Error("User ID not provided");
var uri = this.baseURL + "users/" + userID + "/groups?format=versions";
- var xmlhttp = yield this._makeRequest("GET", uri);
+ var xmlhttp = yield this.makeRequest("GET", uri);
return this._parseJSON(xmlhttp.responseText);
}),
@@ -76,7 +77,7 @@ Zotero.Sync.APIClient.prototype = {
if (!groupID) throw new Error("Group ID not provided");
var uri = this.baseURL + "groups/" + groupID;
- var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 404] });
+ var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 404] });
if (xmlhttp.status == 404) {
return false;
}
@@ -93,7 +94,7 @@ Zotero.Sync.APIClient.prototype = {
if (since) {
params.since = since;
}
- var uri = this._buildRequestURI(params);
+ var uri = this.buildRequestURI(params);
var options = {
successCodes: [200, 304]
};
@@ -102,7 +103,7 @@ Zotero.Sync.APIClient.prototype = {
"If-Modified-Since-Version": since
};
}
- var xmlhttp = yield this._makeRequest("GET", uri, options);
+ var xmlhttp = yield this.makeRequest("GET", uri, options);
if (xmlhttp.status == 304) {
return false;
}
@@ -128,8 +129,8 @@ Zotero.Sync.APIClient.prototype = {
libraryTypeID: libraryTypeID,
since: since || 0
};
- var uri = this._buildRequestURI(params);
- var xmlhttp = yield this._makeRequest("GET", uri, { successCodes: [200, 409] });
+ var uri = this.buildRequestURI(params);
+ var xmlhttp = yield this.makeRequest("GET", uri, { successCodes: [200, 409] });
if (xmlhttp.status == 409) {
Zotero.debug(`'since' value '${since}' is earlier than the beginning of the delete log`);
return false;
@@ -154,7 +155,7 @@ Zotero.Sync.APIClient.prototype = {
* @param {String} libraryType 'user' or 'group'
* @param {Integer} libraryTypeID userID or groupID
* @param {String} objectType 'item', 'collection', 'search'
- * @param {Object} queryParams Query parameters (see _buildRequestURI())
+ * @param {Object} queryParams Query parameters (see buildRequestURI())
* @return {Promise<Object>|FALSE} Object with 'libraryVersion' and 'results'
*/
getVersions: Zotero.Promise.coroutine(function* (libraryType, libraryTypeID, objectType, queryParams, libraryVersion) {
@@ -176,7 +177,7 @@ Zotero.Sync.APIClient.prototype = {
}
// TODO: Use pagination
- var uri = this._buildRequestURI(params);
+ var uri = this.buildRequestURI(params);
var options = {
successCodes: [200, 304]
@@ -186,7 +187,7 @@ Zotero.Sync.APIClient.prototype = {
"If-Modified-Since-Version": libraryVersion
};
}
- var xmlhttp = yield this._makeRequest("GET", uri, options);
+ var xmlhttp = yield this.makeRequest("GET", uri, options);
if (xmlhttp.status == 304) {
return false;
}
@@ -256,10 +257,10 @@ Zotero.Sync.APIClient.prototype = {
if (objectType == 'item') {
params.includeTrashed = 1;
}
- var uri = this._buildRequestURI(params);
+ var uri = this.buildRequestURI(params);
return [
- this._makeRequest("GET", uri)
+ this.makeRequest("GET", uri)
.then(function (xmlhttp) {
return this._parseJSON(xmlhttp.responseText)
}.bind(this))
@@ -294,9 +295,9 @@ Zotero.Sync.APIClient.prototype = {
libraryType: libraryType,
libraryTypeID: libraryTypeID
};
- var uri = this._buildRequestURI(params);
+ var uri = this.buildRequestURI(params);
- var xmlhttp = yield this._makeRequest(method, uri, {
+ var xmlhttp = yield this.makeRequest(method, uri, {
headers: {
"If-Unmodified-Since-Version": version
},
@@ -319,7 +320,7 @@ Zotero.Sync.APIClient.prototype = {
}),
- _buildRequestURI: function (params) {
+ buildRequestURI: function (params) {
var uri = this.baseURL;
switch (params.libraryType) {
@@ -332,6 +333,10 @@ Zotero.Sync.APIClient.prototype = {
break;
}
+ if (params.target === undefined) {
+ throw new Error("'target' not provided");
+ }
+
uri += "/" + params.target;
if (params.objectKey) {
@@ -382,30 +387,33 @@ Zotero.Sync.APIClient.prototype = {
},
- _makeRequest: function (method, uri, options) {
- if (!options) {
- options = {};
+ getHeaders: function (headers = {}) {
+ headers["Zotero-API-Version"] = this.apiVersion;
+ if (this.apiKey) {
+ headers["Zotero-API-Key"] = this.apiKey;
}
- if (!options.headers) {
- options.headers = {};
- }
- options.headers["Zotero-API-Version"] = this.apiVersion;
+ return headers;
+ },
+
+
+ makeRequest: function (method, uri, options = {}) {
+ options.headers = this.getHeaders(options.headers);
options.dontCache = true;
options.foreground = !options.background;
options.responseType = options.responseType || 'text';
- if (this.apiKey) {
- options.headers.Authorization = "Bearer " + this.apiKey;
- }
- var self = this;
- return this.concurrentCaller.fcall(function () {
- return Zotero.HTTP.request(method, uri, options)
- .catch(function (e) {
- if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
- self._checkResponse(e.xmlhttp);
- }
+ return this.caller.start(Zotero.Promise.coroutine(function* () {
+ try {
+ var xmlhttp = yield Zotero.HTTP.request(method, uri, options);
+ this._checkBackoff(xmlhttp);
+ return xmlhttp;
+ }
+ catch (e) {
+ /*if (e instanceof Zotero.HTTP.UnexpectedStatusException) {
+ this._checkRetry(e.xmlhttp);
+ }*/
throw e;
- });
- });
+ }
+ }.bind(this)));
},
@@ -422,21 +430,6 @@ Zotero.Sync.APIClient.prototype = {
},
- _checkResponse: function (xmlhttp) {
- this._checkBackoff(xmlhttp);
- this._checkAuth(xmlhttp);
- },
-
-
- _checkAuth: function (xmlhttp) {
- if (xmlhttp.status == 403) {
- var e = new Zotero.Error(Zotero.getString('sync.error.invalidLogin'), "INVALID_SYNC_LOGIN");
- e.fatal = true;
- throw e;
- }
- },
-
-
_checkBackoff: function (xmlhttp) {
var backoff = xmlhttp.getResponseHeader("Backoff");
if (backoff) {
@@ -444,7 +437,7 @@ Zotero.Sync.APIClient.prototype = {
if (backoff > 3600) {
// TODO: Update status?
- this.concurrentCaller.pause(backoff * 1000);
+ this.caller.pause(backoff * 1000);
}
}
}
diff --git a/chrome/content/zotero/xpcom/sync/syncEngine.js b/chrome/content/zotero/xpcom/sync/syncEngine.js
index 91eafb050..714bc1bfa 100644
--- a/chrome/content/zotero/xpcom/sync/syncEngine.js
+++ b/chrome/content/zotero/xpcom/sync/syncEngine.js
@@ -76,7 +76,13 @@ Zotero.Sync.Data.Engine = function (options) {
onError: this.onError
}
- this.syncCachePromise = Zotero.Promise.resolve().bind(this);
+ Components.utils.import("resource://zotero/concurrentCaller.js");
+ this.syncCacheProcessor = new ConcurrentCaller({
+ id: "Sync Cache Processor",
+ numConcurrent: 1,
+ logger: Zotero.debug,
+ stopOnError: this.stopOnError
+ });
};
Zotero.Sync.Data.Engine.prototype.UPLOAD_RESULT_SUCCESS = 1;
@@ -167,12 +173,8 @@ Zotero.Sync.Data.Engine.prototype.start = Zotero.Promise.coroutine(function* ()
}
}
- // TEMP: make more reliable
- while (this.syncCachePromise.isPending()) {
- Zotero.debug("Waiting for sync cache to be processed");
- yield this.syncCachePromise;
- yield Zotero.Promise.delay(50);
- }
+ Zotero.debug("Waiting for sync cache to be processed");
+ yield this.syncCacheProcessor.wait();
yield Zotero.Libraries.updateLastSyncTime(this.libraryID);
@@ -286,12 +288,7 @@ Zotero.Sync.Data.Engine.prototype._startDownload = Zotero.Promise.coroutine(func
}
// Wait for sync process to clear
- // TEMP: make more reliable
- while (this.syncCachePromise.isPending()) {
- Zotero.debug("Waiting for sync cache to be processed");
- yield this.syncCachePromise;
- yield Zotero.Promise.delay(50);
- }
+ yield this.syncCacheProcessor.wait();
//
// Get deleted objects
@@ -671,7 +668,8 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi
if (state == 'successful') {
// Update local object with saved data if necessary
- yield obj.fromJSON(current.data);
+ yield obj.loadAllData();
+ obj.fromJSON(current.data);
toSave.push(obj);
toCache.push(current);
}
@@ -701,8 +699,11 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi
// Handle failed objects
for (let index in json.results.failed) {
- let e = json.results.failed[index];
- Zotero.logError(e.message);
+ let { code, message } = json.results.failed[index];
+ e = new Error(message);
+ e.name = "ZoteroUploadObjectError";
+ e.code = code;
+ Zotero.logError(e);
// This shouldn't happen, because the upload request includes a library
// version and should prevent an outdated upload before the object version is
@@ -711,12 +712,11 @@ Zotero.Sync.Data.Engine.prototype._startUpload = Zotero.Promise.coroutine(functi
return this.UPLOAD_RESULT_OBJECT_CONFLICT;
}
- if (this.stopOnError) {
- Zotero.debug("WE FAILED!!!");
- throw new Error(e.message);
- }
if (this.onError) {
- this.onError(e.message);
+ this.onError(e);
+ }
+ if (this.stopOnError) {
+ throw new Error(e);
}
batch[index].tries++;
// Mark 400 errors as permanently failed
@@ -990,7 +990,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
this._failedCheck();
let objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
- let ObjectType = objectType[0].toUpperCase() + objectType.substr(1);
+ let ObjectType = Zotero.Utilities.capitalize(objectType);
// TODO: localize
this.setStatus("Updating " + objectTypePlural + " in " + this.libraryName);
@@ -1037,8 +1037,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
let cacheVersions = yield Zotero.Sync.Data.Local.getLatestCacheObjectVersions(
this.libraryID, objectType
);
- // Queue objects that are out of date or don't exist locally and aren't up-to-date
- // in the cache
+ // Queue objects that are out of date or don't exist locally
for (let key in results.versions) {
let version = results.versions[key];
let obj = yield objectsClass.getByLibraryAndKeyAsync(this.libraryID, key, {
@@ -1060,12 +1059,12 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
}
if (obj) {
- Zotero.debug(Zotero.Utilities.capitalize(objectType) + " " + obj.libraryKey
+ Zotero.debug(ObjectType + " " + obj.libraryKey
+ " is older than version in sync cache");
}
else {
- Zotero.debug(Zotero.Utilities.capitalize(objectType) + " "
- + this.libraryID + "/" + key + " in sync cache not found locally");
+ Zotero.debug(ObjectType + " " + this.libraryID + "/" + key
+ + " in sync cache not found locally");
}
toDownload.push(key);
@@ -1127,7 +1126,7 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
break;
}
- yield this.syncCachePromise;
+ yield this.syncCacheProcessor.wait();
yield Zotero.Libraries.setVersion(this.libraryID, lastLibraryVersion);
@@ -1145,20 +1144,19 @@ Zotero.Sync.Data.Engine.prototype._fullSync = Zotero.Promise.coroutine(function*
* @param {String} objectType
*/
Zotero.Sync.Data.Engine.prototype._processCache = function (objectType) {
- var self = this;
- this.syncCachePromise = this.syncCachePromise.then(function () {
- self._failedCheck();
+ this.syncCacheProcessor.start(function () {
+ this._failedCheck();
return Zotero.Sync.Data.Local.processSyncCacheForObjectType(
- self.libraryID, objectType, self.options
+ this.libraryID, objectType, this.options
)
.catch(function (e) {
Zotero.logError(e);
- if (self.stopOnError) {
+ if (this.stopOnError) {
Zotero.debug("WE FAILED!!!");
- self.failed = e;
+ this.failed = e;
}
- });
- })
+ }.bind(this));
+ }.bind(this))
}
diff --git a/chrome/content/zotero/xpcom/sync/syncEventListeners.js b/chrome/content/zotero/xpcom/sync/syncEventListeners.js
index 9f3fff136..82f8ecbfb 100644
--- a/chrome/content/zotero/xpcom/sync/syncEventListeners.js
+++ b/chrome/content/zotero/xpcom/sync/syncEventListeners.js
@@ -39,10 +39,9 @@ Zotero.Sync.EventListeners.ChangeListener = new function () {
var syncSQL = "REPLACE INTO syncDeleteLog (syncObjectTypeID, libraryID, key, synced) "
+ "VALUES (?, ?, ?, 0)";
+ var storageSQL = "REPLACE INTO storageDeleteLog VALUES (?, ?, 0)";
- if (type == 'item' && Zotero.Sync.Storage.WebDAV.includeUserFiles) {
- var storageSQL = "REPLACE INTO storageDeleteLog VALUES (?, ?, 0)";
- }
+ var storageForLibrary = {};
return Zotero.DB.executeTransaction(function* () {
for (let i = 0; i < ids.length; i++) {
@@ -74,18 +73,25 @@ Zotero.Sync.EventListeners.ChangeListener = new function () {
key
]
);
- if (storageSQL && oldItem.itemType == 'attachment' &&
- [
- Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
- Zotero.Attachments.LINK_MODE_IMPORTED_URL
- ].indexOf(oldItem.linkMode) != -1) {
- yield Zotero.DB.queryAsync(
- storageSQL,
- [
- libraryID,
- key
- ]
- );
+
+ if (type == 'item') {
+ if (storageForLibrary[libraryID] === undefined) {
+ storageForLibrary[libraryID] =
+ Zotero.Sync.Storage.Local.getModeForLibrary(libraryID) == 'webdav';
+ }
+ if (storageForLibrary[libraryID] && oldItem.itemType == 'attachment' &&
+ [
+ Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
+ Zotero.Attachments.LINK_MODE_IMPORTED_URL
+ ].indexOf(oldItem.linkMode) != -1) {
+ yield Zotero.DB.queryAsync(
+ storageSQL,
+ [
+ libraryID,
+ key
+ ]
+ );
+ }
}
}
});
@@ -215,3 +221,23 @@ Zotero.Sync.EventListeners.progressListener = {
}
};
+
+
+Zotero.Sync.EventListeners.StorageFileOpenListener = {
+ init: function () {
+ Zotero.Notifier.registerObserver(this, ['file'], 'storageFileOpen');
+ },
+
+ notify: function (event, type, ids, extraData) {
+ if (event == 'open' && type == 'file') {
+ let timestamp = new Date().getTime();
+
+ for (let i = 0; i < ids.length; i++) {
+ Zotero.Sync.Storage.Local.uploadCheckFiles.push({
+ itemID: ids[i],
+ timestamp: timestamp
+ });
+ }
+ }
+ }
+}
diff --git a/chrome/content/zotero/xpcom/sync/syncLocal.js b/chrome/content/zotero/xpcom/sync/syncLocal.js
index a5ba10f7a..104958e03 100644
--- a/chrome/content/zotero/xpcom/sync/syncLocal.js
+++ b/chrome/content/zotero/xpcom/sync/syncLocal.js
@@ -28,6 +28,8 @@ if (!Zotero.Sync.Data) {
}
Zotero.Sync.Data.Local = {
+ _loginManagerHost: 'https://api.zotero.org',
+ _loginManagerRealm: 'Zotero Web API',
_lastSyncTime: null,
_lastClassicSyncTime: null,
@@ -39,6 +41,71 @@ Zotero.Sync.Data.Local = {
}),
+ getAPIKey: function () {
+ var apiKey = Zotero.Prefs.get('devAPIKey');
+ if (apiKey) {
+ return apiKey;
+ }
+ var loginManager = Components.classes["@mozilla.org/login-manager;1"]
+ .getService(Components.interfaces.nsILoginManager);
+ var logins = loginManager.findLogins(
+ {}, this._loginManagerHost, null, this._loginManagerRealm
+ );
+ // Get API from returned array of nsILoginInfo objects
+ if (logins.length) {
+ return logins[0].password;
+ }
+ if (!apiKey) {
+ let username = Zotero.Prefs.get('sync.server.username');
+ if (username) {
+ let password = Zotero.Sync.Data.Local.getLegacyPassword(username);
+ if (!password) {
+ return false;
+ }
+ throw new Error("Unimplemented");
+ // Get API key from server
+
+ // Store API key
+
+ // Remove old logins and username pref
+ }
+ }
+ return apiKey;
+ },
+
+
+ setAPIKey: function (apiKey) {
+ var loginManager = Components.classes["@mozilla.org/login-manager;1"]
+ .getService(Components.interfaces.nsILoginManager);
+ var nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1",
+ Components.interfaces.nsILoginInfo, "init");
+ var loginInfo = new nsLoginInfo(
+ this._loginManagerHost,
+ null,
+ this._loginManagerRealm,
+ 'API Key',
+ apiKey,
+ "",
+ ""
+ );
+ loginManager.addLogin(loginInfo);
+ },
+
+
+ getLegacyPassword: function (username) {
+ var loginManager = Components.classes["@mozilla.org/login-manager;1"]
+ .getService(Components.interfaces.nsILoginManager);
+ var logins = loginManager.findLogins({}, "chrome://zotero", "Zotero Storage Server", null);
+ // Find user from returned array of nsILoginInfo objects
+ for (let login of logins) {
+ if (login.username == username) {
+ return login.password;
+ }
+ }
+ return false;
+ },
+
+
getLastSyncTime: function () {
if (_lastSyncTime === null) {
throw new Error("Last sync time not yet loaded");
@@ -86,7 +153,7 @@ Zotero.Sync.Data.Local = {
var sql = "SELECT " + objectsClass.idColumn + " FROM " + objectsClass.table
+ " WHERE libraryID=? AND synced=0";
- // RETRIEVE PARENT DOWN? EVEN POSSIBLE?
+ // TODO: RETRIEVE PARENT DOWN? EVEN POSSIBLE?
// items via parent
// collections via getDescendents?
@@ -154,6 +221,35 @@ Zotero.Sync.Data.Local = {
}),
+ getCacheObjects: Zotero.Promise.coroutine(function* (objectType, libraryID, keyVersionPairs) {
+ if (!keyVersionPairs.length) return [];
+ var sql = "SELECT data FROM syncCache SC JOIN (SELECT "
+ + keyVersionPairs.map(function (pair) {
+ Zotero.DataObjectUtilities.checkKey(pair[0]);
+ return "'" + pair[0] + "' AS key, " + parseInt(pair[1]) + " AS version";
+ }).join(" UNION SELECT ")
+ + ") AS pairs ON (pairs.key=SC.key AND pairs.version=SC.version) "
+ + "WHERE libraryID=? AND "
+ + "syncObjectTypeID IN (SELECT syncObjectTypeID FROM syncObjectTypes WHERE name=?)";
+ var rows = yield Zotero.DB.columnQueryAsync(sql, [libraryID, objectType]);
+ return rows.map(row => JSON.parse(row));
+ }),
+
+
+ saveCacheObject: Zotero.Promise.coroutine(function* (objectType, libraryID, json) {
+ json = this._checkCacheJSON(json);
+
+ Zotero.debug("Saving to sync cache:");
+ Zotero.debug(json);
+
+ var syncObjectTypeID = Zotero.Sync.Data.Utilities.getSyncObjectTypeID(objectType);
+ var sql = "INSERT OR REPLACE INTO syncCache "
+ + "(libraryID, key, syncObjectTypeID, version, data) VALUES (?, ?, ?, ?, ?)";
+ var params = [libraryID, json.key, syncObjectTypeID, json.version, JSON.stringify(json)];
+ return Zotero.DB.queryAsync(sql, params);
+ }),
+
+
saveCacheObjects: Zotero.Promise.coroutine(function* (objectType, libraryID, jsonArray) {
if (!Array.isArray(jsonArray)) {
throw new Error("'json' must be an array");
@@ -165,20 +261,7 @@ Zotero.Sync.Data.Local = {
return;
}
- jsonArray = jsonArray.map(o => {
- if (o.key === undefined) {
- throw new Error("Missing 'key' property in JSON");
- }
- if (o.version === undefined) {
- throw new Error("Missing 'version' property in JSON");
- }
- // If direct data object passed, wrap in fake response object
- return o.data === undefined ? {
- key: o.key,
- version: o.version,
- data: o
- } : o;
- });
+ jsonArray = jsonArray.map(json => this._checkCacheJSON(json));
Zotero.debug("Saving to sync cache:");
Zotero.debug(jsonArray);
@@ -206,6 +289,22 @@ Zotero.Sync.Data.Local = {
}),
+ _checkCacheJSON: function (json) {
+ if (json.key === undefined) {
+ throw new Error("Missing 'key' property in JSON");
+ }
+ if (json.version === undefined) {
+ throw new Error("Missing 'version' property in JSON");
+ }
+ // If direct data object passed, wrap in fake response object
+ return json.data === undefined ? {
+ key: json.key,
+ version: json.version,
+ data: json
+ } : json;
+ },
+
+
processSyncCache: Zotero.Promise.coroutine(function* (libraryID, options) {
for (let objectType of Zotero.DataObjectUtilities.getTypesForLibrary(libraryID)) {
yield this.processSyncCacheForObjectType(libraryID, objectType, options);
@@ -213,8 +312,7 @@ Zotero.Sync.Data.Local = {
}),
- processSyncCacheForObjectType: Zotero.Promise.coroutine(function* (libraryID, objectType, options) {
- options = options || {};
+ processSyncCacheForObjectType: Zotero.Promise.coroutine(function* (libraryID, objectType, options = {}) {
var objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(objectType);
var objectTypePlural = Zotero.DataObjectUtilities.getObjectTypePlural(objectType);
var ObjectType = Zotero.Utilities.capitalize(objectType);
@@ -227,7 +325,6 @@ Zotero.Sync.Data.Local = {
var numSkipped = 0;
var data = yield this._getUnwrittenData(libraryID, objectType);
-
if (!data.length) {
Zotero.debug("No unwritten " + objectTypePlural + " in sync cache");
return;
@@ -260,9 +357,9 @@ Zotero.Sync.Data.Local = {
for (let i = 0; i < chunk.length; i++) {
let json = chunk[i];
let jsonData = json.data;
- let isNewObject;
let objectKey = json.key;
+ Zotero.debug(`Processing ${objectType} ${libraryID}/${objectKey}`);
Zotero.debug(json);
if (!jsonData) {
@@ -302,26 +399,22 @@ Zotero.Sync.Data.Local = {
}*/
}
+ let isNewObject = false;
+ let skipCache = false;
let obj = yield objectsClass.getByLibraryAndKeyAsync(
libraryID, objectKey, { noCache: true }
);
if (obj) {
Zotero.debug("Matching local " + objectType + " exists", 4);
- isNewObject = false;
- // Local object has not been modified since last sync
- if (obj.synced) {
- // Overwrite local below
- }
- else {
+ // Local object has been modified since last sync
+ if (!obj.synced) {
Zotero.debug("Local " + objectType + " " + obj.libraryKey
+ " has been modified since last sync", 4);
let cachedJSON = yield this.getCacheObject(
objectType, obj.libraryID, obj.key, obj.version
);
- Zotero.debug("GOT CACHED");
- Zotero.debug(cachedJSON);
let jsonDataLocal = yield obj.toJSON();
@@ -333,42 +426,51 @@ Zotero.Sync.Data.Local = {
['dateAdded', 'dateModified']
);
- // If no changes, update local version and keep as unsynced
+ // If no changes, update local version number and mark as synced
if (!result.changes.length && !result.conflicts.length) {
- Zotero.debug("No remote changes to apply to local " + objectType
- + " " + obj.libraryKey);
- yield obj.updateVersion(json.version);
+ Zotero.debug("No remote changes to apply to local "
+ + objectType + " " + obj.libraryKey);
+ obj.version = json.version;
+ obj.synced = true;
+ yield obj.save();
+ continue;
+ }
+
+ if (result.conflicts.length) {
+ if (objectType != 'item') {
+ throw new Error(`Unexpected conflict on ${objectType} object`);
+ }
+ Zotero.debug("Conflict!");
+ conflicts.push({
+ left: jsonDataLocal,
+ right: jsonData,
+ changes: result.changes,
+ conflicts: result.conflicts
+ });
continue;
}
// If no conflicts, apply remote changes automatically
- if (!result.conflicts.length) {
- Zotero.DataObjectUtilities.applyChanges(
- jsonData, result.changes
- );
- let saved = yield this._saveObjectFromJSON(obj, jsonData, options);
- if (saved) numSaved++;
- continue;
- }
-
- if (objectType != 'item') {
- throw new Error(`Unexpected conflict on ${objectType} object`);
- }
-
- conflicts.push({
- left: jsonDataLocal,
- right: jsonData,
- changes: result.changes,
- conflicts: result.conflicts
- });
- continue;
+ Zotero.debug(`Applying remote changes to ${objectType} `
+ + obj.libraryKey);
+ Zotero.debug(result.changes);
+ Zotero.DataObjectUtilities.applyChanges(
+ jsonDataLocal, result.changes
+ );
+ // Transfer properties that aren't in the changeset
+ ['version', 'dateAdded', 'dateModified'].forEach(x => {
+ if (jsonDataLocal[x] !== jsonData[x]) {
+ Zotero.debug(`Applying remote '${x}' value`);
+ }
+ jsonDataLocal[x] = jsonData[x];
+ })
+ jsonData = jsonDataLocal;
}
-
- let saved = yield this._saveObjectFromJSON(obj, jsonData, options);
- if (saved) numSaved++;
}
// Object doesn't exist locally
else {
+ Zotero.debug(ObjectType + " doesn't exist locally");
+
isNewObject = true;
// Check if object has been deleted locally
@@ -376,6 +478,8 @@ Zotero.Sync.Data.Local = {
objectType, libraryID, objectKey
);
if (dateDeleted) {
+ Zotero.debug(ObjectType + " was deleted locally");
+
switch (objectType) {
case 'item':
conflicts.push({
@@ -410,24 +514,30 @@ Zotero.Sync.Data.Local = {
obj.key = objectKey;
yield obj.loadPrimaryData();
- let saved = yield this._saveObjectFromJSON(obj, jsonData, options, {
- // Don't cache new items immediately, which skips reloading after save
- skipCache: true
- });
- if (saved) numSaved++;
+ // Don't cache new items immediately, which skips reloading after save
+ skipCache = true;
+ }
+
+ let saved = yield this._saveObjectFromJSON(
+ obj, jsonData, options, { skipCache }
+ );
+ // Mark updated attachments for download
+ if (saved && objectType == 'item' && obj.isImportedAttachment()) {
+ yield this._checkAttachmentForDownload(
+ obj, jsonData.mtime, isNewObject
+ );
+ }
+
+ if (saved) {
+ numSaved++;
}
}
}.bind(this));
}.bind(this)
);
- // Keep retrying if we skipped any, as long as we're still making progress
- if (numSkipped && numSaved != 0) {
- Zotero.debug("More " + objectTypePlural + " in cache -- continuing");
- yield this.processSyncCacheForObjectType(libraryID, objectType, options);
- }
-
if (conflicts.length) {
+ // Sort conflicts by local Date Modified/Deleted
conflicts.sort(function (a, b) {
var d1 = a.left.dateDeleted || a.left.dateModified;
var d2 = b.left.dateDeleted || b.left.dateModified;
@@ -442,6 +552,7 @@ Zotero.Sync.Data.Local = {
var mergeData = this.resolveConflicts(conflicts);
if (mergeData) {
+ Zotero.debug("Processing resolved conflicts");
let mergeOptions = {};
Object.assign(mergeOptions, options);
// Tell _saveObjectFromJSON not to save with 'synced' set to true
@@ -484,11 +595,55 @@ Zotero.Sync.Data.Local = {
}
}
+ // Keep retrying if we skipped any, as long as we're still making progress
+ if (numSkipped && numSaved != 0) {
+ Zotero.debug("More " + objectTypePlural + " in cache -- continuing");
+ return this.processSyncCacheForObjectType(libraryID, objectType, options);
+ }
+
data = yield this._getUnwrittenData(libraryID, objectType);
- Zotero.debug("Skipping " + data.length + " "
- + (data.length == 1 ? objectType : objectTypePlural)
- + " in sync cache");
- return data;
+ if (data.length) {
+ Zotero.debug(`Skipping ${data.length} `
+ + (data.length == 1 ? objectType : objectTypePlural)
+ + " in sync cache");
+ }
+ }),
+
+
+ _checkAttachmentForDownload: Zotero.Promise.coroutine(function* (item, mtime, isNewObject) {
+ var markToDownload = false;
+ if (!isNewObject) {
+ // Convert previously used Unix timestamps to ms-based timestamps
+ if (mtime < 10000000000) {
+ Zotero.debug("Converting Unix timestamp '" + mtime + "' to ms");
+ mtime = mtime * 1000;
+ }
+ var fmtime = null;
+ try {
+ fmtime = yield item.attachmentModificationTime;
+ }
+ catch (e) {
+ // This will probably fail later too, but ignore it for now
+ Zotero.logError(e);
+ }
+ if (fmtime) {
+ let state = Zotero.Sync.Storage.Local.checkFileModTime(item, fmtime, mtime);
+ if (state !== false) {
+ markToDownload = true;
+ }
+ }
+ else {
+ markToDownload = true;
+ }
+ }
+ else {
+ markToDownload = true;
+ }
+ if (markToDownload) {
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+ }
}),
@@ -501,6 +656,8 @@ Zotero.Sync.Data.Local = {
resolveConflicts: function (conflicts) {
+ Zotero.debug("Showing conflict resolution window");
+
var io = {
dataIn: {
captions: [
@@ -511,9 +668,7 @@ Zotero.Sync.Data.Local = {
conflicts
}
};
-
var url = 'chrome://zotero/content/merge.xul';
-
var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]
.getService(Components.interfaces.nsIWindowMediator);
var lastWin = wm.getMostRecentWindow("navigator:browser");
@@ -553,7 +708,8 @@ Zotero.Sync.Data.Local = {
_saveObjectFromJSON: Zotero.Promise.coroutine(function* (obj, json, options) {
try {
- yield obj.fromJSON(json);
+ yield obj.loadAllData();
+ obj.fromJSON(json);
if (!options.saveAsChanged) {
obj.version = json.version;
obj.synced = true;
@@ -611,6 +767,11 @@ Zotero.Sync.Data.Local = {
var changeset1 = Zotero.DataObjectUtilities.diff(originalJSON, currentJSON, ignoreFields);
var changeset2 = Zotero.DataObjectUtilities.diff(originalJSON, newJSON, ignoreFields);
+ Zotero.debug("CHANGESET1");
+ Zotero.debug(changeset1);
+ Zotero.debug("CHANGESET2");
+ Zotero.debug(changeset2);
+
var conflicts = [];
for (let i = 0; i < changeset1.length; i++) {
@@ -725,27 +886,43 @@ Zotero.Sync.Data.Local = {
var conflicts = [];
for (let i = 0; i < changeset.length; i++) {
- let c = changeset[i];
+ let c2 = changeset[i];
// Member changes are additive only, so ignore removals
- if (c.op.endsWith('-remove')) {
+ if (c2.op.endsWith('-remove')) {
continue;
}
// Record member changes
- if (c.op.startsWith('member-') || c.op.startsWith('property-member-')) {
- changes.push(c);
+ if (c2.op.startsWith('member-') || c2.op.startsWith('property-member-')) {
+ changes.push(c2);
continue;
}
// Automatically apply remote changes for non-items, even if in conflict
if (objectType != 'item') {
- changes.push(c);
+ changes.push(c2);
continue;
}
// Field changes are conflicts
- conflicts.push(c);
+ //
+ // Since we don't know what changed, use only 'add' and 'delete'
+ if (c2.op == 'modify') {
+ c2.op = 'add';
+ }
+ let val = currentJSON[c2.field];
+ let c1 = {
+ field: c2.field,
+ op: val !== undefined ? 'add' : 'delete'
+ };
+ if (val !== undefined) {
+ c1.value = val;
+ }
+ if (c2.op == 'modify') {
+ c2.op = 'add';
+ }
+ conflicts.push([c1, c2]);
}
return { changes, conflicts };
diff --git a/chrome/content/zotero/xpcom/sync/syncRunner.js b/chrome/content/zotero/xpcom/sync/syncRunner.js
index 430639a22..e440b07f1 100644
--- a/chrome/content/zotero/xpcom/sync/syncRunner.js
+++ b/chrome/content/zotero/xpcom/sync/syncRunner.js
@@ -29,34 +29,62 @@ if (!Zotero.Sync) {
Zotero.Sync = {};
}
-Zotero.Sync.Runner_Module = function () {
+// Initialized as Zotero.Sync.Runner in zotero.js
+Zotero.Sync.Runner_Module = function (options = {}) {
+ const stopOnError = true;
+
Zotero.defineProperty(this, 'background', { get: () => _background });
Zotero.defineProperty(this, 'lastSyncStatus', { get: () => _lastSyncStatus });
- const stopOnError = true;
+ this.baseURL = options.baseURL || ZOTERO_CONFIG.API_URL;
+ this.apiVersion = options.apiVersion || ZOTERO_CONFIG.API_VERSION;
+ this.apiKey = options.apiKey || Zotero.Sync.Data.Local.getAPIKey();
+
+ Components.utils.import("resource://zotero/concurrentCaller.js");
+ this.caller = new ConcurrentCaller(4);
+ this.caller.setLogger(msg => Zotero.debug(msg));
+ this.caller.stopOnError = stopOnError;
+ this.caller.onError = function (e) {
+ this.addError(e);
+ if (e.fatal) {
+ this.caller.stop();
+ throw e;
+ }
+ }.bind(this);
var _autoSyncTimer;
var _background;
var _firstInSession = true;
var _syncInProgress = false;
+ var _syncEngines = [];
+ var _storageEngines = [];
+
var _lastSyncStatus;
var _currentSyncStatusLabel;
var _currentLastSyncLabel;
var _errors = [];
+ this.getAPIClient = function () {
+ return new Zotero.Sync.APIClient({
+ baseURL: this.baseURL,
+ apiVersion: this.apiVersion,
+ apiKey: this.apiKey,
+ caller: this.caller
+ });
+ }
+
+
/**
* Begin a sync session
*
- * @param {Object} [options]
- * @param {String} [apiKey]
- * @param {Boolean} [background=false] - Whether this is a background request, which prevents
- * some alerts from being shown
- * @param {String} [baseURL]
- * @param {Integer[]} [libraries] - IDs of libraries to sync
- * @param {Function} [onError] - Function to pass errors to instead of handling internally
- * (used for testing)
+ * @param {Object} [options]
+ * @param {Boolean} [options.background=false] Whether this is a background request, which
+ * prevents some alerts from being shown
+ * @param {Integer[]} [options.libraries] IDs of libraries to sync
+ * @param {Function} [options.onError] Function to pass errors to instead of
+ * handling internally (used for testing)
*/
this.sync = Zotero.Promise.coroutine(function* (options = {}) {
// Clear message list
@@ -84,14 +112,13 @@ Zotero.Sync.Runner_Module = function () {
// Purge deleted objects so they don't cause sync errors (e.g., long tags)
yield Zotero.purgeDataObjects(true);
- options.apiKey = options.apiKey || Zotero.Prefs.get('devAPIKey');
- if (!options.apiKey) {
- let msg = "API key not provided";
+ if (!this.apiKey) {
+ let msg = "API key not set";
let e = new Zotero.Error(msg, 0, { dialogButtonText: null })
this.updateIcons(e);
+ _syncInProgress = false;
return false;
}
- options.baseURL = options.baseURL || ZOTERO_CONFIG.API_URL;
if (_firstInSession) {
options.firstInSession = true;
_firstInSession = false;
@@ -102,66 +129,45 @@ Zotero.Sync.Runner_Module = function () {
this.updateIcons('animate');
try {
- Components.utils.import("resource://zotero/concurrent-caller.js");
- var caller = new ConcurrentCaller(4); // TEMP: one for now
- caller.setLogger(msg => Zotero.debug(msg));
- caller.stopOnError = stopOnError;
- caller.onError = function (e) {
- this.addError(e);
- if (e.fatal) {
- caller.stop();
- throw e;
- }
- }.bind(this);
+ let client = this.getAPIClient();
- // TODO: Use a single client for all operations?
- var client = new Zotero.Sync.APIClient({
- baseURL: options.baseURL,
- apiVersion: ZOTERO_CONFIG.API_VERSION,
- apiKey: options.apiKey,
- concurrentCaller: caller,
- background: options.background
- });
-
- var keyInfo = yield this.checkAccess(client, options);
+ let keyInfo = yield this.checkAccess(client, options);
if (!keyInfo) {
- this.stop();
+ this.end();
Zotero.debug("Syncing cancelled");
return false;
}
- var libraries = yield this.checkLibraries(client, options, keyInfo, libraries);
-
- for (let libraryID of libraries) {
- try {
- let engine = new Zotero.Sync.Data.Engine({
- libraryID: libraryID,
- apiClient: client,
- setStatus: this.setSyncStatus.bind(this),
- stopOnError: stopOnError,
- onError: this.addError.bind(this)
- });
- yield engine.start();
- }
- catch (e) {
- Zotero.debug("Sync failed for library " + libraryID);
- Zotero.debug(e, 1);
- Components.utils.reportError(e);
- this.checkError(e);
+ let engineOptions = {
+ apiClient: client,
+ caller: this.caller,
+ setStatus: this.setSyncStatus.bind(this),
+ stopOnError,
+ onError: function (e) {
if (options.onError) {
options.onError(e);
}
else {
- this.addError(e);
+ this.addError.bind(this);
}
- if (stopOnError || e.fatal) {
- caller.stop();
- break;
- }
- }
- }
+ }.bind(this),
+ background: _background,
+ firstInSession: _firstInSession
+ };
- yield Zotero.Sync.Data.Local.updateLastSyncTime();
+ let nextLibraries = yield this.checkLibraries(
+ client, options, keyInfo, options.libraries
+ );
+ // Sync data, files, and then any data that needs to be uploaded
+ let attempt = 1;
+ while (nextLibraries.length) {
+ if (attempt > 3) {
+ throw new Error("Too many sync attempts -- stopping");
+ }
+ nextLibraries = yield _doDataSync(nextLibraries, engineOptions);
+ nextLibraries = yield _doFileSync(nextLibraries, engineOptions);
+ attempt++;
+ }
}
catch (e) {
if (options.onError) {
@@ -171,62 +177,19 @@ Zotero.Sync.Runner_Module = function () {
this.addError(e);
}
}
-
- this.stop();
+ finally {
+ this.end();
+ }
Zotero.debug("Done syncing");
+ /*if (results.changesMade) {
+ Zotero.debug("Changes made during file sync "
+ + "-- performing additional data sync");
+ this.sync(options);
+ }*/
+
return;
-
- var storageSync = function () {
- Zotero.Sync.Runner.setSyncStatus(Zotero.getString('sync.status.syncingFiles'));
-
- Zotero.Sync.Storage.sync(options)
- .then(function (results) {
- Zotero.debug("File sync is finished");
-
- if (results.errors.length) {
- Zotero.debug(results.errors, 1);
- for each(var e in results.errors) {
- Components.utils.reportError(e);
- }
- Zotero.Sync.Runner.setErrors(results.errors);
- return;
- }
-
- if (results.changesMade) {
- Zotero.debug("Changes made during file sync "
- + "-- performing additional data sync");
- Zotero.Sync.Server.sync(finalCallbacks);
- }
- else {
- Zotero.Sync.Runner.stop();
- }
- })
- .catch(function (e) {
- Zotero.debug("File sync failed", 1);
- Zotero.Sync.Runner.error(e);
- })
- .done();
- };
-
- Zotero.Sync.Server.sync({
- // Sync 1 success
- onSuccess: storageSync,
-
- // Sync 1 skip
- onSkip: storageSync,
-
- // Sync 1 stop
- onStop: function () {
- Zotero.Sync.Runner.stop();
- },
-
- // Sync 1 error
- onError: function (e) {
- Zotero.Sync.Runner.error(e);
- }
- });
});
@@ -242,8 +205,9 @@ Zotero.Sync.Runner_Module = function () {
}
// Sanity check
- if (!json.userID) throw new Error("userID not found in response");
- if (!json.username) throw new Error("username not found in response");
+ if (!json.userID) throw new Error("userID not found in key response");
+ if (!json.username) throw new Error("username not found in key response");
+ if (!json.access) throw new Error("'access' not found in key response");
// Make sure user hasn't changed, and prompt to update database if so
if (!(yield this.checkUser(json.userID, json.username))) {
@@ -446,8 +410,6 @@ Zotero.Sync.Runner_Module = function () {
*
* @param {Integer} userID New userID
* @param {Integer} libraryID New libraryID
- * @param {Integer} noServerData The server account is empty — this is
- * the account after a server clear
* @return {Boolean} - True to continue, false to cancel
*/
this.checkUser = Zotero.Promise.coroutine(function* (userID, username) {
@@ -544,7 +506,154 @@ Zotero.Sync.Runner_Module = function () {
});
+ var _doDataSync = Zotero.Promise.coroutine(function* (libraries, options, skipUpdateLastSyncTime) {
+ var successfulLibraries = [];
+ for (let libraryID of libraries) {
+ try {
+ let opts = {};
+ Object.assign(opts, options);
+ opts.libraryID = libraryID;
+
+ let engine = new Zotero.Sync.Data.Engine(opts);
+ yield engine.start();
+ successfulLibraries.push(libraryID);
+ }
+ catch (e) {
+ Zotero.debug("Sync failed for library " + libraryID);
+ Zotero.logError(e);
+ this.checkError(e);
+ if (options.onError) {
+ options.onError(e);
+ }
+ else {
+ this.addError(e);
+ }
+ if (stopOnError || e.fatal) {
+ Zotero.debug("Stopping on error", 1);
+ options.caller.stop();
+ break;
+ }
+ }
+ }
+ // Update last-sync time if any libraries synced
+ // TEMP: Do we want to show updated time if some libraries haven't synced?
+ if (!libraries.length || successfulLibraries.length) {
+ yield Zotero.Sync.Data.Local.updateLastSyncTime();
+ }
+ return successfulLibraries;
+ }.bind(this));
+
+
+ var _doFileSync = Zotero.Promise.coroutine(function* (libraries, options) {
+ Zotero.debug("Starting file syncing");
+ this.setSyncStatus(Zotero.getString('sync.status.syncingFiles'));
+ let librariesToSync = [];
+ for (let libraryID of libraries) {
+ try {
+ let opts = {};
+ Object.assign(opts, options);
+ opts.libraryID = libraryID;
+
+ let tries = 3;
+ while (true) {
+ if (tries == 0) {
+ throw new Error("Too many file sync attempts for library " + libraryID);
+ }
+ tries--;
+ let engine = new Zotero.Sync.Storage.Engine(opts);
+ let results = yield engine.start();
+ if (results.syncRequired) {
+ librariesToSync.push(libraryID);
+ }
+ else if (results.fileSyncRequired) {
+ Zotero.debug("Another file sync required -- restarting");
+ continue;
+ }
+ break;
+ }
+ }
+ catch (e) {
+ Zotero.debug("File sync failed for library " + libraryID);
+ Zotero.debug(e, 1);
+ Components.utils.reportError(e);
+ this.checkError(e);
+ if (options.onError) {
+ options.onError(e);
+ }
+ else {
+ this.addError(e);
+ }
+ if (stopOnError || e.fatal) {
+ options.caller.stop();
+ break;
+ }
+ }
+ }
+ Zotero.debug("Done with file syncing");
+ return librariesToSync;
+ }.bind(this));
+
+
+ /**
+ * Download a single file on demand (not within a sync process)
+ */
+ this.downloadFile = Zotero.Promise.coroutine(function* (item, requestCallbacks) {
+ if (Zotero.HTTP.browserIsOffline()) {
+ Zotero.debug("Browser is offline", 2);
+ return false;
+ }
+
+ // TEMP
+ var options = {};
+
+ var itemID = item.id;
+ var modeClass = Zotero.Sync.Storage.Local.getClassForLibrary(item.libraryID);
+ var controller = new modeClass({
+ apiClient: this.getAPIClient()
+ });
+
+ // TODO: verify WebDAV on-demand?
+ if (!controller.verified) {
+ Zotero.debug("File syncing is not active for item's library -- skipping download");
+ return false;
+ }
+
+ if (!item.isImportedAttachment()) {
+ throw new Error("Not an imported attachment");
+ }
+
+ if (yield item.getFilePathAsync()) {
+ Zotero.debug("File already exists -- replacing");
+ }
+
+ // TODO: start sync icon?
+ // TODO: create queue for cancelling
+
+ if (!requestCallbacks) {
+ requestCallbacks = {};
+ }
+ var onStart = function (request) {
+ return controller.downloadFile(request);
+ };
+ var request = new Zotero.Sync.Storage.Request({
+ type: 'download',
+ libraryID: item.libraryID,
+ name: item.libraryKey,
+ onStart: requestCallbacks.onStart
+ ? [onStart, requestCallbacks.onStart]
+ : [onStart]
+ });
+ return request.start();
+ });
+
+
this.stop = function () {
+ _syncEngines.forEach(engine => engine.stop());
+ _storageEngines.forEach(engine => engine.stop());
+ }
+
+
+ this.end = function () {
this.updateIcons(_errors);
_errors = [];
_syncInProgress = false;
@@ -669,7 +778,6 @@ Zotero.Sync.Runner_Module = function () {
if (libraryID) {
e.libraryID = libraryID;
}
- Zotero.logError(e);
_errors.push(this.parseError(e));
}
@@ -1027,7 +1135,8 @@ Zotero.Sync.Runner_Module = function () {
var lastSyncTime = Zotero.Sync.Data.Local.getLastSyncTime();
if (!lastSyncTime) {
try {
- lastSyncTime = Zotero.Sync.Data.Local.getLastClassicSyncTime()
+ lastSyncTime = Zotero.Sync.Data.Local.getLastClassicSyncTime();
+ Zotero.debug(lastSyncTime);
}
catch (e) {
Zotero.debug(e, 2);
@@ -1052,5 +1161,3 @@ Zotero.Sync.Runner_Module = function () {
_currentLastSyncLabel.hidden = false;
}
}
-
-Zotero.Sync.Runner = new Zotero.Sync.Runner_Module;
diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js
index 1a6f09e6b..7cbb0ee92 100644
--- a/chrome/content/zotero/xpcom/zotero.js
+++ b/chrome/content/zotero/xpcom/zotero.js
@@ -607,6 +607,7 @@ Components.utils.import("resource://gre/modules/osfile.jsm");
yield Zotero.Sync.Data.Local.init();
yield Zotero.Sync.Data.Utilities.init();
Zotero.Sync.EventListeners.init();
+ Zotero.Sync.Runner = new Zotero.Sync.Runner_Module;
Zotero.MIMETypeHandler.init();
yield Zotero.Proxies.init();
@@ -2706,6 +2707,9 @@ Zotero.Browser = new function() {
if(!win) {
var win = Services.ww.activeWindow;
}
+ if (!win) {
+ throw new Error("Parent window not available for hidden browser");
+ }
}
// Create a hidden browser
diff --git a/chrome/content/zotero/zoteroPane.js b/chrome/content/zotero/zoteroPane.js
index 7840b2444..0126a10cb 100644
--- a/chrome/content/zotero/zoteroPane.js
+++ b/chrome/content/zotero/zoteroPane.js
@@ -1325,6 +1325,7 @@ var ZoteroPane = new function()
else if (item.isAttachment()) {
var attachmentBox = document.getElementById('zotero-attachment-box');
attachmentBox.mode = this.collectionsView.editable ? 'edit' : 'view';
+ yield item.loadNote();
attachmentBox.item = item;
document.getElementById('zotero-item-pane-content').selectedIndex = 3;
@@ -3692,38 +3693,41 @@ var ZoteroPane = new function()
}
}
else {
- if (!item.isImportedAttachment() || !Zotero.Sync.Storage.downloadAsNeeded(item.libraryID)) {
+ if (!item.isImportedAttachment()
+ || !Zotero.Sync.Storage.Local.downloadAsNeeded(item.libraryID)) {
this.showAttachmentNotFoundDialog(itemID, noLocateOnMissing);
return;
}
let downloadedItem = item;
- yield Zotero.Sync.Storage.downloadFile(
- downloadedItem,
- {
- onProgress: function (progress, progressMax) {}
- }
- )
- .then(function () {
- if (!downloadedItem.getFile()) {
- ZoteroPane_Local.showAttachmentNotFoundDialog(downloadedItem.id, noLocateOnMissing);
- return;
- }
-
- // check if unchanged?
- // maybe not necessary, since we'll get an error if there's an error
-
-
- Zotero.Notifier.trigger('redraw', 'item', []);
- Zotero.debug('downloaded');
- Zotero.debug(downloadedItem.id);
- return ZoteroPane_Local.viewAttachment(downloadedItem.id, event, false, forceExternalViewer);
- })
- .catch(function (e) {
+ try {
+ yield Zotero.Sync.Runner.downloadFile(
+ downloadedItem,
+ {
+ onProgress: function (progress, progressMax) {}
+ }
+ );
+ }
+ catch (e) {
// TODO: show error somewhere else
Zotero.debug(e, 1);
ZoteroPane_Local.syncAlert(e);
- });
+ return;
+ }
+
+ if (!(yield downloadedItem.getFilePathAsync())) {
+ ZoteroPane_Local.showAttachmentNotFoundDialog(downloadedItem.id, noLocateOnMissing);
+ return;
+ }
+
+ // check if unchanged?
+ // maybe not necessary, since we'll get an error if there's an error
+
+
+ Zotero.Notifier.trigger('redraw', 'item', []);
+ Zotero.debug('downloaded');
+ Zotero.debug(downloadedItem.id);
+ return ZoteroPane_Local.viewAttachment(downloadedItem.id, event, false, forceExternalViewer);
}
}
});
@@ -3962,7 +3966,7 @@ var ZoteroPane = new function()
this.syncAlert = function (e) {
- e = Zotero.Sync.Runner.parseSyncError(e);
+ e = Zotero.Sync.Runner.parseError(e);
var ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"]
.getService(Components.interfaces.nsIPromptService);
diff --git a/chrome/content/zotero/zoteroPane.xul b/chrome/content/zotero/zoteroPane.xul
index 4f6b73121..003a2da04 100644
--- a/chrome/content/zotero/zoteroPane.xul
+++ b/chrome/content/zotero/zoteroPane.xul
@@ -195,9 +195,10 @@
</hbox>
<hbox align="center" pack="end">
<hbox id="zotero-tb-sync-progress-box" hidden="true" align="center">
+ <!-- TODO: localize -->
<toolbarbutton id="zotero-tb-sync-storage-cancel"
- tooltiptext="Cancel Storage Sync"
- oncommand="Zotero.Sync.Storage.QueueManager.cancel()"/>
+ tooltiptext="Stop sync"
+ oncommand="Zotero.Sync.Runner.stop()"/>
<progressmeter id="zotero-tb-sync-progress" mode="determined"
value="0" tooltip="zotero-tb-sync-progress-tooltip">
</progressmeter>
diff --git a/chrome/locale/en-US/zotero/zotero.properties b/chrome/locale/en-US/zotero/zotero.properties
index 4dcb6ab42..db276f36e 100644
--- a/chrome/locale/en-US/zotero/zotero.properties
+++ b/chrome/locale/en-US/zotero/zotero.properties
@@ -961,12 +961,12 @@ rtfScan.saveTitle = Select a location in which to save the formatted file
rtfScan.scannedFileSuffix = (Scanned)
-file.accessError.theFile = The file '%S'
-file.accessError.aFile = A file
-file.accessError.cannotBe = cannot be
-file.accessError.created = created
-file.accessError.updated = updated
-file.accessError.deleted = deleted
+file.accessError.theFileCannotBeCreated = The file '%S' cannot be created.
+file.accessError.theFileCannotBeUpdated = The file '%S' cannot be updated.
+file.accessError.theFileCannotBeDeleted = The file '%S' cannot be deleted.
+file.accessError.aFileCannotBeCreated = A file cannot be created.
+file.accessError.aFileCannotBeUpdated = A file cannot be updated.
+file.accessError.aFileCannotBeDeleted = A file cannot be deleted.
file.accessError.message.windows = Check that the file is not currently in use, that its permissions allow write access, and that it has a valid filename.
file.accessError.message.other = Check that the file is not currently in use and that its permissions allow write access.
file.accessError.restart = Restarting your computer or disabling security software may also help.
diff --git a/components/zotero-service.js b/components/zotero-service.js
index 63cfc3a99..a3f114dd2 100644
--- a/components/zotero-service.js
+++ b/components/zotero-service.js
@@ -107,11 +107,12 @@ const xpcomFilesLocal = [
'sync/syncRunner',
'sync/syncUtilities',
'storage',
+ 'storage/storageEngine',
+ 'storage/storageLocal',
+ 'storage/storageRequest',
+ 'storage/storageResult',
+ 'storage/storageUtilities',
'storage/streamListener',
- 'storage/queueManager',
- 'storage/queue',
- 'storage/request',
- 'storage/mode',
'storage/zfs',
'storage/webdav',
'syncedSettings',
diff --git a/test/resource/httpd.js b/test/resource/httpd.js
new file mode 100644
index 000000000..c72e47f50
--- /dev/null
+++ b/test/resource/httpd.js
@@ -0,0 +1,5356 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * An implementation of an HTTP server both as a loadable script and as an XPCOM
+ * component. See the accompanying README file for user documentation on
+ * httpd.js.
+ */
+
+this.EXPORTED_SYMBOLS = [
+ "HTTP_400",
+ "HTTP_401",
+ "HTTP_402",
+ "HTTP_403",
+ "HTTP_404",
+ "HTTP_405",
+ "HTTP_406",
+ "HTTP_407",
+ "HTTP_408",
+ "HTTP_409",
+ "HTTP_410",
+ "HTTP_411",
+ "HTTP_412",
+ "HTTP_413",
+ "HTTP_414",
+ "HTTP_415",
+ "HTTP_417",
+ "HTTP_500",
+ "HTTP_501",
+ "HTTP_502",
+ "HTTP_503",
+ "HTTP_504",
+ "HTTP_505",
+ "HttpError",
+ "HttpServer",
+];
+
+Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
+const CC = Components.Constructor;
+
+const PR_UINT32_MAX = Math.pow(2, 32) - 1;
+
+/** True if debugging output is enabled, false otherwise. */
+var DEBUG = false; // non-const *only* so tweakable in server tests
+
+/** True if debugging output should be timestamped. */
+var DEBUG_TIMESTAMP = false; // non-const so tweakable in server tests
+
+var gGlobalObject = this;
+
+/**
+ * Asserts that the given condition holds. If it doesn't, the given message is
+ * dumped, a stack trace is printed, and an exception is thrown to attempt to
+ * stop execution (which unfortunately must rely upon the exception not being
+ * accidentally swallowed by the code that uses it).
+ */
+function NS_ASSERT(cond, msg)
+{
+ if (DEBUG && !cond)
+ {
+ dumpn("###!!!");
+ dumpn("###!!! ASSERTION" + (msg ? ": " + msg : "!"));
+ dumpn("###!!! Stack follows:");
+
+ var stack = new Error().stack.split(/\n/);
+ dumpn(stack.map(function(val) { return "###!!! " + val; }).join("\n"));
+
+ throw Cr.NS_ERROR_ABORT;
+ }
+}
+
+/** Constructs an HTTP error object. */
+this.HttpError = function HttpError(code, description)
+{
+ this.code = code;
+ this.description = description;
+}
+HttpError.prototype =
+{
+ toString: function()
+ {
+ return this.code + " " + this.description;
+ }
+};
+
+/**
+ * Errors thrown to trigger specific HTTP server responses.
+ */
+this.HTTP_400 = new HttpError(400, "Bad Request");
+this.HTTP_401 = new HttpError(401, "Unauthorized");
+this.HTTP_402 = new HttpError(402, "Payment Required");
+this.HTTP_403 = new HttpError(403, "Forbidden");
+this.HTTP_404 = new HttpError(404, "Not Found");
+this.HTTP_405 = new HttpError(405, "Method Not Allowed");
+this.HTTP_406 = new HttpError(406, "Not Acceptable");
+this.HTTP_407 = new HttpError(407, "Proxy Authentication Required");
+this.HTTP_408 = new HttpError(408, "Request Timeout");
+this.HTTP_409 = new HttpError(409, "Conflict");
+this.HTTP_410 = new HttpError(410, "Gone");
+this.HTTP_411 = new HttpError(411, "Length Required");
+this.HTTP_412 = new HttpError(412, "Precondition Failed");
+this.HTTP_413 = new HttpError(413, "Request Entity Too Large");
+this.HTTP_414 = new HttpError(414, "Request-URI Too Long");
+this.HTTP_415 = new HttpError(415, "Unsupported Media Type");
+this.HTTP_417 = new HttpError(417, "Expectation Failed");
+
+this.HTTP_500 = new HttpError(500, "Internal Server Error");
+this.HTTP_501 = new HttpError(501, "Not Implemented");
+this.HTTP_502 = new HttpError(502, "Bad Gateway");
+this.HTTP_503 = new HttpError(503, "Service Unavailable");
+this.HTTP_504 = new HttpError(504, "Gateway Timeout");
+this.HTTP_505 = new HttpError(505, "HTTP Version Not Supported");
+
+/** Creates a hash with fields corresponding to the values in arr. */
+function array2obj(arr)
+{
+ var obj = {};
+ for (var i = 0; i < arr.length; i++)
+ obj[arr[i]] = arr[i];
+ return obj;
+}
+
+/** Returns an array of the integers x through y, inclusive. */
+function range(x, y)
+{
+ var arr = [];
+ for (var i = x; i <= y; i++)
+ arr.push(i);
+ return arr;
+}
+
+/** An object (hash) whose fields are the numbers of all HTTP error codes. */
+const HTTP_ERROR_CODES = array2obj(range(400, 417).concat(range(500, 505)));
+
+
+/**
+ * The character used to distinguish hidden files from non-hidden files, a la
+ * the leading dot in Apache. Since that mechanism also hides files from
+ * easy display in LXR, ls output, etc. however, we choose instead to use a
+ * suffix character. If a requested file ends with it, we append another
+ * when getting the file on the server. If it doesn't, we just look up that
+ * file. Therefore, any file whose name ends with exactly one of the character
+ * is "hidden" and available for use by the server.
+ */
+const HIDDEN_CHAR = "^";
+
+/**
+ * The file name suffix indicating the file containing overridden headers for
+ * a requested file.
+ */
+const HEADERS_SUFFIX = HIDDEN_CHAR + "headers" + HIDDEN_CHAR;
+
+/** Type used to denote SJS scripts for CGI-like functionality. */
+const SJS_TYPE = "sjs";
+
+/** Base for relative timestamps produced by dumpn(). */
+var firstStamp = 0;
+
+/** dump(str) with a trailing "\n" -- only outputs if DEBUG. */
+function dumpn(str)
+{
+ if (DEBUG)
+ {
+ var prefix = "HTTPD-INFO | ";
+ if (DEBUG_TIMESTAMP)
+ {
+ if (firstStamp === 0)
+ firstStamp = Date.now();
+
+ var elapsed = Date.now() - firstStamp; // milliseconds
+ var min = Math.floor(elapsed / 60000);
+ var sec = (elapsed % 60000) / 1000;
+
+ if (sec < 10)
+ prefix += min + ":0" + sec.toFixed(3) + " | ";
+ else
+ prefix += min + ":" + sec.toFixed(3) + " | ";
+ }
+
+ dump(prefix + str + "\n");
+ }
+}
+
+/** Dumps the current JS stack if DEBUG. */
+function dumpStack()
+{
+ // peel off the frames for dumpStack() and Error()
+ var stack = new Error().stack.split(/\n/).slice(2);
+ stack.forEach(dumpn);
+}
+
+
+/** The XPCOM thread manager. */
+var gThreadManager = null;
+
+/** The XPCOM prefs service. */
+var gRootPrefBranch = null;
+function getRootPrefBranch()
+{
+ if (!gRootPrefBranch)
+ {
+ gRootPrefBranch = Cc["@mozilla.org/preferences-service;1"]
+ .getService(Ci.nsIPrefBranch);
+ }
+ return gRootPrefBranch;
+}
+
+/**
+ * JavaScript constructors for commonly-used classes; precreating these is a
+ * speedup over doing the same from base principles. See the docs at
+ * http://developer.mozilla.org/en/docs/Components.Constructor for details.
+ */
+const ServerSocket = CC("@mozilla.org/network/server-socket;1",
+ "nsIServerSocket",
+ "init");
+const ScriptableInputStream = CC("@mozilla.org/scriptableinputstream;1",
+ "nsIScriptableInputStream",
+ "init");
+const Pipe = CC("@mozilla.org/pipe;1",
+ "nsIPipe",
+ "init");
+const FileInputStream = CC("@mozilla.org/network/file-input-stream;1",
+ "nsIFileInputStream",
+ "init");
+const ConverterInputStream = CC("@mozilla.org/intl/converter-input-stream;1",
+ "nsIConverterInputStream",
+ "init");
+const WritablePropertyBag = CC("@mozilla.org/hash-property-bag;1",
+ "nsIWritablePropertyBag2");
+const SupportsString = CC("@mozilla.org/supports-string;1",
+ "nsISupportsString");
+
+/* These two are non-const only so a test can overwrite them. */
+var BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
+ "nsIBinaryInputStream",
+ "setInputStream");
+var BinaryOutputStream = CC("@mozilla.org/binaryoutputstream;1",
+ "nsIBinaryOutputStream",
+ "setOutputStream");
+
+/**
+ * Returns the RFC 822/1123 representation of a date.
+ *
+ * @param date : Number
+ * the date, in milliseconds from midnight (00:00:00), January 1, 1970 GMT
+ * @returns string
+ * the representation of the given date
+ */
+function toDateString(date)
+{
+ //
+ // rfc1123-date = wkday "," SP date1 SP time SP "GMT"
+ // date1 = 2DIGIT SP month SP 4DIGIT
+ // ; day month year (e.g., 02 Jun 1982)
+ // time = 2DIGIT ":" 2DIGIT ":" 2DIGIT
+ // ; 00:00:00 - 23:59:59
+ // wkday = "Mon" | "Tue" | "Wed"
+ // | "Thu" | "Fri" | "Sat" | "Sun"
+ // month = "Jan" | "Feb" | "Mar" | "Apr"
+ // | "May" | "Jun" | "Jul" | "Aug"
+ // | "Sep" | "Oct" | "Nov" | "Dec"
+ //
+
+ const wkdayStrings = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"];
+ const monthStrings = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
+ "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
+
+ /**
+ * Processes a date and returns the encoded UTC time as a string according to
+ * the format specified in RFC 2616.
+ *
+ * @param date : Date
+ * the date to process
+ * @returns string
+ * a string of the form "HH:MM:SS", ranging from "00:00:00" to "23:59:59"
+ */
+ function toTime(date)
+ {
+ var hrs = date.getUTCHours();
+ var rv = (hrs < 10) ? "0" + hrs : hrs;
+
+ var mins = date.getUTCMinutes();
+ rv += ":";
+ rv += (mins < 10) ? "0" + mins : mins;
+
+ var secs = date.getUTCSeconds();
+ rv += ":";
+ rv += (secs < 10) ? "0" + secs : secs;
+
+ return rv;
+ }
+
+ /**
+ * Processes a date and returns the encoded UTC date as a string according to
+ * the date1 format specified in RFC 2616.
+ *
+ * @param date : Date
+ * the date to process
+ * @returns string
+ * a string of the form "HH:MM:SS", ranging from "00:00:00" to "23:59:59"
+ */
+ function toDate1(date)
+ {
+ var day = date.getUTCDate();
+ var month = date.getUTCMonth();
+ var year = date.getUTCFullYear();
+
+ var rv = (day < 10) ? "0" + day : day;
+ rv += " " + monthStrings[month];
+ rv += " " + year;
+
+ return rv;
+ }
+
+ date = new Date(date);
+
+ const fmtString = "%wkday%, %date1% %time% GMT";
+ var rv = fmtString.replace("%wkday%", wkdayStrings[date.getUTCDay()]);
+ rv = rv.replace("%time%", toTime(date));
+ return rv.replace("%date1%", toDate1(date));
+}
+
+/**
+ * Prints out a human-readable representation of the object o and its fields,
+ * omitting those whose names begin with "_" if showMembers != true (to ignore
+ * "private" properties exposed via getters/setters).
+ */
+function printObj(o, showMembers)
+{
+ var s = "******************************\n";
+ s += "o = {\n";
+ for (var i in o)
+ {
+ if (typeof(i) != "string" ||
+ (showMembers || (i.length > 0 && i[0] != "_")))
+ s+= " " + i + ": " + o[i] + ",\n";
+ }
+ s += " };\n";
+ s += "******************************";
+ dumpn(s);
+}
+
+/**
+ * Instantiates a new HTTP server.
+ */
+function nsHttpServer()
+{
+ if (!gThreadManager)
+ gThreadManager = Cc["@mozilla.org/thread-manager;1"].getService();
+
+ /** The port on which this server listens. */
+ this._port = undefined;
+
+ /** The socket associated with this. */
+ this._socket = null;
+
+ /** The handler used to process requests to this server. */
+ this._handler = new ServerHandler(this);
+
+ /** Naming information for this server. */
+ this._identity = new ServerIdentity();
+
+ /**
+ * Indicates when the server is to be shut down at the end of the request.
+ */
+ this._doQuit = false;
+
+ /**
+ * True if the socket in this is closed (and closure notifications have been
+ * sent and processed if the socket was ever opened), false otherwise.
+ */
+ this._socketClosed = true;
+
+ /**
+ * Used for tracking existing connections and ensuring that all connections
+ * are properly cleaned up before server shutdown; increases by 1 for every
+ * new incoming connection.
+ */
+ this._connectionGen = 0;
+
+ /**
+ * Hash of all open connections, indexed by connection number at time of
+ * creation.
+ */
+ this._connections = {};
+}
+nsHttpServer.prototype =
+{
+ classID: Components.ID("{54ef6f81-30af-4b1d-ac55-8ba811293e41}"),
+
+ // NSISERVERSOCKETLISTENER
+
+ /**
+ * Processes an incoming request coming in on the given socket and contained
+ * in the given transport.
+ *
+ * @param socket : nsIServerSocket
+ * the socket through which the request was served
+ * @param trans : nsISocketTransport
+ * the transport for the request/response
+ * @see nsIServerSocketListener.onSocketAccepted
+ */
+ onSocketAccepted: function(socket, trans)
+ {
+ dumpn("*** onSocketAccepted(socket=" + socket + ", trans=" + trans + ")");
+
+ dumpn(">>> new connection on " + trans.host + ":" + trans.port);
+
+ const SEGMENT_SIZE = 8192;
+ const SEGMENT_COUNT = 1024;
+ try
+ {
+ var input = trans.openInputStream(0, SEGMENT_SIZE, SEGMENT_COUNT)
+ .QueryInterface(Ci.nsIAsyncInputStream);
+ var output = trans.openOutputStream(0, 0, 0);
+ }
+ catch (e)
+ {
+ dumpn("*** error opening transport streams: " + e);
+ trans.close(Cr.NS_BINDING_ABORTED);
+ return;
+ }
+
+ var connectionNumber = ++this._connectionGen;
+
+ try
+ {
+ var conn = new Connection(input, output, this, socket.port, trans.port,
+ connectionNumber);
+ var reader = new RequestReader(conn);
+
+ // XXX add request timeout functionality here!
+
+ // Note: must use main thread here, or we might get a GC that will cause
+ // threadsafety assertions. We really need to fix XPConnect so that
+ // you can actually do things in multi-threaded JS. :-(
+ input.asyncWait(reader, 0, 0, gThreadManager.mainThread);
+ }
+ catch (e)
+ {
+ // Assume this connection can't be salvaged and bail on it completely;
+ // don't attempt to close it so that we can assert that any connection
+ // being closed is in this._connections.
+ dumpn("*** error in initial request-processing stages: " + e);
+ trans.close(Cr.NS_BINDING_ABORTED);
+ return;
+ }
+
+ this._connections[connectionNumber] = conn;
+ dumpn("*** starting connection " + connectionNumber);
+ },
+
+ /**
+ * Called when the socket associated with this is closed.
+ *
+ * @param socket : nsIServerSocket
+ * the socket being closed
+ * @param status : nsresult
+ * the reason the socket stopped listening (NS_BINDING_ABORTED if the server
+ * was stopped using nsIHttpServer.stop)
+ * @see nsIServerSocketListener.onStopListening
+ */
+ onStopListening: function(socket, status)
+ {
+ dumpn(">>> shutting down server on port " + socket.port);
+ for (var n in this._connections) {
+ if (!this._connections[n]._requestStarted) {
+ this._connections[n].close();
+ }
+ }
+ this._socketClosed = true;
+ if (this._hasOpenConnections()) {
+ dumpn("*** open connections!!!");
+ }
+ if (!this._hasOpenConnections())
+ {
+ dumpn("*** no open connections, notifying async from onStopListening");
+
+ // Notify asynchronously so that any pending teardown in stop() has a
+ // chance to run first.
+ var self = this;
+ var stopEvent =
+ {
+ run: function()
+ {
+ dumpn("*** _notifyStopped async callback");
+ self._notifyStopped();
+ }
+ };
+ gThreadManager.currentThread
+ .dispatch(stopEvent, Ci.nsIThread.DISPATCH_NORMAL);
+ }
+ },
+
+ // NSIHTTPSERVER
+
+ //
+ // see nsIHttpServer.start
+ //
+ start: function(port)
+ {
+ this._start(port, "localhost")
+ },
+
+ _start: function(port, host)
+ {
+ if (this._socket)
+ throw Cr.NS_ERROR_ALREADY_INITIALIZED;
+
+ this._port = port;
+ this._doQuit = this._socketClosed = false;
+
+ this._host = host;
+
+ // The listen queue needs to be long enough to handle
+ // network.http.max-persistent-connections-per-server or
+ // network.http.max-persistent-connections-per-proxy concurrent
+ // connections, plus a safety margin in case some other process is
+ // talking to the server as well.
+ var prefs = getRootPrefBranch();
+ var maxConnections = 5 + Math.max(
+ prefs.getIntPref("network.http.max-persistent-connections-per-server"),
+ prefs.getIntPref("network.http.max-persistent-connections-per-proxy"));
+
+ try
+ {
+ var loopback = true;
+ if (this._host != "127.0.0.1" && this._host != "localhost") {
+ var loopback = false;
+ }
+
+ // When automatically selecting a port, sometimes the chosen port is
+ // "blocked" from clients. We don't want to use these ports because
+ // tests will intermittently fail. So, we simply keep trying to to
+ // get a server socket until a valid port is obtained. We limit
+ // ourselves to finite attempts just so we don't loop forever.
+ var ios = Cc["@mozilla.org/network/io-service;1"]
+ .getService(Ci.nsIIOService);
+ var socket;
+ for (var i = 100; i; i--)
+ {
+ var temp = new ServerSocket(this._port,
+ loopback, // true = localhost, false = everybody
+ maxConnections);
+
+ var allowed = ios.allowPort(temp.port, "http");
+ if (!allowed)
+ {
+ dumpn(">>>Warning: obtained ServerSocket listens on a blocked " +
+ "port: " + temp.port);
+ }
+
+ if (!allowed && this._port == -1)
+ {
+ dumpn(">>>Throwing away ServerSocket with bad port.");
+ temp.close();
+ continue;
+ }
+
+ socket = temp;
+ break;
+ }
+
+ if (!socket) {
+ throw new Error("No socket server available. Are there no available ports?");
+ }
+
+ dumpn(">>> listening on port " + socket.port + ", " + maxConnections +
+ " pending connections");
+ socket.asyncListen(this);
+ this._port = socket.port;
+ this._identity._initialize(socket.port, host, true);
+ this._socket = socket;
+ }
+ catch (e)
+ {
+ dump("\n!!! could not start server on port " + port + ": " + e + "\n\n");
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+ }
+ },
+
+ //
+ // see nsIHttpServer.stop
+ //
+ stop: function(callback)
+ {
+ if (!callback)
+ throw Cr.NS_ERROR_NULL_POINTER;
+ if (!this._socket)
+ throw Cr.NS_ERROR_UNEXPECTED;
+
+ this._stopCallback = typeof callback === "function"
+ ? callback
+ : function() { callback.onStopped(); };
+
+ dumpn(">>> stopping listening on port " + this._socket.port);
+ this._socket.close();
+ this._socket = null;
+
+ // We can't have this identity any more, and the port on which we're running
+ // this server now could be meaningless the next time around.
+ this._identity._teardown();
+
+ this._doQuit = false;
+
+ // socket-close notification and pending request completion happen async
+ },
+
+ //
+ // see nsIHttpServer.registerFile
+ //
+ registerFile: function(path, file)
+ {
+ if (file && (!file.exists() || file.isDirectory()))
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ this._handler.registerFile(path, file);
+ },
+
+ //
+ // see nsIHttpServer.registerDirectory
+ //
+ registerDirectory: function(path, directory)
+ {
+ // XXX true path validation!
+ if (path.charAt(0) != "/" ||
+ path.charAt(path.length - 1) != "/" ||
+ (directory &&
+ (!directory.exists() || !directory.isDirectory())))
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ // XXX determine behavior of nonexistent /foo/bar when a /foo/bar/ mapping
+ // exists!
+
+ this._handler.registerDirectory(path, directory);
+ },
+
+ //
+ // see nsIHttpServer.registerPathHandler
+ //
+ registerPathHandler: function(path, handler)
+ {
+ this._handler.registerPathHandler(path, handler);
+ },
+
+ //
+ // see nsIHttpServer.registerPrefixHandler
+ //
+ registerPrefixHandler: function(prefix, handler)
+ {
+ this._handler.registerPrefixHandler(prefix, handler);
+ },
+
+ //
+ // see nsIHttpServer.registerErrorHandler
+ //
+ registerErrorHandler: function(code, handler)
+ {
+ this._handler.registerErrorHandler(code, handler);
+ },
+
+ //
+ // see nsIHttpServer.setIndexHandler
+ //
+ setIndexHandler: function(handler)
+ {
+ this._handler.setIndexHandler(handler);
+ },
+
+ //
+ // see nsIHttpServer.registerContentType
+ //
+ registerContentType: function(ext, type)
+ {
+ this._handler.registerContentType(ext, type);
+ },
+
+ //
+ // see nsIHttpServer.serverIdentity
+ //
+ get identity()
+ {
+ return this._identity;
+ },
+
+ //
+ // see nsIHttpServer.getState
+ //
+ getState: function(path, k)
+ {
+ return this._handler._getState(path, k);
+ },
+
+ //
+ // see nsIHttpServer.setState
+ //
+ setState: function(path, k, v)
+ {
+ return this._handler._setState(path, k, v);
+ },
+
+ //
+ // see nsIHttpServer.getSharedState
+ //
+ getSharedState: function(k)
+ {
+ return this._handler._getSharedState(k);
+ },
+
+ //
+ // see nsIHttpServer.setSharedState
+ //
+ setSharedState: function(k, v)
+ {
+ return this._handler._setSharedState(k, v);
+ },
+
+ //
+ // see nsIHttpServer.getObjectState
+ //
+ getObjectState: function(k)
+ {
+ return this._handler._getObjectState(k);
+ },
+
+ //
+ // see nsIHttpServer.setObjectState
+ //
+ setObjectState: function(k, v)
+ {
+ return this._handler._setObjectState(k, v);
+ },
+
+
+ // NSISUPPORTS
+
+ //
+ // see nsISupports.QueryInterface
+ //
+ QueryInterface: function(iid)
+ {
+ if (iid.equals(Ci.nsIHttpServer) ||
+ iid.equals(Ci.nsIServerSocketListener) ||
+ iid.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ },
+
+
+ // NON-XPCOM PUBLIC API
+
+ /**
+ * Returns true iff this server is not running (and is not in the process of
+ * serving any requests still to be processed when the server was last
+ * stopped after being run).
+ */
+ isStopped: function()
+ {
+ return this._socketClosed && !this._hasOpenConnections();
+ },
+
+ // PRIVATE IMPLEMENTATION
+
+ /** True if this server has any open connections to it, false otherwise. */
+ _hasOpenConnections: function()
+ {
+ //
+ // If we have any open connections, they're tracked as numeric properties on
+ // |this._connections|. The non-standard __count__ property could be used
+ // to check whether there are any properties, but standard-wise, even
+ // looking forward to ES5, there's no less ugly yet still O(1) way to do
+ // this.
+ //
+ for (var n in this._connections)
+ return true;
+ return false;
+ },
+
+ /** Calls the server-stopped callback provided when stop() was called. */
+ _notifyStopped: function()
+ {
+ NS_ASSERT(this._stopCallback !== null, "double-notifying?");
+ NS_ASSERT(!this._hasOpenConnections(), "should be done serving by now");
+
+ //
+ // NB: We have to grab this now, null out the member, *then* call the
+ // callback here, or otherwise the callback could (indirectly) futz with
+ // this._stopCallback by starting and immediately stopping this, at
+ // which point we'd be nulling out a field we no longer have a right to
+ // modify.
+ //
+ var callback = this._stopCallback;
+ this._stopCallback = null;
+ try
+ {
+ callback();
+ }
+ catch (e)
+ {
+ // not throwing because this is specified as being usually (but not
+ // always) asynchronous
+ dump("!!! error running onStopped callback: " + e + "\n");
+ }
+ },
+
+ /**
+ * Notifies this server that the given connection has been closed.
+ *
+ * @param connection : Connection
+ * the connection that was closed
+ */
+ _connectionClosed: function(connection)
+ {
+ NS_ASSERT(connection.number in this._connections,
+ "closing a connection " + this + " that we never added to the " +
+ "set of open connections?");
+ NS_ASSERT(this._connections[connection.number] === connection,
+ "connection number mismatch? " +
+ this._connections[connection.number]);
+ delete this._connections[connection.number];
+
+ // Fire a pending server-stopped notification if it's our responsibility.
+ if (!this._hasOpenConnections() && this._socketClosed)
+ this._notifyStopped();
+ // Bug 508125: Add a GC here else we'll use gigabytes of memory running
+ // mochitests. We can't rely on xpcshell doing an automated GC, as that
+ // would interfere with testing GC stuff...
+ Components.utils.forceGC();
+ },
+
+ /**
+ * Requests that the server be shut down when possible.
+ */
+ _requestQuit: function()
+ {
+ dumpn(">>> requesting a quit");
+ dumpStack();
+ this._doQuit = true;
+ }
+};
+
+this.HttpServer = nsHttpServer;
+
+//
+// RFC 2396 section 3.2.2:
+//
+// host = hostname | IPv4address
+// hostname = *( domainlabel "." ) toplabel [ "." ]
+// domainlabel = alphanum | alphanum *( alphanum | "-" ) alphanum
+// toplabel = alpha | alpha *( alphanum | "-" ) alphanum
+// IPv4address = 1*digit "." 1*digit "." 1*digit "." 1*digit
+//
+
+const HOST_REGEX =
+ new RegExp("^(?:" +
+ // *( domainlabel "." )
+ "(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)*" +
+ // toplabel
+ "[a-z](?:[a-z0-9-]*[a-z0-9])?" +
+ "|" +
+ // IPv4 address
+ "\\d+\\.\\d+\\.\\d+\\.\\d+" +
+ ")$",
+ "i");
+
+
+/**
+ * Represents the identity of a server. An identity consists of a set of
+ * (scheme, host, port) tuples denoted as locations (allowing a single server to
+ * serve multiple sites or to be used behind both HTTP and HTTPS proxies for any
+ * host/port). Any incoming request must be to one of these locations, or it
+ * will be rejected with an HTTP 400 error. One location, denoted as the
+ * primary location, is the location assigned in contexts where a location
+ * cannot otherwise be endogenously derived, such as for HTTP/1.0 requests.
+ *
+ * A single identity may contain at most one location per unique host/port pair;
+ * other than that, no restrictions are placed upon what locations may
+ * constitute an identity.
+ */
+function ServerIdentity()
+{
+ /** The scheme of the primary location. */
+ this._primaryScheme = "http";
+
+ /** The hostname of the primary location. */
+ this._primaryHost = "127.0.0.1"
+
+ /** The port number of the primary location. */
+ this._primaryPort = -1;
+
+ /**
+ * The current port number for the corresponding server, stored so that a new
+ * primary location can always be set if the current one is removed.
+ */
+ this._defaultPort = -1;
+
+ /**
+ * Maps hosts to maps of ports to schemes, e.g. the following would represent
+ * https://example.com:789/ and http://example.org/:
+ *
+ * {
+ * "xexample.com": { 789: "https" },
+ * "xexample.org": { 80: "http" }
+ * }
+ *
+ * Note the "x" prefix on hostnames, which prevents collisions with special
+ * JS names like "prototype".
+ */
+ this._locations = { "xlocalhost": {} };
+}
+ServerIdentity.prototype =
+{
+ // NSIHTTPSERVERIDENTITY
+
+ //
+ // see nsIHttpServerIdentity.primaryScheme
+ //
+ get primaryScheme()
+ {
+ if (this._primaryPort === -1)
+ throw Cr.NS_ERROR_NOT_INITIALIZED;
+ return this._primaryScheme;
+ },
+
+ //
+ // see nsIHttpServerIdentity.primaryHost
+ //
+ get primaryHost()
+ {
+ if (this._primaryPort === -1)
+ throw Cr.NS_ERROR_NOT_INITIALIZED;
+ return this._primaryHost;
+ },
+
+ //
+ // see nsIHttpServerIdentity.primaryPort
+ //
+ get primaryPort()
+ {
+ if (this._primaryPort === -1)
+ throw Cr.NS_ERROR_NOT_INITIALIZED;
+ return this._primaryPort;
+ },
+
+ //
+ // see nsIHttpServerIdentity.add
+ //
+ add: function(scheme, host, port)
+ {
+ this._validate(scheme, host, port);
+
+ var entry = this._locations["x" + host];
+ if (!entry)
+ this._locations["x" + host] = entry = {};
+
+ entry[port] = scheme;
+ },
+
+ //
+ // see nsIHttpServerIdentity.remove
+ //
+ remove: function(scheme, host, port)
+ {
+ this._validate(scheme, host, port);
+
+ var entry = this._locations["x" + host];
+ if (!entry)
+ return false;
+
+ var present = port in entry;
+ delete entry[port];
+
+ if (this._primaryScheme == scheme &&
+ this._primaryHost == host &&
+ this._primaryPort == port &&
+ this._defaultPort !== -1)
+ {
+ // Always keep at least one identity in existence at any time, unless
+ // we're in the process of shutting down (the last condition above).
+ this._primaryPort = -1;
+ this._initialize(this._defaultPort, host, false);
+ }
+
+ return present;
+ },
+
+ //
+ // see nsIHttpServerIdentity.has
+ //
+ has: function(scheme, host, port)
+ {
+ this._validate(scheme, host, port);
+
+ return "x" + host in this._locations &&
+ scheme === this._locations["x" + host][port];
+ },
+
+ //
+ // see nsIHttpServerIdentity.has
+ //
+ getScheme: function(host, port)
+ {
+ this._validate("http", host, port);
+
+ var entry = this._locations["x" + host];
+ if (!entry)
+ return "";
+
+ return entry[port] || "";
+ },
+
+ //
+ // see nsIHttpServerIdentity.setPrimary
+ //
+ setPrimary: function(scheme, host, port)
+ {
+ this._validate(scheme, host, port);
+
+ this.add(scheme, host, port);
+
+ this._primaryScheme = scheme;
+ this._primaryHost = host;
+ this._primaryPort = port;
+ },
+
+
+ // NSISUPPORTS
+
+ //
+ // see nsISupports.QueryInterface
+ //
+ QueryInterface: function(iid)
+ {
+ if (iid.equals(Ci.nsIHttpServerIdentity) || iid.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ },
+
+
+ // PRIVATE IMPLEMENTATION
+
+ /**
+ * Initializes the primary name for the corresponding server, based on the
+ * provided port number.
+ */
+ _initialize: function(port, host, addSecondaryDefault)
+ {
+ this._host = host;
+ if (this._primaryPort !== -1)
+ this.add("http", host, port);
+ else
+ this.setPrimary("http", "localhost", port);
+ this._defaultPort = port;
+
+ // Only add this if we're being called at server startup
+ if (addSecondaryDefault && host != "127.0.0.1")
+ this.add("http", "127.0.0.1", port);
+ },
+
+ /**
+ * Called at server shutdown time, unsets the primary location only if it was
+ * the default-assigned location and removes the default location from the
+ * set of locations used.
+ */
+ _teardown: function()
+ {
+ if (this._host != "127.0.0.1") {
+ // Not the default primary location, nothing special to do here
+ this.remove("http", "127.0.0.1", this._defaultPort);
+ }
+
+ // This is a *very* tricky bit of reasoning here; make absolutely sure the
+ // tests for this code pass before you commit changes to it.
+ if (this._primaryScheme == "http" &&
+ this._primaryHost == this._host &&
+ this._primaryPort == this._defaultPort)
+ {
+ // Make sure we don't trigger the readding logic in .remove(), then remove
+ // the default location.
+ var port = this._defaultPort;
+ this._defaultPort = -1;
+ this.remove("http", this._host, port);
+
+ // Ensure a server start triggers the setPrimary() path in ._initialize()
+ this._primaryPort = -1;
+ }
+ else
+ {
+ // No reason not to remove directly as it's not our primary location
+ this.remove("http", this._host, this._defaultPort);
+ }
+ },
+
+ /**
+ * Ensures scheme, host, and port are all valid with respect to RFC 2396.
+ *
+ * @throws NS_ERROR_ILLEGAL_VALUE
+ * if any argument doesn't match the corresponding production
+ */
+ _validate: function(scheme, host, port)
+ {
+ if (scheme !== "http" && scheme !== "https")
+ {
+ dumpn("*** server only supports http/https schemes: '" + scheme + "'");
+ dumpStack();
+ throw Cr.NS_ERROR_ILLEGAL_VALUE;
+ }
+ if (!HOST_REGEX.test(host))
+ {
+ dumpn("*** unexpected host: '" + host + "'");
+ throw Cr.NS_ERROR_ILLEGAL_VALUE;
+ }
+ if (port < 0 || port > 65535)
+ {
+ dumpn("*** unexpected port: '" + port + "'");
+ throw Cr.NS_ERROR_ILLEGAL_VALUE;
+ }
+ }
+};
+
+
+/**
+ * Represents a connection to the server (and possibly in the future the thread
+ * on which the connection is processed).
+ *
+ * @param input : nsIInputStream
+ * stream from which incoming data on the connection is read
+ * @param output : nsIOutputStream
+ * stream to write data out the connection
+ * @param server : nsHttpServer
+ * the server handling the connection
+ * @param port : int
+ * the port on which the server is running
+ * @param outgoingPort : int
+ * the outgoing port used by this connection
+ * @param number : uint
+ * a serial number used to uniquely identify this connection
+ */
+function Connection(input, output, server, port, outgoingPort, number)
+{
+ dumpn("*** opening new connection " + number + " on port " + outgoingPort);
+
+ /** Stream of incoming data. */
+ this.input = input;
+
+ /** Stream for outgoing data. */
+ this.output = output;
+
+ /** The server associated with this request. */
+ this.server = server;
+
+ /** The port on which the server is running. */
+ this.port = port;
+
+ /** The outgoing poort used by this connection. */
+ this._outgoingPort = outgoingPort;
+
+ /** The serial number of this connection. */
+ this.number = number;
+
+ /**
+ * The request for which a response is being generated, null if the
+ * incoming request has not been fully received or if it had errors.
+ */
+ this.request = null;
+
+ /** This allows a connection to disambiguate between a peer initiating a
+ * close and the socket being forced closed on shutdown.
+ */
+ this._closed = false;
+
+ /** State variable for debugging. */
+ this._processed = false;
+
+ /** whether or not 1st line of request has been received */
+ this._requestStarted = false;
+}
+Connection.prototype =
+{
+ /** Closes this connection's input/output streams. */
+ close: function()
+ {
+ if (this._closed)
+ return;
+
+ dumpn("*** closing connection " + this.number +
+ " on port " + this._outgoingPort);
+
+ this.input.close();
+ this.output.close();
+ this._closed = true;
+
+ var server = this.server;
+ server._connectionClosed(this);
+
+ // If an error triggered a server shutdown, act on it now
+ if (server._doQuit)
+ server.stop(function() { /* not like we can do anything better */ });
+ },
+
+ /**
+ * Initiates processing of this connection, using the data in the given
+ * request.
+ *
+ * @param request : Request
+ * the request which should be processed
+ */
+ process: function(request)
+ {
+ NS_ASSERT(!this._closed && !this._processed);
+
+ this._processed = true;
+
+ this.request = request;
+ this.server._handler.handleResponse(this);
+ },
+
+ /**
+ * Initiates processing of this connection, generating a response with the
+ * given HTTP error code.
+ *
+ * @param code : uint
+ * an HTTP code, so in the range [0, 1000)
+ * @param request : Request
+ * incomplete data about the incoming request (since there were errors
+ * during its processing
+ */
+ processError: function(code, request)
+ {
+ NS_ASSERT(!this._closed && !this._processed);
+
+ this._processed = true;
+ this.request = request;
+ this.server._handler.handleError(code, this);
+ },
+
+ /** Converts this to a string for debugging purposes. */
+ toString: function()
+ {
+ return "<Connection(" + this.number +
+ (this.request ? ", " + this.request.path : "") +"): " +
+ (this._closed ? "closed" : "open") + ">";
+ },
+
+ requestStarted: function()
+ {
+ this._requestStarted = true;
+ }
+};
+
+
+
+/** Returns an array of count bytes from the given input stream. */
+function readBytes(inputStream, count)
+{
+ return new BinaryInputStream(inputStream).readByteArray(count);
+}
+
+
+
+/** Request reader processing states; see RequestReader for details. */
+const READER_IN_REQUEST_LINE = 0;
+const READER_IN_HEADERS = 1;
+const READER_IN_BODY = 2;
+const READER_FINISHED = 3;
+
+
+/**
+ * Reads incoming request data asynchronously, does any necessary preprocessing,
+ * and forwards it to the request handler. Processing occurs in three states:
+ *
+ * READER_IN_REQUEST_LINE Reading the request's status line
+ * READER_IN_HEADERS Reading headers in the request
+ * READER_IN_BODY Reading the body of the request
+ * READER_FINISHED Entire request has been read and processed
+ *
+ * During the first two stages, initial metadata about the request is gathered
+ * into a Request object. Once the status line and headers have been processed,
+ * we start processing the body of the request into the Request. Finally, when
+ * the entire body has been read, we create a Response and hand it off to the
+ * ServerHandler to be given to the appropriate request handler.
+ *
+ * @param connection : Connection
+ * the connection for the request being read
+ */
+function RequestReader(connection)
+{
+ /** Connection metadata for this request. */
+ this._connection = connection;
+
+ /**
+ * A container providing line-by-line access to the raw bytes that make up the
+ * data which has been read from the connection but has not yet been acted
+ * upon (by passing it to the request handler or by extracting request
+ * metadata from it).
+ */
+ this._data = new LineData();
+
+ /**
+ * The amount of data remaining to be read from the body of this request.
+ * After all headers in the request have been read this is the value in the
+ * Content-Length header, but as the body is read its value decreases to zero.
+ */
+ this._contentLength = 0;
+
+ /** The current state of parsing the incoming request. */
+ this._state = READER_IN_REQUEST_LINE;
+
+ /** Metadata constructed from the incoming request for the request handler. */
+ this._metadata = new Request(connection.port);
+
+ /**
+ * Used to preserve state if we run out of line data midway through a
+ * multi-line header. _lastHeaderName stores the name of the header, while
+ * _lastHeaderValue stores the value we've seen so far for the header.
+ *
+ * These fields are always either both undefined or both strings.
+ */
+ this._lastHeaderName = this._lastHeaderValue = undefined;
+}
+RequestReader.prototype =
+{
+ // NSIINPUTSTREAMCALLBACK
+
+ /**
+ * Called when more data from the incoming request is available. This method
+ * then reads the available data from input and deals with that data as
+ * necessary, depending upon the syntax of already-downloaded data.
+ *
+ * @param input : nsIAsyncInputStream
+ * the stream of incoming data from the connection
+ */
+ onInputStreamReady: function(input)
+ {
+ dumpn("*** onInputStreamReady(input=" + input + ") on thread " +
+ gThreadManager.currentThread + " (main is " +
+ gThreadManager.mainThread + ")");
+ dumpn("*** this._state == " + this._state);
+
+ // Handle cases where we get more data after a request error has been
+ // discovered but *before* we can close the connection.
+ var data = this._data;
+ if (!data)
+ return;
+
+ try
+ {
+ data.appendBytes(readBytes(input, input.available()));
+ }
+ catch (e)
+ {
+ if (streamClosed(e))
+ {
+ dumpn("*** WARNING: unexpected error when reading from socket; will " +
+ "be treated as if the input stream had been closed");
+ dumpn("*** WARNING: actual error was: " + e);
+ }
+
+ // We've lost a race -- input has been closed, but we're still expecting
+ // to read more data. available() will throw in this case, and since
+ // we're dead in the water now, destroy the connection.
+ dumpn("*** onInputStreamReady called on a closed input, destroying " +
+ "connection");
+ this._connection.close();
+ return;
+ }
+
+ switch (this._state)
+ {
+ default:
+ NS_ASSERT(false, "invalid state: " + this._state);
+ break;
+
+ case READER_IN_REQUEST_LINE:
+ if (!this._processRequestLine())
+ break;
+ /* fall through */
+
+ case READER_IN_HEADERS:
+ if (!this._processHeaders())
+ break;
+ /* fall through */
+
+ case READER_IN_BODY:
+ this._processBody();
+ }
+
+ if (this._state != READER_FINISHED)
+ input.asyncWait(this, 0, 0, gThreadManager.currentThread);
+ },
+
+ //
+ // see nsISupports.QueryInterface
+ //
+ QueryInterface: function(aIID)
+ {
+ if (aIID.equals(Ci.nsIInputStreamCallback) ||
+ aIID.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ },
+
+
+ // PRIVATE API
+
+ /**
+ * Processes unprocessed, downloaded data as a request line.
+ *
+ * @returns boolean
+ * true iff the request line has been fully processed
+ */
+ _processRequestLine: function()
+ {
+ NS_ASSERT(this._state == READER_IN_REQUEST_LINE);
+
+ // Servers SHOULD ignore any empty line(s) received where a Request-Line
+ // is expected (section 4.1).
+ var data = this._data;
+ var line = {};
+ var readSuccess;
+ while ((readSuccess = data.readLine(line)) && line.value == "")
+ dumpn("*** ignoring beginning blank line...");
+
+ // if we don't have a full line, wait until we do
+ if (!readSuccess)
+ return false;
+
+ // we have the first non-blank line
+ try
+ {
+ this._parseRequestLine(line.value);
+ this._state = READER_IN_HEADERS;
+ this._connection.requestStarted();
+ return true;
+ }
+ catch (e)
+ {
+ this._handleError(e);
+ return false;
+ }
+ },
+
+ /**
+ * Processes stored data, assuming it is either at the beginning or in
+ * the middle of processing request headers.
+ *
+ * @returns boolean
+ * true iff header data in the request has been fully processed
+ */
+ _processHeaders: function()
+ {
+ NS_ASSERT(this._state == READER_IN_HEADERS);
+
+ // XXX things to fix here:
+ //
+ // - need to support RFC 2047-encoded non-US-ASCII characters
+
+ try
+ {
+ var done = this._parseHeaders();
+ if (done)
+ {
+ var request = this._metadata;
+
+ // XXX this is wrong for requests with transfer-encodings applied to
+ // them, particularly chunked (which by its nature can have no
+ // meaningful Content-Length header)!
+ this._contentLength = request.hasHeader("Content-Length")
+ ? parseInt(request.getHeader("Content-Length"), 10)
+ : 0;
+ dumpn("_processHeaders, Content-length=" + this._contentLength);
+
+ this._state = READER_IN_BODY;
+ }
+ return done;
+ }
+ catch (e)
+ {
+ this._handleError(e);
+ return false;
+ }
+ },
+
+ /**
+ * Processes stored data, assuming it is either at the beginning or in
+ * the middle of processing the request body.
+ *
+ * @returns boolean
+ * true iff the request body has been fully processed
+ */
+ _processBody: function()
+ {
+ NS_ASSERT(this._state == READER_IN_BODY);
+
+ // XXX handle chunked transfer-coding request bodies!
+
+ try
+ {
+ if (this._contentLength > 0)
+ {
+ var data = this._data.purge();
+ var count = Math.min(data.length, this._contentLength);
+ dumpn("*** loading data=" + data + " len=" + data.length +
+ " excess=" + (data.length - count));
+
+ var bos = new BinaryOutputStream(this._metadata._bodyOutputStream);
+ bos.writeByteArray(data, count);
+ this._contentLength -= count;
+ }
+
+ dumpn("*** remaining body data len=" + this._contentLength);
+ if (this._contentLength == 0)
+ {
+ this._validateRequest();
+ this._state = READER_FINISHED;
+ this._handleResponse();
+ return true;
+ }
+
+ return false;
+ }
+ catch (e)
+ {
+ this._handleError(e);
+ return false;
+ }
+ },
+
+ /**
+ * Does various post-header checks on the data in this request.
+ *
+ * @throws : HttpError
+ * if the request was malformed in some way
+ */
+ _validateRequest: function()
+ {
+ NS_ASSERT(this._state == READER_IN_BODY);
+
+ dumpn("*** _validateRequest");
+
+ var metadata = this._metadata;
+ var headers = metadata._headers;
+
+ // 19.6.1.1 -- servers MUST report 400 to HTTP/1.1 requests w/o Host header
+ var identity = this._connection.server.identity;
+ if (metadata._httpVersion.atLeast(nsHttpVersion.HTTP_1_1))
+ {
+ if (!headers.hasHeader("Host"))
+ {
+ dumpn("*** malformed HTTP/1.1 or greater request with no Host header!");
+ throw HTTP_400;
+ }
+
+ // If the Request-URI wasn't absolute, then we need to determine our host.
+ // We have to determine what scheme was used to access us based on the
+ // server identity data at this point, because the request just doesn't
+ // contain enough data on its own to do this, sadly.
+ if (!metadata._host)
+ {
+ var host, port;
+ var hostPort = headers.getHeader("Host");
+ var colon = hostPort.indexOf(":");
+ if (colon < 0)
+ {
+ host = hostPort;
+ port = "";
+ }
+ else
+ {
+ host = hostPort.substring(0, colon);
+ port = hostPort.substring(colon + 1);
+ }
+
+ // NB: We allow an empty port here because, oddly, a colon may be
+ // present even without a port number, e.g. "example.com:"; in this
+ // case the default port applies.
+ if (!HOST_REGEX.test(host) || !/^\d*$/.test(port))
+ {
+ dumpn("*** malformed hostname (" + hostPort + ") in Host " +
+ "header, 400 time");
+ throw HTTP_400;
+ }
+
+ // If we're not given a port, we're stuck, because we don't know what
+ // scheme to use to look up the correct port here, in general. Since
+ // the HTTPS case requires a tunnel/proxy and thus requires that the
+ // requested URI be absolute (and thus contain the necessary
+ // information), let's assume HTTP will prevail and use that.
+ port = +port || 80;
+
+ var scheme = identity.getScheme(host, port);
+ if (!scheme)
+ {
+ dumpn("*** unrecognized hostname (" + hostPort + ") in Host " +
+ "header, 400 time");
+ throw HTTP_400;
+ }
+
+ metadata._scheme = scheme;
+ metadata._host = host;
+ metadata._port = port;
+ }
+ }
+ else
+ {
+ NS_ASSERT(metadata._host === undefined,
+ "HTTP/1.0 doesn't allow absolute paths in the request line!");
+
+ metadata._scheme = identity.primaryScheme;
+ metadata._host = identity.primaryHost;
+ metadata._port = identity.primaryPort;
+ }
+
+ NS_ASSERT(identity.has(metadata._scheme, metadata._host, metadata._port),
+ "must have a location we recognize by now!");
+ },
+
+ /**
+ * Handles responses in case of error, either in the server or in the request.
+ *
+ * @param e
+ * the specific error encountered, which is an HttpError in the case where
+ * the request is in some way invalid or cannot be fulfilled; if this isn't
+ * an HttpError we're going to be paranoid and shut down, because that
+ * shouldn't happen, ever
+ */
+ _handleError: function(e)
+ {
+ // Don't fall back into normal processing!
+ this._state = READER_FINISHED;
+
+ var server = this._connection.server;
+ if (e instanceof HttpError)
+ {
+ var code = e.code;
+ }
+ else
+ {
+ dumpn("!!! UNEXPECTED ERROR: " + e +
+ (e.lineNumber ? ", line " + e.lineNumber : ""));
+
+ // no idea what happened -- be paranoid and shut down
+ code = 500;
+ server._requestQuit();
+ }
+
+ // make attempted reuse of data an error
+ this._data = null;
+
+ this._connection.processError(code, this._metadata);
+ },
+
+ /**
+ * Now that we've read the request line and headers, we can actually hand off
+ * the request to be handled.
+ *
+ * This method is called once per request, after the request line and all
+ * headers and the body, if any, have been received.
+ */
+ _handleResponse: function()
+ {
+ NS_ASSERT(this._state == READER_FINISHED);
+
+ // We don't need the line-based data any more, so make attempted reuse an
+ // error.
+ this._data = null;
+
+ this._connection.process(this._metadata);
+ },
+
+
+ // PARSING
+
+ /**
+ * Parses the request line for the HTTP request associated with this.
+ *
+ * @param line : string
+ * the request line
+ */
+ _parseRequestLine: function(line)
+ {
+ NS_ASSERT(this._state == READER_IN_REQUEST_LINE);
+
+ dumpn("*** _parseRequestLine('" + line + "')");
+
+ var metadata = this._metadata;
+
+ // clients and servers SHOULD accept any amount of SP or HT characters
+ // between fields, even though only a single SP is required (section 19.3)
+ var request = line.split(/[ \t]+/);
+ if (!request || request.length != 3)
+ {
+ dumpn("*** No request in line");
+ throw HTTP_400;
+ }
+
+ metadata._method = request[0];
+
+ // get the HTTP version
+ var ver = request[2];
+ var match = ver.match(/^HTTP\/(\d+\.\d+)$/);
+ if (!match)
+ {
+ dumpn("*** No HTTP version in line");
+ throw HTTP_400;
+ }
+
+ // determine HTTP version
+ try
+ {
+ metadata._httpVersion = new nsHttpVersion(match[1]);
+ if (!metadata._httpVersion.atLeast(nsHttpVersion.HTTP_1_0))
+ throw "unsupported HTTP version";
+ }
+ catch (e)
+ {
+ // we support HTTP/1.0 and HTTP/1.1 only
+ throw HTTP_501;
+ }
+
+
+ var fullPath = request[1];
+ var serverIdentity = this._connection.server.identity;
+
+ var scheme, host, port;
+
+ if (fullPath.charAt(0) != "/")
+ {
+ // No absolute paths in the request line in HTTP prior to 1.1
+ if (!metadata._httpVersion.atLeast(nsHttpVersion.HTTP_1_1))
+ {
+ dumpn("*** Metadata version too low");
+ throw HTTP_400;
+ }
+
+ try
+ {
+ var uri = Cc["@mozilla.org/network/io-service;1"]
+ .getService(Ci.nsIIOService)
+ .newURI(fullPath, null, null);
+ fullPath = uri.path;
+ scheme = uri.scheme;
+ host = metadata._host = uri.asciiHost;
+ port = uri.port;
+ if (port === -1)
+ {
+ if (scheme === "http")
+ {
+ port = 80;
+ }
+ else if (scheme === "https")
+ {
+ port = 443;
+ }
+ else
+ {
+ dumpn("*** Unknown scheme: " + scheme);
+ throw HTTP_400;
+ }
+ }
+ }
+ catch (e)
+ {
+ // If the host is not a valid host on the server, the response MUST be a
+ // 400 (Bad Request) error message (section 5.2). Alternately, the URI
+ // is malformed.
+ dumpn("*** Threw when dealing with URI: " + e);
+ throw HTTP_400;
+ }
+
+ if (!serverIdentity.has(scheme, host, port) || fullPath.charAt(0) != "/")
+ {
+ dumpn("*** serverIdentity unknown or path does not start with '/'");
+ throw HTTP_400;
+ }
+ }
+
+ var splitter = fullPath.indexOf("?");
+ if (splitter < 0)
+ {
+ // _queryString already set in ctor
+ metadata._path = fullPath;
+ }
+ else
+ {
+ metadata._path = fullPath.substring(0, splitter);
+ metadata._queryString = fullPath.substring(splitter + 1);
+ }
+
+ metadata._scheme = scheme;
+ metadata._host = host;
+ metadata._port = port;
+ },
+
+ /**
+ * Parses all available HTTP headers in this until the header-ending CRLFCRLF,
+ * adding them to the store of headers in the request.
+ *
+ * @throws
+ * HTTP_400 if the headers are malformed
+ * @returns boolean
+ * true if all headers have now been processed, false otherwise
+ */
+ _parseHeaders: function()
+ {
+ NS_ASSERT(this._state == READER_IN_HEADERS);
+
+ dumpn("*** _parseHeaders");
+
+ var data = this._data;
+
+ var headers = this._metadata._headers;
+ var lastName = this._lastHeaderName;
+ var lastVal = this._lastHeaderValue;
+
+ var line = {};
+ while (true)
+ {
+ dumpn("*** Last name: '" + lastName + "'");
+ dumpn("*** Last val: '" + lastVal + "'");
+ NS_ASSERT(!((lastVal === undefined) ^ (lastName === undefined)),
+ lastName === undefined ?
+ "lastVal without lastName? lastVal: '" + lastVal + "'" :
+ "lastName without lastVal? lastName: '" + lastName + "'");
+
+ if (!data.readLine(line))
+ {
+ // save any data we have from the header we might still be processing
+ this._lastHeaderName = lastName;
+ this._lastHeaderValue = lastVal;
+ return false;
+ }
+
+ var lineText = line.value;
+ dumpn("*** Line text: '" + lineText + "'");
+ var firstChar = lineText.charAt(0);
+
+ // blank line means end of headers
+ if (lineText == "")
+ {
+ // we're finished with the previous header
+ if (lastName)
+ {
+ try
+ {
+ headers.setHeader(lastName, lastVal, true);
+ }
+ catch (e)
+ {
+ dumpn("*** setHeader threw on last header, e == " + e);
+ throw HTTP_400;
+ }
+ }
+ else
+ {
+ // no headers in request -- valid for HTTP/1.0 requests
+ }
+
+ // either way, we're done processing headers
+ this._state = READER_IN_BODY;
+ return true;
+ }
+ else if (firstChar == " " || firstChar == "\t")
+ {
+ // multi-line header if we've already seen a header line
+ if (!lastName)
+ {
+ dumpn("We don't have a header to continue!");
+ throw HTTP_400;
+ }
+
+ // append this line's text to the value; starts with SP/HT, so no need
+ // for separating whitespace
+ lastVal += lineText;
+ }
+ else
+ {
+ // we have a new header, so set the old one (if one existed)
+ if (lastName)
+ {
+ try
+ {
+ headers.setHeader(lastName, lastVal, true);
+ }
+ catch (e)
+ {
+ dumpn("*** setHeader threw on a header, e == " + e);
+ throw HTTP_400;
+ }
+ }
+
+ var colon = lineText.indexOf(":"); // first colon must be splitter
+ if (colon < 1)
+ {
+ dumpn("*** No colon or missing header field-name");
+ throw HTTP_400;
+ }
+
+ // set header name, value (to be set in the next loop, usually)
+ lastName = lineText.substring(0, colon);
+ lastVal = lineText.substring(colon + 1);
+ } // empty, continuation, start of header
+ } // while (true)
+ }
+};
+
+
+/** The character codes for CR and LF. */
+const CR = 0x0D, LF = 0x0A;
+
+/**
+ * Calculates the number of characters before the first CRLF pair in array, or
+ * -1 if the array contains no CRLF pair.
+ *
+ * @param array : Array
+ * an array of numbers in the range [0, 256), each representing a single
+ * character; the first CRLF is the lowest index i where
+ * |array[i] == "\r".charCodeAt(0)| and |array[i+1] == "\n".charCodeAt(0)|,
+ * if such an |i| exists, and -1 otherwise
+ * @param start : uint
+ * start index from which to begin searching in array
+ * @returns int
+ * the index of the first CRLF if any were present, -1 otherwise
+ */
+function findCRLF(array, start)
+{
+ for (var i = array.indexOf(CR, start); i >= 0; i = array.indexOf(CR, i + 1))
+ {
+ if (array[i + 1] == LF)
+ return i;
+ }
+ return -1;
+}
+
+
+/**
+ * A container which provides line-by-line access to the arrays of bytes with
+ * which it is seeded.
+ */
+function LineData()
+{
+ /** An array of queued bytes from which to get line-based characters. */
+ this._data = [];
+
+ /** Start index from which to search for CRLF. */
+ this._start = 0;
+}
+LineData.prototype =
+{
+ /**
+ * Appends the bytes in the given array to the internal data cache maintained
+ * by this.
+ */
+ appendBytes: function(bytes)
+ {
+ var count = bytes.length;
+ var quantum = 262144; // just above half SpiderMonkey's argument-count limit
+ if (count < quantum)
+ {
+ Array.prototype.push.apply(this._data, bytes);
+ return;
+ }
+
+ // Large numbers of bytes may cause Array.prototype.push to be called with
+ // more arguments than the JavaScript engine supports. In that case append
+ // bytes in fixed-size amounts until all bytes are appended.
+ for (var start = 0; start < count; start += quantum)
+ {
+ var slice = bytes.slice(start, Math.min(start + quantum, count));
+ Array.prototype.push.apply(this._data, slice);
+ }
+ },
+
+ /**
+ * Removes and returns a line of data, delimited by CRLF, from this.
+ *
+ * @param out
+ * an object whose "value" property will be set to the first line of text
+ * present in this, sans CRLF, if this contains a full CRLF-delimited line
+ * of text; if this doesn't contain enough data, the value of the property
+ * is undefined
+ * @returns boolean
+ * true if a full line of data could be read from the data in this, false
+ * otherwise
+ */
+ readLine: function(out)
+ {
+ var data = this._data;
+ var length = findCRLF(data, this._start);
+ if (length < 0)
+ {
+ this._start = data.length;
+
+ // But if our data ends in a CR, we have to back up one, because
+ // the first byte in the next packet might be an LF and if we
+ // start looking at data.length we won't find it.
+ if (data.length > 0 && data[data.length - 1] === CR)
+ --this._start;
+
+ return false;
+ }
+
+ // Reset for future lines.
+ this._start = 0;
+
+ //
+ // We have the index of the CR, so remove all the characters, including
+ // CRLF, from the array with splice, and convert the removed array
+ // (excluding the trailing CRLF characters) into the corresponding string.
+ //
+ var leading = data.splice(0, length + 2);
+ var quantum = 262144;
+ var line = "";
+ for (var start = 0; start < length; start += quantum)
+ {
+ var slice = leading.slice(start, Math.min(start + quantum, length));
+ line += String.fromCharCode.apply(null, slice);
+ }
+
+ out.value = line;
+ return true;
+ },
+
+ /**
+ * Removes the bytes currently within this and returns them in an array.
+ *
+ * @returns Array
+ * the bytes within this when this method is called
+ */
+ purge: function()
+ {
+ var data = this._data;
+ this._data = [];
+ return data;
+ }
+};
+
+
+
+/**
+ * Creates a request-handling function for an nsIHttpRequestHandler object.
+ */
+function createHandlerFunc(handler)
+{
+ return function(metadata, response) { handler.handle(metadata, response); };
+}
+
+
+/**
+ * The default handler for directories; writes an HTML response containing a
+ * slightly-formatted directory listing.
+ */
+function defaultIndexHandler(metadata, response)
+{
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var path = htmlEscape(decodeURI(metadata.path));
+
+ //
+ // Just do a very basic bit of directory listings -- no need for too much
+ // fanciness, especially since we don't have a style sheet in which we can
+ // stick rules (don't want to pollute the default path-space).
+ //
+
+ var body = '<html>\
+ <head>\
+ <title>' + path + '</title>\
+ </head>\
+ <body>\
+ <h1>' + path + '</h1>\
+ <ol style="list-style-type: none">';
+
+ var directory = metadata.getProperty("directory");
+ NS_ASSERT(directory && directory.isDirectory());
+
+ var fileList = [];
+ var files = directory.directoryEntries;
+ while (files.hasMoreElements())
+ {
+ var f = files.getNext().QueryInterface(Ci.nsIFile);
+ var name = f.leafName;
+ if (!f.isHidden() &&
+ (name.charAt(name.length - 1) != HIDDEN_CHAR ||
+ name.charAt(name.length - 2) == HIDDEN_CHAR))
+ fileList.push(f);
+ }
+
+ fileList.sort(fileSort);
+
+ for (var i = 0; i < fileList.length; i++)
+ {
+ var file = fileList[i];
+ try
+ {
+ var name = file.leafName;
+ if (name.charAt(name.length - 1) == HIDDEN_CHAR)
+ name = name.substring(0, name.length - 1);
+ var sep = file.isDirectory() ? "/" : "";
+
+ // Note: using " to delimit the attribute here because encodeURIComponent
+ // passes through '.
+ var item = '<li><a href="' + encodeURIComponent(name) + sep + '">' +
+ htmlEscape(name) + sep +
+ '</a></li>';
+
+ body += item;
+ }
+ catch (e) { /* some file system error, ignore the file */ }
+ }
+
+ body += ' </ol>\
+ </body>\
+ </html>';
+
+ response.bodyOutputStream.write(body, body.length);
+}
+
+/**
+ * Sorts a and b (nsIFile objects) into an aesthetically pleasing order.
+ */
+function fileSort(a, b)
+{
+ var dira = a.isDirectory(), dirb = b.isDirectory();
+
+ if (dira && !dirb)
+ return -1;
+ if (dirb && !dira)
+ return 1;
+
+ var namea = a.leafName.toLowerCase(), nameb = b.leafName.toLowerCase();
+ return nameb > namea ? -1 : 1;
+}
+
+
+/**
+ * Converts an externally-provided path into an internal path for use in
+ * determining file mappings.
+ *
+ * @param path
+ * the path to convert
+ * @param encoded
+ * true if the given path should be passed through decodeURI prior to
+ * conversion
+ * @throws URIError
+ * if path is incorrectly encoded
+ */
+function toInternalPath(path, encoded)
+{
+ if (encoded)
+ path = decodeURI(path);
+
+ var comps = path.split("/");
+ for (var i = 0, sz = comps.length; i < sz; i++)
+ {
+ var comp = comps[i];
+ if (comp.charAt(comp.length - 1) == HIDDEN_CHAR)
+ comps[i] = comp + HIDDEN_CHAR;
+ }
+ return comps.join("/");
+}
+
+const PERMS_READONLY = (4 << 6) | (4 << 3) | 4;
+
+/**
+ * Adds custom-specified headers for the given file to the given response, if
+ * any such headers are specified.
+ *
+ * @param file
+ * the file on the disk which is to be written
+ * @param metadata
+ * metadata about the incoming request
+ * @param response
+ * the Response to which any specified headers/data should be written
+ * @throws HTTP_500
+ * if an error occurred while processing custom-specified headers
+ */
+function maybeAddHeaders(file, metadata, response)
+{
+ var name = file.leafName;
+ if (name.charAt(name.length - 1) == HIDDEN_CHAR)
+ name = name.substring(0, name.length - 1);
+
+ var headerFile = file.parent;
+ headerFile.append(name + HEADERS_SUFFIX);
+
+ if (!headerFile.exists())
+ return;
+
+ const PR_RDONLY = 0x01;
+ var fis = new FileInputStream(headerFile, PR_RDONLY, PERMS_READONLY,
+ Ci.nsIFileInputStream.CLOSE_ON_EOF);
+
+ try
+ {
+ var lis = new ConverterInputStream(fis, "UTF-8", 1024, 0x0);
+ lis.QueryInterface(Ci.nsIUnicharLineInputStream);
+
+ var line = {value: ""};
+ var more = lis.readLine(line);
+
+ if (!more && line.value == "")
+ return;
+
+
+ // request line
+
+ var status = line.value;
+ if (status.indexOf("HTTP ") == 0)
+ {
+ status = status.substring(5);
+ var space = status.indexOf(" ");
+ var code, description;
+ if (space < 0)
+ {
+ code = status;
+ description = "";
+ }
+ else
+ {
+ code = status.substring(0, space);
+ description = status.substring(space + 1, status.length);
+ }
+
+ response.setStatusLine(metadata.httpVersion, parseInt(code, 10), description);
+
+ line.value = "";
+ more = lis.readLine(line);
+ }
+
+ // headers
+ while (more || line.value != "")
+ {
+ var header = line.value;
+ var colon = header.indexOf(":");
+
+ response.setHeader(header.substring(0, colon),
+ header.substring(colon + 1, header.length),
+ false); // allow overriding server-set headers
+
+ line.value = "";
+ more = lis.readLine(line);
+ }
+ }
+ catch (e)
+ {
+ dumpn("WARNING: error in headers for " + metadata.path + ": " + e);
+ throw HTTP_500;
+ }
+ finally
+ {
+ fis.close();
+ }
+}
+
+
+/**
+ * An object which handles requests for a server, executing default and
+ * overridden behaviors as instructed by the code which uses and manipulates it.
+ * Default behavior includes the paths / and /trace (diagnostics), with some
+ * support for HTTP error pages for various codes and fallback to HTTP 500 if
+ * those codes fail for any reason.
+ *
+ * @param server : nsHttpServer
+ * the server in which this handler is being used
+ */
+function ServerHandler(server)
+{
+ // FIELDS
+
+ /**
+ * The nsHttpServer instance associated with this handler.
+ */
+ this._server = server;
+
+ /**
+ * A FileMap object containing the set of path->nsILocalFile mappings for
+ * all directory mappings set in the server (e.g., "/" for /var/www/html/,
+ * "/foo/bar/" for /local/path/, and "/foo/bar/baz/" for /local/path2).
+ *
+ * Note carefully: the leading and trailing "/" in each path (not file) are
+ * removed before insertion to simplify the code which uses this. You have
+ * been warned!
+ */
+ this._pathDirectoryMap = new FileMap();
+
+ /**
+ * Custom request handlers for the server in which this resides. Path-handler
+ * pairs are stored as property-value pairs in this property.
+ *
+ * @see ServerHandler.prototype._defaultPaths
+ */
+ this._overridePaths = {};
+
+ /**
+ * Custom request handlers for the path prefixes on the server in which this
+ * resides. Path-handler pairs are stored as property-value pairs in this
+ * property.
+ *
+ * @see ServerHandler.prototype._defaultPaths
+ */
+ this._overridePrefixes = {};
+
+ /**
+ * Custom request handlers for the error handlers in the server in which this
+ * resides. Path-handler pairs are stored as property-value pairs in this
+ * property.
+ *
+ * @see ServerHandler.prototype._defaultErrors
+ */
+ this._overrideErrors = {};
+
+ /**
+ * Maps file extensions to their MIME types in the server, overriding any
+ * mapping that might or might not exist in the MIME service.
+ */
+ this._mimeMappings = {};
+
+ /**
+ * The default handler for requests for directories, used to serve directories
+ * when no index file is present.
+ */
+ this._indexHandler = defaultIndexHandler;
+
+ /** Per-path state storage for the server. */
+ this._state = {};
+
+ /** Entire-server state storage. */
+ this._sharedState = {};
+
+ /** Entire-server state storage for nsISupports values. */
+ this._objectState = {};
+}
+ServerHandler.prototype =
+{
+ // PUBLIC API
+
+ /**
+ * Handles a request to this server, responding to the request appropriately
+ * and initiating server shutdown if necessary.
+ *
+ * This method never throws an exception.
+ *
+ * @param connection : Connection
+ * the connection for this request
+ */
+ handleResponse: function(connection)
+ {
+ var request = connection.request;
+ var response = new Response(connection);
+
+ var path = request.path;
+ dumpn("*** path == " + path);
+
+ try
+ {
+ try
+ {
+ if (path in this._overridePaths)
+ {
+ // explicit paths first, then files based on existing directory mappings,
+ // then (if the file doesn't exist) built-in server default paths
+ dumpn("calling override for " + path);
+ this._overridePaths[path](request, response);
+ }
+ else
+ {
+ var longestPrefix = "";
+ for (let prefix in this._overridePrefixes) {
+ if (prefix.length > longestPrefix.length &&
+ path.substr(0, prefix.length) == prefix)
+ {
+ longestPrefix = prefix;
+ }
+ }
+ if (longestPrefix.length > 0)
+ {
+ dumpn("calling prefix override for " + longestPrefix);
+ this._overridePrefixes[longestPrefix](request, response);
+ }
+ else
+ {
+ this._handleDefault(request, response);
+ }
+ }
+ }
+ catch (e)
+ {
+ if (response.partiallySent())
+ {
+ response.abort(e);
+ return;
+ }
+
+ if (!(e instanceof HttpError))
+ {
+ dumpn("*** unexpected error: e == " + e);
+ throw HTTP_500;
+ }
+ if (e.code !== 404)
+ throw e;
+
+ dumpn("*** default: " + (path in this._defaultPaths));
+
+ response = new Response(connection);
+ if (path in this._defaultPaths)
+ this._defaultPaths[path](request, response);
+ else
+ throw HTTP_404;
+ }
+ }
+ catch (e)
+ {
+ if (response.partiallySent())
+ {
+ response.abort(e);
+ return;
+ }
+
+ var errorCode = "internal";
+
+ try
+ {
+ if (!(e instanceof HttpError))
+ throw e;
+
+ errorCode = e.code;
+ dumpn("*** errorCode == " + errorCode);
+
+ response = new Response(connection);
+ if (e.customErrorHandling)
+ e.customErrorHandling(response);
+ this._handleError(errorCode, request, response);
+ return;
+ }
+ catch (e2)
+ {
+ dumpn("*** error handling " + errorCode + " error: " +
+ "e2 == " + e2 + ", shutting down server");
+
+ connection.server._requestQuit();
+ response.abort(e2);
+ return;
+ }
+ }
+
+ response.complete();
+ },
+
+ //
+ // see nsIHttpServer.registerFile
+ //
+ registerFile: function(path, file)
+ {
+ if (!file)
+ {
+ dumpn("*** unregistering '" + path + "' mapping");
+ delete this._overridePaths[path];
+ return;
+ }
+
+ dumpn("*** registering '" + path + "' as mapping to " + file.path);
+ file = file.clone();
+
+ var self = this;
+ this._overridePaths[path] =
+ function(request, response)
+ {
+ if (!file.exists())
+ throw HTTP_404;
+
+ response.setStatusLine(request.httpVersion, 200, "OK");
+ self._writeFileResponse(request, file, response, 0, file.fileSize);
+ };
+ },
+
+ //
+ // see nsIHttpServer.registerPathHandler
+ //
+ registerPathHandler: function(path, handler)
+ {
+ // XXX true path validation!
+ if (path.charAt(0) != "/")
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ this._handlerToField(handler, this._overridePaths, path);
+ },
+
+ //
+ // see nsIHttpServer.registerPrefixHandler
+ //
+ registerPrefixHandler: function(path, handler)
+ {
+ // XXX true path validation!
+ if (path.charAt(0) != "/" || path.charAt(path.length - 1) != "/")
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ this._handlerToField(handler, this._overridePrefixes, path);
+ },
+
+ //
+ // see nsIHttpServer.registerDirectory
+ //
+ registerDirectory: function(path, directory)
+ {
+ // strip off leading and trailing '/' so that we can use lastIndexOf when
+ // determining exactly how a path maps onto a mapped directory --
+ // conditional is required here to deal with "/".substring(1, 0) being
+ // converted to "/".substring(0, 1) per the JS specification
+ var key = path.length == 1 ? "" : path.substring(1, path.length - 1);
+
+ // the path-to-directory mapping code requires that the first character not
+ // be "/", or it will go into an infinite loop
+ if (key.charAt(0) == "/")
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ key = toInternalPath(key, false);
+
+ if (directory)
+ {
+ dumpn("*** mapping '" + path + "' to the location " + directory.path);
+ this._pathDirectoryMap.put(key, directory);
+ }
+ else
+ {
+ dumpn("*** removing mapping for '" + path + "'");
+ this._pathDirectoryMap.put(key, null);
+ }
+ },
+
+ //
+ // see nsIHttpServer.registerErrorHandler
+ //
+ registerErrorHandler: function(err, handler)
+ {
+ if (!(err in HTTP_ERROR_CODES))
+ dumpn("*** WARNING: registering non-HTTP/1.1 error code " +
+ "(" + err + ") handler -- was this intentional?");
+
+ this._handlerToField(handler, this._overrideErrors, err);
+ },
+
+ //
+ // see nsIHttpServer.setIndexHandler
+ //
+ setIndexHandler: function(handler)
+ {
+ if (!handler)
+ handler = defaultIndexHandler;
+ else if (typeof(handler) != "function")
+ handler = createHandlerFunc(handler);
+
+ this._indexHandler = handler;
+ },
+
+ //
+ // see nsIHttpServer.registerContentType
+ //
+ registerContentType: function(ext, type)
+ {
+ if (!type)
+ delete this._mimeMappings[ext];
+ else
+ this._mimeMappings[ext] = headerUtils.normalizeFieldValue(type);
+ },
+
+ // PRIVATE API
+
+ /**
+ * Sets or remove (if handler is null) a handler in an object with a key.
+ *
+ * @param handler
+ * a handler, either function or an nsIHttpRequestHandler
+ * @param dict
+ * The object to attach the handler to.
+ * @param key
+ * The field name of the handler.
+ */
+ _handlerToField: function(handler, dict, key)
+ {
+ // for convenience, handler can be a function if this is run from xpcshell
+ if (typeof(handler) == "function")
+ dict[key] = handler;
+ else if (handler)
+ dict[key] = createHandlerFunc(handler);
+ else
+ delete dict[key];
+ },
+
+ /**
+ * Handles a request which maps to a file in the local filesystem (if a base
+ * path has already been set; otherwise the 404 error is thrown).
+ *
+ * @param metadata : Request
+ * metadata for the incoming request
+ * @param response : Response
+ * an uninitialized Response to the given request, to be initialized by a
+ * request handler
+ * @throws HTTP_###
+ * if an HTTP error occurred (usually HTTP_404); note that in this case the
+ * calling code must handle post-processing of the response
+ */
+ _handleDefault: function(metadata, response)
+ {
+ dumpn("*** _handleDefault()");
+
+ response.setStatusLine(metadata.httpVersion, 200, "OK");
+
+ var path = metadata.path;
+ NS_ASSERT(path.charAt(0) == "/", "invalid path: <" + path + ">");
+
+ // determine the actual on-disk file; this requires finding the deepest
+ // path-to-directory mapping in the requested URL
+ var file = this._getFileForPath(path);
+
+ // the "file" might be a directory, in which case we either serve the
+ // contained index.html or make the index handler write the response
+ if (file.exists() && file.isDirectory())
+ {
+ file.append("index.html"); // make configurable?
+ if (!file.exists() || file.isDirectory())
+ {
+ metadata._ensurePropertyBag();
+ metadata._bag.setPropertyAsInterface("directory", file.parent);
+ this._indexHandler(metadata, response);
+ return;
+ }
+ }
+
+ // alternately, the file might not exist
+ if (!file.exists())
+ throw HTTP_404;
+
+ var start, end;
+ if (metadata._httpVersion.atLeast(nsHttpVersion.HTTP_1_1) &&
+ metadata.hasHeader("Range") &&
+ this._getTypeFromFile(file) !== SJS_TYPE)
+ {
+ var rangeMatch = metadata.getHeader("Range").match(/^bytes=(\d+)?-(\d+)?$/);
+ if (!rangeMatch)
+ {
+ dumpn("*** Range header bogosity: '" + metadata.getHeader("Range") + "'");
+ throw HTTP_400;
+ }
+
+ if (rangeMatch[1] !== undefined)
+ start = parseInt(rangeMatch[1], 10);
+
+ if (rangeMatch[2] !== undefined)
+ end = parseInt(rangeMatch[2], 10);
+
+ if (start === undefined && end === undefined)
+ {
+ dumpn("*** More Range header bogosity: '" + metadata.getHeader("Range") + "'");
+ throw HTTP_400;
+ }
+
+ // No start given, so the end is really the count of bytes from the
+ // end of the file.
+ if (start === undefined)
+ {
+ start = Math.max(0, file.fileSize - end);
+ end = file.fileSize - 1;
+ }
+
+ // start and end are inclusive
+ if (end === undefined || end >= file.fileSize)
+ end = file.fileSize - 1;
+
+ if (start !== undefined && start >= file.fileSize) {
+ var HTTP_416 = new HttpError(416, "Requested Range Not Satisfiable");
+ HTTP_416.customErrorHandling = function(errorResponse)
+ {
+ maybeAddHeaders(file, metadata, errorResponse);
+ };
+ throw HTTP_416;
+ }
+
+ if (end < start)
+ {
+ response.setStatusLine(metadata.httpVersion, 200, "OK");
+ start = 0;
+ end = file.fileSize - 1;
+ }
+ else
+ {
+ response.setStatusLine(metadata.httpVersion, 206, "Partial Content");
+ var contentRange = "bytes " + start + "-" + end + "/" + file.fileSize;
+ response.setHeader("Content-Range", contentRange);
+ }
+ }
+ else
+ {
+ start = 0;
+ end = file.fileSize - 1;
+ }
+
+ // finally...
+ dumpn("*** handling '" + path + "' as mapping to " + file.path + " from " +
+ start + " to " + end + " inclusive");
+ this._writeFileResponse(metadata, file, response, start, end - start + 1);
+ },
+
+ /**
+ * Writes an HTTP response for the given file, including setting headers for
+ * file metadata.
+ *
+ * @param metadata : Request
+ * the Request for which a response is being generated
+ * @param file : nsILocalFile
+ * the file which is to be sent in the response
+ * @param response : Response
+ * the response to which the file should be written
+ * @param offset: uint
+ * the byte offset to skip to when writing
+ * @param count: uint
+ * the number of bytes to write
+ */
+ _writeFileResponse: function(metadata, file, response, offset, count)
+ {
+ const PR_RDONLY = 0x01;
+
+ var type = this._getTypeFromFile(file);
+ if (type === SJS_TYPE)
+ {
+ var fis = new FileInputStream(file, PR_RDONLY, PERMS_READONLY,
+ Ci.nsIFileInputStream.CLOSE_ON_EOF);
+
+ try
+ {
+ var sis = new ScriptableInputStream(fis);
+ var s = Cu.Sandbox(gGlobalObject);
+ s.importFunction(dump, "dump");
+ s.importFunction(atob, "atob");
+ s.importFunction(btoa, "btoa");
+
+ // Define a basic key-value state-preservation API across requests, with
+ // keys initially corresponding to the empty string.
+ var self = this;
+ var path = metadata.path;
+ s.importFunction(function getState(k)
+ {
+ return self._getState(path, k);
+ });
+ s.importFunction(function setState(k, v)
+ {
+ self._setState(path, k, v);
+ });
+ s.importFunction(function getSharedState(k)
+ {
+ return self._getSharedState(k);
+ });
+ s.importFunction(function setSharedState(k, v)
+ {
+ self._setSharedState(k, v);
+ });
+ s.importFunction(function getObjectState(k, callback)
+ {
+ callback(self._getObjectState(k));
+ });
+ s.importFunction(function setObjectState(k, v)
+ {
+ self._setObjectState(k, v);
+ });
+ s.importFunction(function registerPathHandler(p, h)
+ {
+ self.registerPathHandler(p, h);
+ });
+
+ // Make it possible for sjs files to access their location
+ this._setState(path, "__LOCATION__", file.path);
+
+ try
+ {
+ // Alas, the line number in errors dumped to console when calling the
+ // request handler is simply an offset from where we load the SJS file.
+ // Work around this in a reasonably non-fragile way by dynamically
+ // getting the line number where we evaluate the SJS file. Don't
+ // separate these two lines!
+ var line = new Error().lineNumber;
+ Cu.evalInSandbox(sis.read(file.fileSize), s, "latest");
+ }
+ catch (e)
+ {
+ dumpn("*** syntax error in SJS at " + file.path + ": " + e);
+ throw HTTP_500;
+ }
+
+ try
+ {
+ s.handleRequest(metadata, response);
+ }
+ catch (e)
+ {
+ dump("*** error running SJS at " + file.path + ": " +
+ e + " on line " +
+ (e instanceof Error
+ ? e.lineNumber + " in httpd.js"
+ : (e.lineNumber - line)) + "\n");
+ throw HTTP_500;
+ }
+ }
+ finally
+ {
+ fis.close();
+ }
+ }
+ else
+ {
+ try
+ {
+ response.setHeader("Last-Modified",
+ toDateString(file.lastModifiedTime),
+ false);
+ }
+ catch (e) { /* lastModifiedTime threw, ignore */ }
+
+ response.setHeader("Content-Type", type, false);
+ maybeAddHeaders(file, metadata, response);
+ response.setHeader("Content-Length", "" + count, false);
+
+ var fis = new FileInputStream(file, PR_RDONLY, PERMS_READONLY,
+ Ci.nsIFileInputStream.CLOSE_ON_EOF);
+
+ offset = offset || 0;
+ count = count || file.fileSize;
+ NS_ASSERT(offset === 0 || offset < file.fileSize, "bad offset");
+ NS_ASSERT(count >= 0, "bad count");
+ NS_ASSERT(offset + count <= file.fileSize, "bad total data size");
+
+ try
+ {
+ if (offset !== 0)
+ {
+ // Seek (or read, if seeking isn't supported) to the correct offset so
+ // the data sent to the client matches the requested range.
+ if (fis instanceof Ci.nsISeekableStream)
+ fis.seek(Ci.nsISeekableStream.NS_SEEK_SET, offset);
+ else
+ new ScriptableInputStream(fis).read(offset);
+ }
+ }
+ catch (e)
+ {
+ fis.close();
+ throw e;
+ }
+
+ let writeMore = function () {
+ gThreadManager.currentThread
+ .dispatch(writeData, Ci.nsIThread.DISPATCH_NORMAL);
+ }
+
+ var input = new BinaryInputStream(fis);
+ var output = new BinaryOutputStream(response.bodyOutputStream);
+ var writeData =
+ {
+ run: function()
+ {
+ var chunkSize = Math.min(65536, count);
+ count -= chunkSize;
+ NS_ASSERT(count >= 0, "underflow");
+
+ try
+ {
+ var data = input.readByteArray(chunkSize);
+ NS_ASSERT(data.length === chunkSize,
+ "incorrect data returned? got " + data.length +
+ ", expected " + chunkSize);
+ output.writeByteArray(data, data.length);
+ if (count === 0)
+ {
+ fis.close();
+ response.finish();
+ }
+ else
+ {
+ writeMore();
+ }
+ }
+ catch (e)
+ {
+ try
+ {
+ fis.close();
+ }
+ finally
+ {
+ response.finish();
+ }
+ throw e;
+ }
+ }
+ };
+
+ writeMore();
+
+ // Now that we know copying will start, flag the response as async.
+ response.processAsync();
+ }
+ },
+
+ /**
+ * Get the value corresponding to a given key for the given path for SJS state
+ * preservation across requests.
+ *
+ * @param path : string
+ * the path from which the given state is to be retrieved
+ * @param k : string
+ * the key whose corresponding value is to be returned
+ * @returns string
+ * the corresponding value, which is initially the empty string
+ */
+ _getState: function(path, k)
+ {
+ var state = this._state;
+ if (path in state && k in state[path])
+ return state[path][k];
+ return "";
+ },
+
+ /**
+ * Set the value corresponding to a given key for the given path for SJS state
+ * preservation across requests.
+ *
+ * @param path : string
+ * the path from which the given state is to be retrieved
+ * @param k : string
+ * the key whose corresponding value is to be set
+ * @param v : string
+ * the value to be set
+ */
+ _setState: function(path, k, v)
+ {
+ if (typeof v !== "string")
+ throw new Error("non-string value passed");
+ var state = this._state;
+ if (!(path in state))
+ state[path] = {};
+ state[path][k] = v;
+ },
+
+ /**
+ * Get the value corresponding to a given key for SJS state preservation
+ * across requests.
+ *
+ * @param k : string
+ * the key whose corresponding value is to be returned
+ * @returns string
+ * the corresponding value, which is initially the empty string
+ */
+ _getSharedState: function(k)
+ {
+ var state = this._sharedState;
+ if (k in state)
+ return state[k];
+ return "";
+ },
+
+ /**
+ * Set the value corresponding to a given key for SJS state preservation
+ * across requests.
+ *
+ * @param k : string
+ * the key whose corresponding value is to be set
+ * @param v : string
+ * the value to be set
+ */
+ _setSharedState: function(k, v)
+ {
+ if (typeof v !== "string")
+ throw new Error("non-string value passed");
+ this._sharedState[k] = v;
+ },
+
+ /**
+ * Returns the object associated with the given key in the server for SJS
+ * state preservation across requests.
+ *
+ * @param k : string
+ * the key whose corresponding object is to be returned
+ * @returns nsISupports
+ * the corresponding object, or null if none was present
+ */
+ _getObjectState: function(k)
+ {
+ if (typeof k !== "string")
+ throw new Error("non-string key passed");
+ return this._objectState[k] || null;
+ },
+
+ /**
+ * Sets the object associated with the given key in the server for SJS
+ * state preservation across requests.
+ *
+ * @param k : string
+ * the key whose corresponding object is to be set
+ * @param v : nsISupports
+ * the object to be associated with the given key; may be null
+ */
+ _setObjectState: function(k, v)
+ {
+ if (typeof k !== "string")
+ throw new Error("non-string key passed");
+ if (typeof v !== "object")
+ throw new Error("non-object value passed");
+ if (v && !("QueryInterface" in v))
+ {
+ throw new Error("must pass an nsISupports; use wrappedJSObject to ease " +
+ "pain when using the server from JS");
+ }
+
+ this._objectState[k] = v;
+ },
+
+ /**
+ * Gets a content-type for the given file, first by checking for any custom
+ * MIME-types registered with this handler for the file's extension, second by
+ * asking the global MIME service for a content-type, and finally by failing
+ * over to application/octet-stream.
+ *
+ * @param file : nsIFile
+ * the nsIFile for which to get a file type
+ * @returns string
+ * the best content-type which can be determined for the file
+ */
+ _getTypeFromFile: function(file)
+ {
+ try
+ {
+ var name = file.leafName;
+ var dot = name.lastIndexOf(".");
+ if (dot > 0)
+ {
+ var ext = name.slice(dot + 1);
+ if (ext in this._mimeMappings)
+ return this._mimeMappings[ext];
+ }
+ return Cc["@mozilla.org/uriloader/external-helper-app-service;1"]
+ .getService(Ci.nsIMIMEService)
+ .getTypeFromFile(file);
+ }
+ catch (e)
+ {
+ return "application/octet-stream";
+ }
+ },
+
+ /**
+ * Returns the nsILocalFile which corresponds to the path, as determined using
+ * all registered path->directory mappings and any paths which are explicitly
+ * overridden.
+ *
+ * @param path : string
+ * the server path for which a file should be retrieved, e.g. "/foo/bar"
+ * @throws HttpError
+ * when the correct action is the corresponding HTTP error (i.e., because no
+ * mapping was found for a directory in path, the referenced file doesn't
+ * exist, etc.)
+ * @returns nsILocalFile
+ * the file to be sent as the response to a request for the path
+ */
+ _getFileForPath: function(path)
+ {
+ // decode and add underscores as necessary
+ try
+ {
+ path = toInternalPath(path, true);
+ }
+ catch (e)
+ {
+ dumpn("*** toInternalPath threw " + e);
+ throw HTTP_400; // malformed path
+ }
+
+ // next, get the directory which contains this path
+ var pathMap = this._pathDirectoryMap;
+
+ // An example progression of tmp for a path "/foo/bar/baz/" might be:
+ // "foo/bar/baz/", "foo/bar/baz", "foo/bar", "foo", ""
+ var tmp = path.substring(1);
+ while (true)
+ {
+ // do we have a match for current head of the path?
+ var file = pathMap.get(tmp);
+ if (file)
+ {
+ // XXX hack; basically disable showing mapping for /foo/bar/ when the
+ // requested path was /foo/bar, because relative links on the page
+ // will all be incorrect -- we really need the ability to easily
+ // redirect here instead
+ if (tmp == path.substring(1) &&
+ tmp.length != 0 &&
+ tmp.charAt(tmp.length - 1) != "/")
+ file = null;
+ else
+ break;
+ }
+
+ // if we've finished trying all prefixes, exit
+ if (tmp == "")
+ break;
+
+ tmp = tmp.substring(0, tmp.lastIndexOf("/"));
+ }
+
+ // no mapping applies, so 404
+ if (!file)
+ throw HTTP_404;
+
+
+ // last, get the file for the path within the determined directory
+ var parentFolder = file.parent;
+ var dirIsRoot = (parentFolder == null);
+
+ // Strategy here is to append components individually, making sure we
+ // never move above the given directory; this allows paths such as
+ // "<file>/foo/../bar" but prevents paths such as "<file>/../base-sibling";
+ // this component-wise approach also means the code works even on platforms
+ // which don't use "/" as the directory separator, such as Windows
+ var leafPath = path.substring(tmp.length + 1);
+ var comps = leafPath.split("/");
+ for (var i = 0, sz = comps.length; i < sz; i++)
+ {
+ var comp = comps[i];
+
+ if (comp == "..")
+ file = file.parent;
+ else if (comp == "." || comp == "")
+ continue;
+ else
+ file.append(comp);
+
+ if (!dirIsRoot && file.equals(parentFolder))
+ throw HTTP_403;
+ }
+
+ return file;
+ },
+
+ /**
+ * Writes the error page for the given HTTP error code over the given
+ * connection.
+ *
+ * @param errorCode : uint
+ * the HTTP error code to be used
+ * @param connection : Connection
+ * the connection on which the error occurred
+ */
+ handleError: function(errorCode, connection)
+ {
+ var response = new Response(connection);
+
+ dumpn("*** error in request: " + errorCode);
+
+ this._handleError(errorCode, new Request(connection.port), response);
+ },
+
+ /**
+ * Handles a request which generates the given error code, using the
+ * user-defined error handler if one has been set, gracefully falling back to
+ * the x00 status code if the code has no handler, and failing to status code
+ * 500 if all else fails.
+ *
+ * @param errorCode : uint
+ * the HTTP error which is to be returned
+ * @param metadata : Request
+ * metadata for the request, which will often be incomplete since this is an
+ * error
+ * @param response : Response
+ * an uninitialized Response should be initialized when this method
+ * completes with information which represents the desired error code in the
+ * ideal case or a fallback code in abnormal circumstances (i.e., 500 is a
+ * fallback for 505, per HTTP specs)
+ */
+ _handleError: function(errorCode, metadata, response)
+ {
+ if (!metadata)
+ throw Cr.NS_ERROR_NULL_POINTER;
+
+ var errorX00 = errorCode - (errorCode % 100);
+
+ try
+ {
+ if (!(errorCode in HTTP_ERROR_CODES))
+ dumpn("*** WARNING: requested invalid error: " + errorCode);
+
+ // RFC 2616 says that we should try to handle an error by its class if we
+ // can't otherwise handle it -- if that fails, we revert to handling it as
+ // a 500 internal server error, and if that fails we throw and shut down
+ // the server
+
+ // actually handle the error
+ try
+ {
+ if (errorCode in this._overrideErrors)
+ this._overrideErrors[errorCode](metadata, response);
+ else
+ this._defaultErrors[errorCode](metadata, response);
+ }
+ catch (e)
+ {
+ if (response.partiallySent())
+ {
+ response.abort(e);
+ return;
+ }
+
+ // don't retry the handler that threw
+ if (errorX00 == errorCode)
+ throw HTTP_500;
+
+ dumpn("*** error in handling for error code " + errorCode + ", " +
+ "falling back to " + errorX00 + "...");
+ response = new Response(response._connection);
+ if (errorX00 in this._overrideErrors)
+ this._overrideErrors[errorX00](metadata, response);
+ else if (errorX00 in this._defaultErrors)
+ this._defaultErrors[errorX00](metadata, response);
+ else
+ throw HTTP_500;
+ }
+ }
+ catch (e)
+ {
+ if (response.partiallySent())
+ {
+ response.abort();
+ return;
+ }
+
+ // we've tried everything possible for a meaningful error -- now try 500
+ dumpn("*** error in handling for error code " + errorX00 + ", falling " +
+ "back to 500...");
+
+ try
+ {
+ response = new Response(response._connection);
+ if (500 in this._overrideErrors)
+ this._overrideErrors[500](metadata, response);
+ else
+ this._defaultErrors[500](metadata, response);
+ }
+ catch (e2)
+ {
+ dumpn("*** multiple errors in default error handlers!");
+ dumpn("*** e == " + e + ", e2 == " + e2);
+ response.abort(e2);
+ return;
+ }
+ }
+
+ response.complete();
+ },
+
+ // FIELDS
+
+ /**
+ * This object contains the default handlers for the various HTTP error codes.
+ */
+ _defaultErrors:
+ {
+ 400: function(metadata, response)
+ {
+ // none of the data in metadata is reliable, so hard-code everything here
+ response.setStatusLine("1.1", 400, "Bad Request");
+ response.setHeader("Content-Type", "text/plain;charset=utf-8", false);
+
+ var body = "Bad request\n";
+ response.bodyOutputStream.write(body, body.length);
+ },
+ 403: function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion, 403, "Forbidden");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head><title>403 Forbidden</title></head>\
+ <body>\
+ <h1>403 Forbidden</h1>\
+ </body>\
+ </html>";
+ response.bodyOutputStream.write(body, body.length);
+ },
+ 404: function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion, 404, "Not Found");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head><title>404 Not Found</title></head>\
+ <body>\
+ <h1>404 Not Found</h1>\
+ <p>\
+ <span style='font-family: monospace;'>" +
+ htmlEscape(metadata.path) +
+ "</span> was not found.\
+ </p>\
+ </body>\
+ </html>";
+ response.bodyOutputStream.write(body, body.length);
+ },
+ 416: function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion,
+ 416,
+ "Requested Range Not Satisfiable");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head>\
+ <title>416 Requested Range Not Satisfiable</title></head>\
+ <body>\
+ <h1>416 Requested Range Not Satisfiable</h1>\
+ <p>The byte range was not valid for the\
+ requested resource.\
+ </p>\
+ </body>\
+ </html>";
+ response.bodyOutputStream.write(body, body.length);
+ },
+ 500: function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion,
+ 500,
+ "Internal Server Error");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head><title>500 Internal Server Error</title></head>\
+ <body>\
+ <h1>500 Internal Server Error</h1>\
+ <p>Something's broken in this server and\
+ needs to be fixed.</p>\
+ </body>\
+ </html>";
+ response.bodyOutputStream.write(body, body.length);
+ },
+ 501: function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion, 501, "Not Implemented");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head><title>501 Not Implemented</title></head>\
+ <body>\
+ <h1>501 Not Implemented</h1>\
+ <p>This server is not (yet) Apache.</p>\
+ </body>\
+ </html>";
+ response.bodyOutputStream.write(body, body.length);
+ },
+ 505: function(metadata, response)
+ {
+ response.setStatusLine("1.1", 505, "HTTP Version Not Supported");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head><title>505 HTTP Version Not Supported</title></head>\
+ <body>\
+ <h1>505 HTTP Version Not Supported</h1>\
+ <p>This server only supports HTTP/1.0 and HTTP/1.1\
+ connections.</p>\
+ </body>\
+ </html>";
+ response.bodyOutputStream.write(body, body.length);
+ }
+ },
+
+ /**
+ * Contains handlers for the default set of URIs contained in this server.
+ */
+ _defaultPaths:
+ {
+ "/": function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion, 200, "OK");
+ response.setHeader("Content-Type", "text/html;charset=utf-8", false);
+
+ var body = "<html>\
+ <head><title>httpd.js</title></head>\
+ <body>\
+ <h1>httpd.js</h1>\
+ <p>If you're seeing this page, httpd.js is up and\
+ serving requests! Now set a base path and serve some\
+ files!</p>\
+ </body>\
+ </html>";
+
+ response.bodyOutputStream.write(body, body.length);
+ },
+
+ "/trace": function(metadata, response)
+ {
+ response.setStatusLine(metadata.httpVersion, 200, "OK");
+ response.setHeader("Content-Type", "text/plain;charset=utf-8", false);
+
+ var body = "Request-URI: " +
+ metadata.scheme + "://" + metadata.host + ":" + metadata.port +
+ metadata.path + "\n\n";
+ body += "Request (semantically equivalent, slightly reformatted):\n\n";
+ body += metadata.method + " " + metadata.path;
+
+ if (metadata.queryString)
+ body += "?" + metadata.queryString;
+
+ body += " HTTP/" + metadata.httpVersion + "\r\n";
+
+ var headEnum = metadata.headers;
+ while (headEnum.hasMoreElements())
+ {
+ var fieldName = headEnum.getNext()
+ .QueryInterface(Ci.nsISupportsString)
+ .data;
+ body += fieldName + ": " + metadata.getHeader(fieldName) + "\r\n";
+ }
+
+ response.bodyOutputStream.write(body, body.length);
+ }
+ }
+};
+
+
+/**
+ * Maps absolute paths to files on the local file system (as nsILocalFiles).
+ */
+function FileMap()
+{
+ /** Hash which will map paths to nsILocalFiles. */
+ this._map = {};
+}
+FileMap.prototype =
+{
+ // PUBLIC API
+
+ /**
+ * Maps key to a clone of the nsILocalFile value if value is non-null;
+ * otherwise, removes any extant mapping for key.
+ *
+ * @param key : string
+ * string to which a clone of value is mapped
+ * @param value : nsILocalFile
+ * the file to map to key, or null to remove a mapping
+ */
+ put: function(key, value)
+ {
+ if (value)
+ this._map[key] = value.clone();
+ else
+ delete this._map[key];
+ },
+
+ /**
+ * Returns a clone of the nsILocalFile mapped to key, or null if no such
+ * mapping exists.
+ *
+ * @param key : string
+ * key to which the returned file maps
+ * @returns nsILocalFile
+ * a clone of the mapped file, or null if no mapping exists
+ */
+ get: function(key)
+ {
+ var val = this._map[key];
+ return val ? val.clone() : null;
+ }
+};
+
+
+// Response CONSTANTS
+
+// token = *<any CHAR except CTLs or separators>
+// CHAR = <any US-ASCII character (0-127)>
+// CTL = <any US-ASCII control character (0-31) and DEL (127)>
+// separators = "(" | ")" | "<" | ">" | "@"
+// | "," | ";" | ":" | "\" | <">
+// | "/" | "[" | "]" | "?" | "="
+// | "{" | "}" | SP | HT
+const IS_TOKEN_ARRAY =
+ [0, 0, 0, 0, 0, 0, 0, 0, // 0
+ 0, 0, 0, 0, 0, 0, 0, 0, // 8
+ 0, 0, 0, 0, 0, 0, 0, 0, // 16
+ 0, 0, 0, 0, 0, 0, 0, 0, // 24
+
+ 0, 1, 0, 1, 1, 1, 1, 1, // 32
+ 0, 0, 1, 1, 0, 1, 1, 0, // 40
+ 1, 1, 1, 1, 1, 1, 1, 1, // 48
+ 1, 1, 0, 0, 0, 0, 0, 0, // 56
+
+ 0, 1, 1, 1, 1, 1, 1, 1, // 64
+ 1, 1, 1, 1, 1, 1, 1, 1, // 72
+ 1, 1, 1, 1, 1, 1, 1, 1, // 80
+ 1, 1, 1, 0, 0, 0, 1, 1, // 88
+
+ 1, 1, 1, 1, 1, 1, 1, 1, // 96
+ 1, 1, 1, 1, 1, 1, 1, 1, // 104
+ 1, 1, 1, 1, 1, 1, 1, 1, // 112
+ 1, 1, 1, 0, 1, 0, 1]; // 120
+
+
+/**
+ * Determines whether the given character code is a CTL.
+ *
+ * @param code : uint
+ * the character code
+ * @returns boolean
+ * true if code is a CTL, false otherwise
+ */
+function isCTL(code)
+{
+ return (code >= 0 && code <= 31) || (code == 127);
+}
+
+/**
+ * Represents a response to an HTTP request, encapsulating all details of that
+ * response. This includes all headers, the HTTP version, status code and
+ * explanation, and the entity itself.
+ *
+ * @param connection : Connection
+ * the connection over which this response is to be written
+ */
+function Response(connection)
+{
+ /** The connection over which this response will be written. */
+ this._connection = connection;
+
+ /**
+ * The HTTP version of this response; defaults to 1.1 if not set by the
+ * handler.
+ */
+ this._httpVersion = nsHttpVersion.HTTP_1_1;
+
+ /**
+ * The HTTP code of this response; defaults to 200.
+ */
+ this._httpCode = 200;
+
+ /**
+ * The description of the HTTP code in this response; defaults to "OK".
+ */
+ this._httpDescription = "OK";
+
+ /**
+ * An nsIHttpHeaders object in which the headers in this response should be
+ * stored. This property is null after the status line and headers have been
+ * written to the network, and it may be modified up until it is cleared,
+ * except if this._finished is set first (in which case headers are written
+ * asynchronously in response to a finish() call not preceded by
+ * flushHeaders()).
+ */
+ this._headers = new nsHttpHeaders();
+
+ /**
+ * Set to true when this response is ended (completely constructed if possible
+ * and the connection closed); further actions on this will then fail.
+ */
+ this._ended = false;
+
+ /**
+ * A stream used to hold data written to the body of this response.
+ */
+ this._bodyOutputStream = null;
+
+ /**
+ * A stream containing all data that has been written to the body of this
+ * response so far. (Async handlers make the data contained in this
+ * unreliable as a way of determining content length in general, but auxiliary
+ * saved information can sometimes be used to guarantee reliability.)
+ */
+ this._bodyInputStream = null;
+
+ /**
+ * A stream copier which copies data to the network. It is initially null
+ * until replaced with a copier for response headers; when headers have been
+ * fully sent it is replaced with a copier for the response body, remaining
+ * so for the duration of response processing.
+ */
+ this._asyncCopier = null;
+
+ /**
+ * True if this response has been designated as being processed
+ * asynchronously rather than for the duration of a single call to
+ * nsIHttpRequestHandler.handle.
+ */
+ this._processAsync = false;
+
+ /**
+ * True iff finish() has been called on this, signaling that no more changes
+ * to this may be made.
+ */
+ this._finished = false;
+
+ /**
+ * True iff powerSeized() has been called on this, signaling that this
+ * response is to be handled manually by the response handler (which may then
+ * send arbitrary data in response, even non-HTTP responses).
+ */
+ this._powerSeized = false;
+}
+Response.prototype =
+{
+ // PUBLIC CONSTRUCTION API
+
+ //
+ // see nsIHttpResponse.bodyOutputStream
+ //
+ get bodyOutputStream()
+ {
+ if (this._finished)
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+
+ if (!this._bodyOutputStream)
+ {
+ var pipe = new Pipe(true, false, Response.SEGMENT_SIZE, PR_UINT32_MAX,
+ null);
+ this._bodyOutputStream = pipe.outputStream;
+ this._bodyInputStream = pipe.inputStream;
+ if (this._processAsync || this._powerSeized)
+ this._startAsyncProcessor();
+ }
+
+ return this._bodyOutputStream;
+ },
+
+ //
+ // see nsIHttpResponse.write
+ //
+ write: function(data)
+ {
+ if (this._finished)
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+
+ var dataAsString = String(data);
+ this.bodyOutputStream.write(dataAsString, dataAsString.length);
+ },
+
+ //
+ // see nsIHttpResponse.setStatusLine
+ //
+ setStatusLine: function(httpVersion, code, description)
+ {
+ if (!this._headers || this._finished || this._powerSeized)
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+ this._ensureAlive();
+
+ if (!(code >= 0 && code < 1000))
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ try
+ {
+ var httpVer;
+ // avoid version construction for the most common cases
+ if (!httpVersion || httpVersion == "1.1")
+ httpVer = nsHttpVersion.HTTP_1_1;
+ else if (httpVersion == "1.0")
+ httpVer = nsHttpVersion.HTTP_1_0;
+ else
+ httpVer = new nsHttpVersion(httpVersion);
+ }
+ catch (e)
+ {
+ throw Cr.NS_ERROR_INVALID_ARG;
+ }
+
+ // Reason-Phrase = *<TEXT, excluding CR, LF>
+ // TEXT = <any OCTET except CTLs, but including LWS>
+ //
+ // XXX this ends up disallowing octets which aren't Unicode, I think -- not
+ // much to do if description is IDL'd as string
+ if (!description)
+ description = "";
+ for (var i = 0; i < description.length; i++)
+ if (isCTL(description.charCodeAt(i)) && description.charAt(i) != "\t")
+ throw Cr.NS_ERROR_INVALID_ARG;
+
+ // set the values only after validation to preserve atomicity
+ this._httpDescription = description;
+ this._httpCode = code;
+ this._httpVersion = httpVer;
+ },
+
+ //
+ // see nsIHttpResponse.setHeader
+ //
+ setHeader: function(name, value, merge)
+ {
+ if (!this._headers || this._finished || this._powerSeized)
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+ this._ensureAlive();
+
+ this._headers.setHeader(name, value, merge);
+ },
+
+ //
+ // see nsIHttpResponse.processAsync
+ //
+ processAsync: function()
+ {
+ if (this._finished)
+ throw Cr.NS_ERROR_UNEXPECTED;
+ if (this._powerSeized)
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+ if (this._processAsync)
+ return;
+ this._ensureAlive();
+
+ dumpn("*** processing connection " + this._connection.number + " async");
+ this._processAsync = true;
+
+ /*
+ * Either the bodyOutputStream getter or this method is responsible for
+ * starting the asynchronous processor and catching writes of data to the
+ * response body of async responses as they happen, for the purpose of
+ * forwarding those writes to the actual connection's output stream.
+ * If bodyOutputStream is accessed first, calling this method will create
+ * the processor (when it first is clear that body data is to be written
+ * immediately, not buffered). If this method is called first, accessing
+ * bodyOutputStream will create the processor. If only this method is
+ * called, we'll write nothing, neither headers nor the nonexistent body,
+ * until finish() is called. Since that delay is easily avoided by simply
+ * getting bodyOutputStream or calling write(""), we don't worry about it.
+ */
+ if (this._bodyOutputStream && !this._asyncCopier)
+ this._startAsyncProcessor();
+ },
+
+ //
+ // see nsIHttpResponse.seizePower
+ //
+ seizePower: function()
+ {
+ if (this._processAsync)
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+ if (this._finished)
+ throw Cr.NS_ERROR_UNEXPECTED;
+ if (this._powerSeized)
+ return;
+ this._ensureAlive();
+
+ dumpn("*** forcefully seizing power over connection " +
+ this._connection.number + "...");
+
+ // Purge any already-written data without sending it. We could as easily
+ // swap out the streams entirely, but that makes it possible to acquire and
+ // unknowingly use a stale reference, so we require there only be one of
+ // each stream ever for any response to avoid this complication.
+ if (this._asyncCopier)
+ this._asyncCopier.cancel(Cr.NS_BINDING_ABORTED);
+ this._asyncCopier = null;
+ if (this._bodyOutputStream)
+ {
+ var input = new BinaryInputStream(this._bodyInputStream);
+ var avail;
+ while ((avail = input.available()) > 0)
+ input.readByteArray(avail);
+ }
+
+ this._powerSeized = true;
+ if (this._bodyOutputStream)
+ this._startAsyncProcessor();
+ },
+
+ //
+ // see nsIHttpResponse.finish
+ //
+ finish: function()
+ {
+ if (!this._processAsync && !this._powerSeized)
+ throw Cr.NS_ERROR_UNEXPECTED;
+ if (this._finished)
+ return;
+
+ dumpn("*** finishing connection " + this._connection.number);
+ this._startAsyncProcessor(); // in case bodyOutputStream was never accessed
+ if (this._bodyOutputStream)
+ this._bodyOutputStream.close();
+ this._finished = true;
+ },
+
+
+ // NSISUPPORTS
+
+ //
+ // see nsISupports.QueryInterface
+ //
+ QueryInterface: function(iid)
+ {
+ if (iid.equals(Ci.nsIHttpResponse) || iid.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ },
+
+
+ // POST-CONSTRUCTION API (not exposed externally)
+
+ /**
+ * The HTTP version number of this, as a string (e.g. "1.1").
+ */
+ get httpVersion()
+ {
+ this._ensureAlive();
+ return this._httpVersion.toString();
+ },
+
+ /**
+ * The HTTP status code of this response, as a string of three characters per
+ * RFC 2616.
+ */
+ get httpCode()
+ {
+ this._ensureAlive();
+
+ var codeString = (this._httpCode < 10 ? "0" : "") +
+ (this._httpCode < 100 ? "0" : "") +
+ this._httpCode;
+ return codeString;
+ },
+
+ /**
+ * The description of the HTTP status code of this response, or "" if none is
+ * set.
+ */
+ get httpDescription()
+ {
+ this._ensureAlive();
+
+ return this._httpDescription;
+ },
+
+ /**
+ * The headers in this response, as an nsHttpHeaders object.
+ */
+ get headers()
+ {
+ this._ensureAlive();
+
+ return this._headers;
+ },
+
+ //
+ // see nsHttpHeaders.getHeader
+ //
+ getHeader: function(name)
+ {
+ this._ensureAlive();
+
+ return this._headers.getHeader(name);
+ },
+
+ /**
+ * Determines whether this response may be abandoned in favor of a newly
+ * constructed response. A response may be abandoned only if it is not being
+ * sent asynchronously and if raw control over it has not been taken from the
+ * server.
+ *
+ * @returns boolean
+ * true iff no data has been written to the network
+ */
+ partiallySent: function()
+ {
+ dumpn("*** partiallySent()");
+ return this._processAsync || this._powerSeized;
+ },
+
+ /**
+ * If necessary, kicks off the remaining request processing needed to be done
+ * after a request handler performs its initial work upon this response.
+ */
+ complete: function()
+ {
+ dumpn("*** complete()");
+ if (this._processAsync || this._powerSeized)
+ {
+ NS_ASSERT(this._processAsync ^ this._powerSeized,
+ "can't both send async and relinquish power");
+ return;
+ }
+
+ NS_ASSERT(!this.partiallySent(), "completing a partially-sent response?");
+
+ this._startAsyncProcessor();
+
+ // Now make sure we finish processing this request!
+ if (this._bodyOutputStream)
+ this._bodyOutputStream.close();
+ },
+
+ /**
+ * Abruptly ends processing of this response, usually due to an error in an
+ * incoming request but potentially due to a bad error handler. Since we
+ * cannot handle the error in the usual way (giving an HTTP error page in
+ * response) because data may already have been sent (or because the response
+ * might be expected to have been generated asynchronously or completely from
+ * scratch by the handler), we stop processing this response and abruptly
+ * close the connection.
+ *
+ * @param e : Error
+ * the exception which precipitated this abort, or null if no such exception
+ * was generated
+ */
+ abort: function(e)
+ {
+ dumpn("*** abort(<" + e + ">)");
+
+ // This response will be ended by the processor if one was created.
+ var copier = this._asyncCopier;
+ if (copier)
+ {
+ // We dispatch asynchronously here so that any pending writes of data to
+ // the connection will be deterministically written. This makes it easier
+ // to specify exact behavior, and it makes observable behavior more
+ // predictable for clients. Note that the correctness of this depends on
+ // callbacks in response to _waitToReadData in WriteThroughCopier
+ // happening asynchronously with respect to the actual writing of data to
+ // bodyOutputStream, as they currently do; if they happened synchronously,
+ // an event which ran before this one could write more data to the
+ // response body before we get around to canceling the copier. We have
+ // tests for this in test_seizepower.js, however, and I can't think of a
+ // way to handle both cases without removing bodyOutputStream access and
+ // moving its effective write(data, length) method onto Response, which
+ // would be slower and require more code than this anyway.
+ gThreadManager.currentThread.dispatch({
+ run: function()
+ {
+ dumpn("*** canceling copy asynchronously...");
+ copier.cancel(Cr.NS_ERROR_UNEXPECTED);
+ }
+ }, Ci.nsIThread.DISPATCH_NORMAL);
+ }
+ else
+ {
+ this.end();
+ }
+ },
+
+ /**
+ * Closes this response's network connection, marks the response as finished,
+ * and notifies the server handler that the request is done being processed.
+ */
+ end: function()
+ {
+ NS_ASSERT(!this._ended, "ending this response twice?!?!");
+
+ this._connection.close();
+ if (this._bodyOutputStream)
+ this._bodyOutputStream.close();
+
+ this._finished = true;
+ this._ended = true;
+ },
+
+ // PRIVATE IMPLEMENTATION
+
+ /**
+ * Sends the status line and headers of this response if they haven't been
+ * sent and initiates the process of copying data written to this response's
+ * body to the network.
+ */
+ _startAsyncProcessor: function()
+ {
+ dumpn("*** _startAsyncProcessor()");
+
+ // Handle cases where we're being called a second time. The former case
+ // happens when this is triggered both by complete() and by processAsync(),
+ // while the latter happens when processAsync() in conjunction with sent
+ // data causes abort() to be called.
+ if (this._asyncCopier || this._ended)
+ {
+ dumpn("*** ignoring second call to _startAsyncProcessor");
+ return;
+ }
+
+ // Send headers if they haven't been sent already and should be sent, then
+ // asynchronously continue to send the body.
+ if (this._headers && !this._powerSeized)
+ {
+ this._sendHeaders();
+ return;
+ }
+
+ this._headers = null;
+ this._sendBody();
+ },
+
+ /**
+ * Signals that all modifications to the response status line and headers are
+ * complete and then sends that data over the network to the client. Once
+ * this method completes, a different response to the request that resulted
+ * in this response cannot be sent -- the only possible action in case of
+ * error is to abort the response and close the connection.
+ */
+ _sendHeaders: function()
+ {
+ dumpn("*** _sendHeaders()");
+
+ NS_ASSERT(this._headers);
+ NS_ASSERT(!this._powerSeized);
+
+ // request-line
+ var statusLine = "HTTP/" + this.httpVersion + " " +
+ this.httpCode + " " +
+ this.httpDescription + "\r\n";
+
+ // header post-processing
+
+ var headers = this._headers;
+ headers.setHeader("Connection", "close", false);
+ headers.setHeader("Server", "httpd.js", false);
+ if (!headers.hasHeader("Date"))
+ headers.setHeader("Date", toDateString(Date.now()), false);
+
+ // Any response not being processed asynchronously must have an associated
+ // Content-Length header for reasons of backwards compatibility with the
+ // initial server, which fully buffered every response before sending it.
+ // Beyond that, however, it's good to do this anyway because otherwise it's
+ // impossible to test behaviors that depend on the presence or absence of a
+ // Content-Length header.
+ if (!this._processAsync)
+ {
+ dumpn("*** non-async response, set Content-Length");
+
+ var bodyStream = this._bodyInputStream;
+ var avail = bodyStream ? bodyStream.available() : 0;
+
+ // XXX assumes stream will always report the full amount of data available
+ headers.setHeader("Content-Length", "" + avail, false);
+ }
+
+
+ // construct and send response
+ dumpn("*** header post-processing completed, sending response head...");
+
+ // request-line
+ var preambleData = [statusLine];
+
+ // headers
+ var headEnum = headers.enumerator;
+ while (headEnum.hasMoreElements())
+ {
+ var fieldName = headEnum.getNext()
+ .QueryInterface(Ci.nsISupportsString)
+ .data;
+ var values = headers.getHeaderValues(fieldName);
+ for (var i = 0, sz = values.length; i < sz; i++)
+ preambleData.push(fieldName + ": " + values[i] + "\r\n");
+ }
+
+ // end request-line/headers
+ preambleData.push("\r\n");
+
+ var preamble = preambleData.join("");
+
+ var responseHeadPipe = new Pipe(true, false, 0, PR_UINT32_MAX, null);
+ responseHeadPipe.outputStream.write(preamble, preamble.length);
+
+ var response = this;
+ var copyObserver =
+ {
+ onStartRequest: function(request, cx)
+ {
+ dumpn("*** preamble copying started");
+ },
+
+ onStopRequest: function(request, cx, statusCode)
+ {
+ dumpn("*** preamble copying complete " +
+ "[status=0x" + statusCode.toString(16) + "]");
+
+ if (!Components.isSuccessCode(statusCode))
+ {
+ dumpn("!!! header copying problems: non-success statusCode, " +
+ "ending response");
+
+ response.end();
+ }
+ else
+ {
+ response._sendBody();
+ }
+ },
+
+ QueryInterface: function(aIID)
+ {
+ if (aIID.equals(Ci.nsIRequestObserver) || aIID.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ }
+ };
+
+ var headerCopier = this._asyncCopier =
+ new WriteThroughCopier(responseHeadPipe.inputStream,
+ this._connection.output,
+ copyObserver, null);
+
+ responseHeadPipe.outputStream.close();
+
+ // Forbid setting any more headers or modifying the request line.
+ this._headers = null;
+ },
+
+ /**
+ * Asynchronously writes the body of the response (or the entire response, if
+ * seizePower() has been called) to the network.
+ */
+ _sendBody: function()
+ {
+ dumpn("*** _sendBody");
+
+ NS_ASSERT(!this._headers, "still have headers around but sending body?");
+
+ // If no body data was written, we're done
+ if (!this._bodyInputStream)
+ {
+ dumpn("*** empty body, response finished");
+ this.end();
+ return;
+ }
+
+ var response = this;
+ var copyObserver =
+ {
+ onStartRequest: function(request, context)
+ {
+ dumpn("*** onStartRequest");
+ },
+
+ onStopRequest: function(request, cx, statusCode)
+ {
+ dumpn("*** onStopRequest [status=0x" + statusCode.toString(16) + "]");
+
+ if (statusCode === Cr.NS_BINDING_ABORTED)
+ {
+ dumpn("*** terminating copy observer without ending the response");
+ }
+ else
+ {
+ if (!Components.isSuccessCode(statusCode))
+ dumpn("*** WARNING: non-success statusCode in onStopRequest");
+
+ response.end();
+ }
+ },
+
+ QueryInterface: function(aIID)
+ {
+ if (aIID.equals(Ci.nsIRequestObserver) || aIID.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ }
+ };
+
+ dumpn("*** starting async copier of body data...");
+ this._asyncCopier =
+ new WriteThroughCopier(this._bodyInputStream, this._connection.output,
+ copyObserver, null);
+ },
+
+ /** Ensures that this hasn't been ended. */
+ _ensureAlive: function()
+ {
+ NS_ASSERT(!this._ended, "not handling response lifetime correctly");
+ }
+};
+
+/**
+ * Size of the segments in the buffer used in storing response data and writing
+ * it to the socket.
+ */
+Response.SEGMENT_SIZE = 8192;
+
+/** Serves double duty in WriteThroughCopier implementation. */
+function notImplemented()
+{
+ throw Cr.NS_ERROR_NOT_IMPLEMENTED;
+}
+
+/** Returns true iff the given exception represents stream closure. */
+function streamClosed(e)
+{
+ return e === Cr.NS_BASE_STREAM_CLOSED ||
+ (typeof e === "object" && e.result === Cr.NS_BASE_STREAM_CLOSED);
+}
+
+/** Returns true iff the given exception represents a blocked stream. */
+function wouldBlock(e)
+{
+ return e === Cr.NS_BASE_STREAM_WOULD_BLOCK ||
+ (typeof e === "object" && e.result === Cr.NS_BASE_STREAM_WOULD_BLOCK);
+}
+
+/**
+ * Copies data from source to sink as it becomes available, when that data can
+ * be written to sink without blocking.
+ *
+ * @param source : nsIAsyncInputStream
+ * the stream from which data is to be read
+ * @param sink : nsIAsyncOutputStream
+ * the stream to which data is to be copied
+ * @param observer : nsIRequestObserver
+ * an observer which will be notified when the copy starts and finishes
+ * @param context : nsISupports
+ * context passed to observer when notified of start/stop
+ * @throws NS_ERROR_NULL_POINTER
+ * if source, sink, or observer are null
+ */
+function WriteThroughCopier(source, sink, observer, context)
+{
+ if (!source || !sink || !observer)
+ throw Cr.NS_ERROR_NULL_POINTER;
+
+ /** Stream from which data is being read. */
+ this._source = source;
+
+ /** Stream to which data is being written. */
+ this._sink = sink;
+
+ /** Observer watching this copy. */
+ this._observer = observer;
+
+ /** Context for the observer watching this. */
+ this._context = context;
+
+ /**
+ * True iff this is currently being canceled (cancel has been called, the
+ * callback may not yet have been made).
+ */
+ this._canceled = false;
+
+ /**
+ * False until all data has been read from input and written to output, at
+ * which point this copy is completed and cancel() is asynchronously called.
+ */
+ this._completed = false;
+
+ /** Required by nsIRequest, meaningless. */
+ this.loadFlags = 0;
+ /** Required by nsIRequest, meaningless. */
+ this.loadGroup = null;
+ /** Required by nsIRequest, meaningless. */
+ this.name = "response-body-copy";
+
+ /** Status of this request. */
+ this.status = Cr.NS_OK;
+
+ /** Arrays of byte strings waiting to be written to output. */
+ this._pendingData = [];
+
+ // start copying
+ try
+ {
+ observer.onStartRequest(this, context);
+ this._waitToReadData();
+ this._waitForSinkClosure();
+ }
+ catch (e)
+ {
+ dumpn("!!! error starting copy: " + e +
+ ("lineNumber" in e ? ", line " + e.lineNumber : ""));
+ dumpn(e.stack);
+ this.cancel(Cr.NS_ERROR_UNEXPECTED);
+ }
+}
+WriteThroughCopier.prototype =
+{
+ /* nsISupports implementation */
+
+ QueryInterface: function(iid)
+ {
+ if (iid.equals(Ci.nsIInputStreamCallback) ||
+ iid.equals(Ci.nsIOutputStreamCallback) ||
+ iid.equals(Ci.nsIRequest) ||
+ iid.equals(Ci.nsISupports))
+ {
+ return this;
+ }
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ },
+
+
+ // NSIINPUTSTREAMCALLBACK
+
+ /**
+ * Receives a more-data-in-input notification and writes the corresponding
+ * data to the output.
+ *
+ * @param input : nsIAsyncInputStream
+ * the input stream on whose data we have been waiting
+ */
+ onInputStreamReady: function(input)
+ {
+ if (this._source === null)
+ return;
+
+ dumpn("*** onInputStreamReady");
+
+ //
+ // Ordinarily we'll read a non-zero amount of data from input, queue it up
+ // to be written and then wait for further callbacks. The complications in
+ // this method are the cases where we deviate from that behavior when errors
+ // occur or when copying is drawing to a finish.
+ //
+ // The edge cases when reading data are:
+ //
+ // Zero data is read
+ // If zero data was read, we're at the end of available data, so we can
+ // should stop reading and move on to writing out what we have (or, if
+ // we've already done that, onto notifying of completion).
+ // A stream-closed exception is thrown
+ // This is effectively a less kind version of zero data being read; the
+ // only difference is that we notify of completion with that result
+ // rather than with NS_OK.
+ // Some other exception is thrown
+ // This is the least kind result. We don't know what happened, so we
+ // act as though the stream closed except that we notify of completion
+ // with the result NS_ERROR_UNEXPECTED.
+ //
+
+ var bytesWanted = 0, bytesConsumed = -1;
+ try
+ {
+ input = new BinaryInputStream(input);
+
+ bytesWanted = Math.min(input.available(), Response.SEGMENT_SIZE);
+ dumpn("*** input wanted: " + bytesWanted);
+
+ if (bytesWanted > 0)
+ {
+ var data = input.readByteArray(bytesWanted);
+ bytesConsumed = data.length;
+ this._pendingData.push(String.fromCharCode.apply(String, data));
+ }
+
+ dumpn("*** " + bytesConsumed + " bytes read");
+
+ // Handle the zero-data edge case in the same place as all other edge
+ // cases are handled.
+ if (bytesWanted === 0)
+ throw Cr.NS_BASE_STREAM_CLOSED;
+ }
+ catch (e)
+ {
+ if (streamClosed(e))
+ {
+ dumpn("*** input stream closed");
+ e = bytesWanted === 0 ? Cr.NS_OK : Cr.NS_ERROR_UNEXPECTED;
+ }
+ else
+ {
+ dumpn("!!! unexpected error reading from input, canceling: " + e);
+ e = Cr.NS_ERROR_UNEXPECTED;
+ }
+
+ this._doneReadingSource(e);
+ return;
+ }
+
+ var pendingData = this._pendingData;
+
+ NS_ASSERT(bytesConsumed > 0);
+ NS_ASSERT(pendingData.length > 0, "no pending data somehow?");
+ NS_ASSERT(pendingData[pendingData.length - 1].length > 0,
+ "buffered zero bytes of data?");
+
+ NS_ASSERT(this._source !== null);
+
+ // Reading has gone great, and we've gotten data to write now. What if we
+ // don't have a place to write that data, because output went away just
+ // before this read? Drop everything on the floor, including new data, and
+ // cancel at this point.
+ if (this._sink === null)
+ {
+ pendingData.length = 0;
+ this._doneReadingSource(Cr.NS_ERROR_UNEXPECTED);
+ return;
+ }
+
+ // Okay, we've read the data, and we know we have a place to write it. We
+ // need to queue up the data to be written, but *only* if none is queued
+ // already -- if data's already queued, the code that actually writes the
+ // data will make sure to wait on unconsumed pending data.
+ try
+ {
+ if (pendingData.length === 1)
+ this._waitToWriteData();
+ }
+ catch (e)
+ {
+ dumpn("!!! error waiting to write data just read, swallowing and " +
+ "writing only what we already have: " + e);
+ this._doneWritingToSink(Cr.NS_ERROR_UNEXPECTED);
+ return;
+ }
+
+ // Whee! We successfully read some data, and it's successfully queued up to
+ // be written. All that remains now is to wait for more data to read.
+ try
+ {
+ this._waitToReadData();
+ }
+ catch (e)
+ {
+ dumpn("!!! error waiting to read more data: " + e);
+ this._doneReadingSource(Cr.NS_ERROR_UNEXPECTED);
+ }
+ },
+
+
+ // NSIOUTPUTSTREAMCALLBACK
+
+ /**
+ * Callback when data may be written to the output stream without blocking, or
+ * when the output stream has been closed.
+ *
+ * @param output : nsIAsyncOutputStream
+ * the output stream on whose writability we've been waiting, also known as
+ * this._sink
+ */
+ onOutputStreamReady: function(output)
+ {
+ if (this._sink === null)
+ return;
+
+ dumpn("*** onOutputStreamReady");
+
+ var pendingData = this._pendingData;
+ if (pendingData.length === 0)
+ {
+ // There's no pending data to write. The only way this can happen is if
+ // we're waiting on the output stream's closure, so we can respond to a
+ // copying failure as quickly as possible (rather than waiting for data to
+ // be available to read and then fail to be copied). Therefore, we must
+ // be done now -- don't bother to attempt to write anything and wrap
+ // things up.
+ dumpn("!!! output stream closed prematurely, ending copy");
+
+ this._doneWritingToSink(Cr.NS_ERROR_UNEXPECTED);
+ return;
+ }
+
+
+ NS_ASSERT(pendingData[0].length > 0, "queued up an empty quantum?");
+
+ //
+ // Write out the first pending quantum of data. The possible errors here
+ // are:
+ //
+ // The write might fail because we can't write that much data
+ // Okay, we've written what we can now, so re-queue what's left and
+ // finish writing it out later.
+ // The write failed because the stream was closed
+ // Discard pending data that we can no longer write, stop reading, and
+ // signal that copying finished.
+ // Some other error occurred.
+ // Same as if the stream were closed, but notify with the status
+ // NS_ERROR_UNEXPECTED so the observer knows something was wonky.
+ //
+
+ try
+ {
+ var quantum = pendingData[0];
+
+ // XXX |quantum| isn't guaranteed to be ASCII, so we're relying on
+ // undefined behavior! We're only using this because writeByteArray
+ // is unusably broken for asynchronous output streams; see bug 532834
+ // for details.
+ var bytesWritten = output.write(quantum, quantum.length);
+ if (bytesWritten === quantum.length)
+ pendingData.shift();
+ else
+ pendingData[0] = quantum.substring(bytesWritten);
+
+ dumpn("*** wrote " + bytesWritten + " bytes of data");
+ }
+ catch (e)
+ {
+ if (wouldBlock(e))
+ {
+ NS_ASSERT(pendingData.length > 0,
+ "stream-blocking exception with no data to write?");
+ NS_ASSERT(pendingData[0].length > 0,
+ "stream-blocking exception with empty quantum?");
+ this._waitToWriteData();
+ return;
+ }
+
+ if (streamClosed(e))
+ dumpn("!!! output stream prematurely closed, signaling error...");
+ else
+ dumpn("!!! unknown error: " + e + ", quantum=" + quantum);
+
+ this._doneWritingToSink(Cr.NS_ERROR_UNEXPECTED);
+ return;
+ }
+
+ // The day is ours! Quantum written, now let's see if we have more data
+ // still to write.
+ try
+ {
+ if (pendingData.length > 0)
+ {
+ this._waitToWriteData();
+ return;
+ }
+ }
+ catch (e)
+ {
+ dumpn("!!! unexpected error waiting to write pending data: " + e);
+ this._doneWritingToSink(Cr.NS_ERROR_UNEXPECTED);
+ return;
+ }
+
+ // Okay, we have no more pending data to write -- but might we get more in
+ // the future?
+ if (this._source !== null)
+ {
+ /*
+ * If we might, then wait for the output stream to be closed. (We wait
+ * only for closure because we have no data to write -- and if we waited
+ * for a specific amount of data, we would get repeatedly notified for no
+ * reason if over time the output stream permitted more and more data to
+ * be written to it without blocking.)
+ */
+ this._waitForSinkClosure();
+ }
+ else
+ {
+ /*
+ * On the other hand, if we can't have more data because the input
+ * stream's gone away, then it's time to notify of copy completion.
+ * Victory!
+ */
+ this._sink = null;
+ this._cancelOrDispatchCancelCallback(Cr.NS_OK);
+ }
+ },
+
+
+ // NSIREQUEST
+
+ /** Returns true if the cancel observer hasn't been notified yet. */
+ isPending: function()
+ {
+ return !this._completed;
+ },
+
+ /** Not implemented, don't use! */
+ suspend: notImplemented,
+ /** Not implemented, don't use! */
+ resume: notImplemented,
+
+ /**
+ * Cancels data reading from input, asynchronously writes out any pending
+ * data, and causes the observer to be notified with the given error code when
+ * all writing has finished.
+ *
+ * @param status : nsresult
+ * the status to pass to the observer when data copying has been canceled
+ */
+ cancel: function(status)
+ {
+ dumpn("*** cancel(" + status.toString(16) + ")");
+
+ if (this._canceled)
+ {
+ dumpn("*** suppressing a late cancel");
+ return;
+ }
+
+ this._canceled = true;
+ this.status = status;
+
+ // We could be in the middle of absolutely anything at this point. Both
+ // input and output might still be around, we might have pending data to
+ // write, and in general we know nothing about the state of the world. We
+ // therefore must assume everything's in progress and take everything to its
+ // final steady state (or so far as it can go before we need to finish
+ // writing out remaining data).
+
+ this._doneReadingSource(status);
+ },
+
+
+ // PRIVATE IMPLEMENTATION
+
+ /**
+ * Stop reading input if we haven't already done so, passing e as the status
+ * when closing the stream, and kick off a copy-completion notice if no more
+ * data remains to be written.
+ *
+ * @param e : nsresult
+ * the status to be used when closing the input stream
+ */
+ _doneReadingSource: function(e)
+ {
+ dumpn("*** _doneReadingSource(0x" + e.toString(16) + ")");
+
+ this._finishSource(e);
+ if (this._pendingData.length === 0)
+ this._sink = null;
+ else
+ NS_ASSERT(this._sink !== null, "null output?");
+
+ // If we've written out all data read up to this point, then it's time to
+ // signal completion.
+ if (this._sink === null)
+ {
+ NS_ASSERT(this._pendingData.length === 0, "pending data still?");
+ this._cancelOrDispatchCancelCallback(e);
+ }
+ },
+
+ /**
+ * Stop writing output if we haven't already done so, discard any data that
+ * remained to be sent, close off input if it wasn't already closed, and kick
+ * off a copy-completion notice.
+ *
+ * @param e : nsresult
+ * the status to be used when closing input if it wasn't already closed
+ */
+ _doneWritingToSink: function(e)
+ {
+ dumpn("*** _doneWritingToSink(0x" + e.toString(16) + ")");
+
+ this._pendingData.length = 0;
+ this._sink = null;
+ this._doneReadingSource(e);
+ },
+
+ /**
+ * Completes processing of this copy: either by canceling the copy if it
+ * hasn't already been canceled using the provided status, or by dispatching
+ * the cancel callback event (with the originally provided status, of course)
+ * if it already has been canceled.
+ *
+ * @param status : nsresult
+ * the status code to use to cancel this, if this hasn't already been
+ * canceled
+ */
+ _cancelOrDispatchCancelCallback: function(status)
+ {
+ dumpn("*** _cancelOrDispatchCancelCallback(" + status + ")");
+
+ NS_ASSERT(this._source === null, "should have finished input");
+ NS_ASSERT(this._sink === null, "should have finished output");
+ NS_ASSERT(this._pendingData.length === 0, "should have no pending data");
+
+ if (!this._canceled)
+ {
+ this.cancel(status);
+ return;
+ }
+
+ var self = this;
+ var event =
+ {
+ run: function()
+ {
+ dumpn("*** onStopRequest async callback");
+
+ self._completed = true;
+ try
+ {
+ self._observer.onStopRequest(self, self._context, self.status);
+ }
+ catch (e)
+ {
+ NS_ASSERT(false,
+ "how are we throwing an exception here? we control " +
+ "all the callers! " + e);
+ }
+ }
+ };
+
+ gThreadManager.currentThread.dispatch(event, Ci.nsIThread.DISPATCH_NORMAL);
+ },
+
+ /**
+ * Kicks off another wait for more data to be available from the input stream.
+ */
+ _waitToReadData: function()
+ {
+ dumpn("*** _waitToReadData");
+ this._source.asyncWait(this, 0, Response.SEGMENT_SIZE,
+ gThreadManager.mainThread);
+ },
+
+ /**
+ * Kicks off another wait until data can be written to the output stream.
+ */
+ _waitToWriteData: function()
+ {
+ dumpn("*** _waitToWriteData");
+
+ var pendingData = this._pendingData;
+ NS_ASSERT(pendingData.length > 0, "no pending data to write?");
+ NS_ASSERT(pendingData[0].length > 0, "buffered an empty write?");
+
+ this._sink.asyncWait(this, 0, pendingData[0].length,
+ gThreadManager.mainThread);
+ },
+
+ /**
+ * Kicks off a wait for the sink to which data is being copied to be closed.
+ * We wait for stream closure when we don't have any data to be copied, rather
+ * than waiting to write a specific amount of data. We can't wait to write
+ * data because the sink might be infinitely writable, and if no data appears
+ * in the source for a long time we might have to spin quite a bit waiting to
+ * write, waiting to write again, &c. Waiting on stream closure instead means
+ * we'll get just one notification if the sink dies. Note that when data
+ * starts arriving from the sink we'll resume waiting for data to be written,
+ * dropping this closure-only callback entirely.
+ */
+ _waitForSinkClosure: function()
+ {
+ dumpn("*** _waitForSinkClosure");
+
+ this._sink.asyncWait(this, Ci.nsIAsyncOutputStream.WAIT_CLOSURE_ONLY, 0,
+ gThreadManager.mainThread);
+ },
+
+ /**
+ * Closes input with the given status, if it hasn't already been closed;
+ * otherwise a no-op.
+ *
+ * @param status : nsresult
+ * status code use to close the source stream if necessary
+ */
+ _finishSource: function(status)
+ {
+ dumpn("*** _finishSource(" + status.toString(16) + ")");
+
+ if (this._source !== null)
+ {
+ this._source.closeWithStatus(status);
+ this._source = null;
+ }
+ }
+};
+
+
+/**
+ * A container for utility functions used with HTTP headers.
+ */
+const headerUtils =
+{
+ /**
+ * Normalizes fieldName (by converting it to lowercase) and ensures it is a
+ * valid header field name (although not necessarily one specified in RFC
+ * 2616).
+ *
+ * @throws NS_ERROR_INVALID_ARG
+ * if fieldName does not match the field-name production in RFC 2616
+ * @returns string
+ * fieldName converted to lowercase if it is a valid header, for characters
+ * where case conversion is possible
+ */
+ normalizeFieldName: function(fieldName)
+ {
+ if (fieldName == "")
+ {
+ dumpn("*** Empty fieldName");
+ throw Cr.NS_ERROR_INVALID_ARG;
+ }
+
+ for (var i = 0, sz = fieldName.length; i < sz; i++)
+ {
+ if (!IS_TOKEN_ARRAY[fieldName.charCodeAt(i)])
+ {
+ dumpn(fieldName + " is not a valid header field name!");
+ throw Cr.NS_ERROR_INVALID_ARG;
+ }
+ }
+
+ return fieldName.toLowerCase();
+ },
+
+ /**
+ * Ensures that fieldValue is a valid header field value (although not
+ * necessarily as specified in RFC 2616 if the corresponding field name is
+ * part of the HTTP protocol), normalizes the value if it is, and
+ * returns the normalized value.
+ *
+ * @param fieldValue : string
+ * a value to be normalized as an HTTP header field value
+ * @throws NS_ERROR_INVALID_ARG
+ * if fieldValue does not match the field-value production in RFC 2616
+ * @returns string
+ * fieldValue as a normalized HTTP header field value
+ */
+ normalizeFieldValue: function(fieldValue)
+ {
+ // field-value = *( field-content | LWS )
+ // field-content = <the OCTETs making up the field-value
+ // and consisting of either *TEXT or combinations
+ // of token, separators, and quoted-string>
+ // TEXT = <any OCTET except CTLs,
+ // but including LWS>
+ // LWS = [CRLF] 1*( SP | HT )
+ //
+ // quoted-string = ( <"> *(qdtext | quoted-pair ) <"> )
+ // qdtext = <any TEXT except <">>
+ // quoted-pair = "\" CHAR
+ // CHAR = <any US-ASCII character (octets 0 - 127)>
+
+ // Any LWS that occurs between field-content MAY be replaced with a single
+ // SP before interpreting the field value or forwarding the message
+ // downstream (section 4.2); we replace 1*LWS with a single SP
+ var val = fieldValue.replace(/(?:(?:\r\n)?[ \t]+)+/g, " ");
+
+ // remove leading/trailing LWS (which has been converted to SP)
+ val = val.replace(/^ +/, "").replace(/ +$/, "");
+
+ // that should have taken care of all CTLs, so val should contain no CTLs
+ dumpn("*** Normalized value: '" + val + "'");
+ for (var i = 0, len = val.length; i < len; i++)
+ if (isCTL(val.charCodeAt(i)))
+ {
+ dump("*** Char " + i + " has charcode " + val.charCodeAt(i));
+ throw Cr.NS_ERROR_INVALID_ARG;
+ }
+
+ // XXX disallows quoted-pair where CHAR is a CTL -- will not invalidly
+ // normalize, however, so this can be construed as a tightening of the
+ // spec and not entirely as a bug
+ return val;
+ }
+};
+
+
+
+/**
+ * Converts the given string into a string which is safe for use in an HTML
+ * context.
+ *
+ * @param str : string
+ * the string to make HTML-safe
+ * @returns string
+ * an HTML-safe version of str
+ */
+function htmlEscape(str)
+{
+ // this is naive, but it'll work
+ var s = "";
+ for (var i = 0; i < str.length; i++)
+ s += "&#" + str.charCodeAt(i) + ";";
+ return s;
+}
+
+
+/**
+ * Constructs an object representing an HTTP version (see section 3.1).
+ *
+ * @param versionString
+ * a string of the form "#.#", where # is an non-negative decimal integer with
+ * or without leading zeros
+ * @throws
+ * if versionString does not specify a valid HTTP version number
+ */
+function nsHttpVersion(versionString)
+{
+ var matches = /^(\d+)\.(\d+)$/.exec(versionString);
+ if (!matches)
+ throw "Not a valid HTTP version!";
+
+ /** The major version number of this, as a number. */
+ this.major = parseInt(matches[1], 10);
+
+ /** The minor version number of this, as a number. */
+ this.minor = parseInt(matches[2], 10);
+
+ if (isNaN(this.major) || isNaN(this.minor) ||
+ this.major < 0 || this.minor < 0)
+ throw "Not a valid HTTP version!";
+}
+nsHttpVersion.prototype =
+{
+ /**
+ * Returns the standard string representation of the HTTP version represented
+ * by this (e.g., "1.1").
+ */
+ toString: function ()
+ {
+ return this.major + "." + this.minor;
+ },
+
+ /**
+ * Returns true if this represents the same HTTP version as otherVersion,
+ * false otherwise.
+ *
+ * @param otherVersion : nsHttpVersion
+ * the version to compare against this
+ */
+ equals: function (otherVersion)
+ {
+ return this.major == otherVersion.major &&
+ this.minor == otherVersion.minor;
+ },
+
+ /** True if this >= otherVersion, false otherwise. */
+ atLeast: function(otherVersion)
+ {
+ return this.major > otherVersion.major ||
+ (this.major == otherVersion.major &&
+ this.minor >= otherVersion.minor);
+ }
+};
+
+nsHttpVersion.HTTP_1_0 = new nsHttpVersion("1.0");
+nsHttpVersion.HTTP_1_1 = new nsHttpVersion("1.1");
+
+
+/**
+ * An object which stores HTTP headers for a request or response.
+ *
+ * Note that since headers are case-insensitive, this object converts headers to
+ * lowercase before storing them. This allows the getHeader and hasHeader
+ * methods to work correctly for any case of a header, but it means that the
+ * values returned by .enumerator may not be equal case-sensitively to the
+ * values passed to setHeader when adding headers to this.
+ */
+function nsHttpHeaders()
+{
+ /**
+ * A hash of headers, with header field names as the keys and header field
+ * values as the values. Header field names are case-insensitive, but upon
+ * insertion here they are converted to lowercase. Header field values are
+ * normalized upon insertion to contain no leading or trailing whitespace.
+ *
+ * Note also that per RFC 2616, section 4.2, two headers with the same name in
+ * a message may be treated as one header with the same field name and a field
+ * value consisting of the separate field values joined together with a "," in
+ * their original order. This hash stores multiple headers with the same name
+ * in this manner.
+ */
+ this._headers = {};
+}
+nsHttpHeaders.prototype =
+{
+ /**
+ * Sets the header represented by name and value in this.
+ *
+ * @param name : string
+ * the header name
+ * @param value : string
+ * the header value
+ * @throws NS_ERROR_INVALID_ARG
+ * if name or value is not a valid header component
+ */
+ setHeader: function(fieldName, fieldValue, merge)
+ {
+ var name = headerUtils.normalizeFieldName(fieldName);
+ var value = headerUtils.normalizeFieldValue(fieldValue);
+
+ // The following three headers are stored as arrays because their real-world
+ // syntax prevents joining individual headers into a single header using
+ // ",". See also <http://hg.mozilla.org/mozilla-central/diff/9b2a99adc05e/netwerk/protocol/http/src/nsHttpHeaderArray.cpp#l77>
+ if (merge && name in this._headers)
+ {
+ if (name === "www-authenticate" ||
+ name === "proxy-authenticate" ||
+ name === "set-cookie")
+ {
+ this._headers[name].push(value);
+ }
+ else
+ {
+ this._headers[name][0] += "," + value;
+ NS_ASSERT(this._headers[name].length === 1,
+ "how'd a non-special header have multiple values?")
+ }
+ }
+ else
+ {
+ this._headers[name] = [value];
+ }
+ },
+
+ /**
+ * Returns the value for the header specified by this.
+ *
+ * @throws NS_ERROR_INVALID_ARG
+ * if fieldName does not constitute a valid header field name
+ * @throws NS_ERROR_NOT_AVAILABLE
+ * if the given header does not exist in this
+ * @returns string
+ * the field value for the given header, possibly with non-semantic changes
+ * (i.e., leading/trailing whitespace stripped, whitespace runs replaced
+ * with spaces, etc.) at the option of the implementation; multiple
+ * instances of the header will be combined with a comma, except for
+ * the three headers noted in the description of getHeaderValues
+ */
+ getHeader: function(fieldName)
+ {
+ return this.getHeaderValues(fieldName).join("\n");
+ },
+
+ /**
+ * Returns the value for the header specified by fieldName as an array.
+ *
+ * @throws NS_ERROR_INVALID_ARG
+ * if fieldName does not constitute a valid header field name
+ * @throws NS_ERROR_NOT_AVAILABLE
+ * if the given header does not exist in this
+ * @returns [string]
+ * an array of all the header values in this for the given
+ * header name. Header values will generally be collapsed
+ * into a single header by joining all header values together
+ * with commas, but certain headers (Proxy-Authenticate,
+ * WWW-Authenticate, and Set-Cookie) violate the HTTP spec
+ * and cannot be collapsed in this manner. For these headers
+ * only, the returned array may contain multiple elements if
+ * that header has been added more than once.
+ */
+ getHeaderValues: function(fieldName)
+ {
+ var name = headerUtils.normalizeFieldName(fieldName);
+
+ if (name in this._headers)
+ return this._headers[name];
+ else
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+ },
+
+ /**
+ * Returns true if a header with the given field name exists in this, false
+ * otherwise.
+ *
+ * @param fieldName : string
+ * the field name whose existence is to be determined in this
+ * @throws NS_ERROR_INVALID_ARG
+ * if fieldName does not constitute a valid header field name
+ * @returns boolean
+ * true if the header's present, false otherwise
+ */
+ hasHeader: function(fieldName)
+ {
+ var name = headerUtils.normalizeFieldName(fieldName);
+ return (name in this._headers);
+ },
+
+ /**
+ * Returns a new enumerator over the field names of the headers in this, as
+ * nsISupportsStrings. The names returned will be in lowercase, regardless of
+ * how they were input using setHeader (header names are case-insensitive per
+ * RFC 2616).
+ */
+ get enumerator()
+ {
+ var headers = [];
+ for (var i in this._headers)
+ {
+ var supports = new SupportsString();
+ supports.data = i;
+ headers.push(supports);
+ }
+
+ return new nsSimpleEnumerator(headers);
+ }
+};
+
+
+/**
+ * Constructs an nsISimpleEnumerator for the given array of items.
+ *
+ * @param items : Array
+ * the items, which must all implement nsISupports
+ */
+function nsSimpleEnumerator(items)
+{
+ this._items = items;
+ this._nextIndex = 0;
+}
+nsSimpleEnumerator.prototype =
+{
+ hasMoreElements: function()
+ {
+ return this._nextIndex < this._items.length;
+ },
+ getNext: function()
+ {
+ if (!this.hasMoreElements())
+ throw Cr.NS_ERROR_NOT_AVAILABLE;
+
+ return this._items[this._nextIndex++];
+ },
+ QueryInterface: function(aIID)
+ {
+ if (Ci.nsISimpleEnumerator.equals(aIID) ||
+ Ci.nsISupports.equals(aIID))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ }
+};
+
+
+/**
+ * A representation of the data in an HTTP request.
+ *
+ * @param port : uint
+ * the port on which the server receiving this request runs
+ */
+function Request(port)
+{
+ /** Method of this request, e.g. GET or POST. */
+ this._method = "";
+
+ /** Path of the requested resource; empty paths are converted to '/'. */
+ this._path = "";
+
+ /** Query string, if any, associated with this request (not including '?'). */
+ this._queryString = "";
+
+ /** Scheme of requested resource, usually http, always lowercase. */
+ this._scheme = "http";
+
+ /** Hostname on which the requested resource resides. */
+ this._host = undefined;
+
+ /** Port number over which the request was received. */
+ this._port = port;
+
+ var bodyPipe = new Pipe(false, false, 0, PR_UINT32_MAX, null);
+
+ /** Stream from which data in this request's body may be read. */
+ this._bodyInputStream = bodyPipe.inputStream;
+
+ /** Stream to which data in this request's body is written. */
+ this._bodyOutputStream = bodyPipe.outputStream;
+
+ /**
+ * The headers in this request.
+ */
+ this._headers = new nsHttpHeaders();
+
+ /**
+ * For the addition of ad-hoc properties and new functionality without having
+ * to change nsIHttpRequest every time; currently lazily created, as its only
+ * use is in directory listings.
+ */
+ this._bag = null;
+}
+Request.prototype =
+{
+ // SERVER METADATA
+
+ //
+ // see nsIHttpRequest.scheme
+ //
+ get scheme()
+ {
+ return this._scheme;
+ },
+
+ //
+ // see nsIHttpRequest.host
+ //
+ get host()
+ {
+ return this._host;
+ },
+
+ //
+ // see nsIHttpRequest.port
+ //
+ get port()
+ {
+ return this._port;
+ },
+
+ // REQUEST LINE
+
+ //
+ // see nsIHttpRequest.method
+ //
+ get method()
+ {
+ return this._method;
+ },
+
+ //
+ // see nsIHttpRequest.httpVersion
+ //
+ get httpVersion()
+ {
+ return this._httpVersion.toString();
+ },
+
+ //
+ // see nsIHttpRequest.path
+ //
+ get path()
+ {
+ return this._path;
+ },
+
+ //
+ // see nsIHttpRequest.queryString
+ //
+ get queryString()
+ {
+ return this._queryString;
+ },
+
+ // HEADERS
+
+ //
+ // see nsIHttpRequest.getHeader
+ //
+ getHeader: function(name)
+ {
+ return this._headers.getHeader(name);
+ },
+
+ //
+ // see nsIHttpRequest.hasHeader
+ //
+ hasHeader: function(name)
+ {
+ return this._headers.hasHeader(name);
+ },
+
+ //
+ // see nsIHttpRequest.headers
+ //
+ get headers()
+ {
+ return this._headers.enumerator;
+ },
+
+ //
+ // see nsIPropertyBag.enumerator
+ //
+ get enumerator()
+ {
+ this._ensurePropertyBag();
+ return this._bag.enumerator;
+ },
+
+ //
+ // see nsIHttpRequest.headers
+ //
+ get bodyInputStream()
+ {
+ return this._bodyInputStream;
+ },
+
+ //
+ // see nsIPropertyBag.getProperty
+ //
+ getProperty: function(name)
+ {
+ this._ensurePropertyBag();
+ return this._bag.getProperty(name);
+ },
+
+
+ // NSISUPPORTS
+
+ //
+ // see nsISupports.QueryInterface
+ //
+ QueryInterface: function(iid)
+ {
+ if (iid.equals(Ci.nsIHttpRequest) || iid.equals(Ci.nsISupports))
+ return this;
+
+ throw Cr.NS_ERROR_NO_INTERFACE;
+ },
+
+
+ // PRIVATE IMPLEMENTATION
+
+ /** Ensures a property bag has been created for ad-hoc behaviors. */
+ _ensurePropertyBag: function()
+ {
+ if (!this._bag)
+ this._bag = new WritablePropertyBag();
+ }
+};
+
+
+// XPCOM trappings
+
+this.NSGetFactory = XPCOMUtils.generateNSGetFactory([nsHttpServer]);
+
+/**
+ * Creates a new HTTP server listening for loopback traffic on the given port,
+ * starts it, and runs the server until the server processes a shutdown request,
+ * spinning an event loop so that events posted by the server's socket are
+ * processed.
+ *
+ * This method is primarily intended for use in running this script from within
+ * xpcshell and running a functional HTTP server without having to deal with
+ * non-essential details.
+ *
+ * Note that running multiple servers using variants of this method probably
+ * doesn't work, simply due to how the internal event loop is spun and stopped.
+ *
+ * @note
+ * This method only works with Mozilla 1.9 (i.e., Firefox 3 or trunk code);
+ * you should use this server as a component in Mozilla 1.8.
+ * @param port
+ * the port on which the server will run, or -1 if there exists no preference
+ * for a specific port; note that attempting to use some values for this
+ * parameter (particularly those below 1024) may cause this method to throw or
+ * may result in the server being prematurely shut down
+ * @param basePath
+ * a local directory from which requests will be served (i.e., if this is
+ * "/home/jwalden/" then a request to /index.html will load
+ * /home/jwalden/index.html); if this is omitted, only the default URLs in
+ * this server implementation will be functional
+ */
+function server(port, basePath)
+{
+ if (basePath)
+ {
+ var lp = Cc["@mozilla.org/file/local;1"]
+ .createInstance(Ci.nsILocalFile);
+ lp.initWithPath(basePath);
+ }
+
+ // if you're running this, you probably want to see debugging info
+ DEBUG = true;
+
+ var srv = new nsHttpServer();
+ if (lp)
+ srv.registerDirectory("/", lp);
+ srv.registerContentType("sjs", SJS_TYPE);
+ srv.identity.setPrimary("http", "localhost", port);
+ srv.start(port);
+
+ var thread = gThreadManager.currentThread;
+ while (!srv.isStopped())
+ thread.processNextEvent(true);
+
+ // get rid of any pending requests
+ while (thread.hasPendingEvents())
+ thread.processNextEvent(true);
+
+ DEBUG = false;
+}
diff --git a/test/tests/data/snapshot/img.gif b/test/tests/data/snapshot/img.gif
new file mode 100644
index 000000000..f191b280c
Binary files /dev/null and b/test/tests/data/snapshot/img.gif differ
diff --git a/test/tests/data/test.html b/test/tests/data/test.html
new file mode 100644
index 000000000..2835ff283
--- /dev/null
+++ b/test/tests/data/test.html
@@ -0,0 +1,8 @@
+<html>
+ <head>
+ <meta charset="utf-8"/>
+ </head>
+ <body>
+ <p>This is a test.</p>
+ </body>
+</html>
diff --git a/test/tests/data/test.txt b/test/tests/data/test.txt
new file mode 100644
index 000000000..6de7b8c69
--- /dev/null
+++ b/test/tests/data/test.txt
@@ -0,0 +1 @@
+This is a test file.
diff --git a/test/tests/itemTest.js b/test/tests/itemTest.js
index 486677187..0981e93a0 100644
--- a/test/tests/itemTest.js
+++ b/test/tests/itemTest.js
@@ -537,6 +537,10 @@ describe("Zotero.Item", function () {
file.append(filename);
assert.equal(item.getFilePath(), file.path);
});
+
+ it.skip("should get and set a filename for a base-dir-relative file", function* () {
+
+ })
})
describe("#attachmentPath", function () {
@@ -608,11 +612,13 @@ describe("Zotero.Item", function () {
assert.equal(OS.Path.basename(path), newName)
yield OS.File.exists(path);
+ // File should be flagged for upload
+ // DEBUG: Is this necessary?
assert.equal(
- (yield Zotero.Sync.Storage.getSyncState(item.id)),
+ (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
);
- assert.isNull(yield Zotero.Sync.Storage.getSyncedHash(item.id));
+ assert.isNull(yield Zotero.Sync.Storage.Local.getSyncedHash(item.id));
})
})
diff --git a/test/tests/storageEngineTest.js b/test/tests/storageEngineTest.js
new file mode 100644
index 000000000..1efc1b476
--- /dev/null
+++ b/test/tests/storageEngineTest.js
@@ -0,0 +1,822 @@
+"use strict";
+
+describe("Zotero.Sync.Storage.Engine", function () {
+ Components.utils.import("resource://zotero-unit/httpd.js");
+
+ var win;
+ var apiKey = Zotero.Utilities.randomString(24);
+ var port = 16213;
+ var baseURL = `http://localhost:${port}/`;
+ var server;
+
+ var responses = {};
+
+ var setup = Zotero.Promise.coroutine(function* (options = {}) {
+ server = sinon.fakeServer.create();
+ server.autoRespond = true;
+
+ Components.utils.import("resource://zotero/concurrentCaller.js");
+ var caller = new ConcurrentCaller(1);
+ caller.setLogger(msg => Zotero.debug(msg));
+ caller.stopOnError = true;
+
+ Components.utils.import("resource://zotero/config.js");
+ var client = new Zotero.Sync.APIClient({
+ baseURL,
+ apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
+ apiKey,
+ caller,
+ background: options.background || true
+ });
+
+ var engine = new Zotero.Sync.Storage.Engine({
+ apiClient: client,
+ libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
+ stopOnError: true
+ });
+
+ return { engine, client, caller };
+ });
+
+ function setResponse(response) {
+ setHTTPResponse(server, baseURL, response, responses);
+ }
+
+ function parseQueryString(str) {
+ var queryStringParams = str.split('&');
+ var params = {};
+ for (let param of queryStringParams) {
+ let [ key, val ] = param.split('=');
+ params[key] = decodeURIComponent(val);
+ }
+ return params;
+ }
+
+ function assertAPIKey(request) {
+ assert.equal(request.requestHeaders["Zotero-API-Key"], apiKey);
+ }
+
+ //
+ // Tests
+ //
+ before(function* () {
+ })
+ beforeEach(function* () {
+ Zotero.debug("BEFORE HERE");
+ yield resetDB({
+ thisArg: this,
+ skipBundledFiles: true
+ });
+ Zotero.debug("DONE RESET");
+ win = yield loadZoteroPane();
+
+ Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
+
+ this.httpd = new HttpServer();
+ this.httpd.start(port);
+
+ yield Zotero.Users.setCurrentUserID(1);
+ yield Zotero.Users.setCurrentUsername("testuser");
+
+ // Set download-on-sync by default
+ Zotero.Sync.Storage.Local.downloadOnSync(
+ Zotero.Libraries.userLibraryID, true
+ );
+ Zotero.debug("DONE BEFORE");
+ })
+ afterEach(function* () {
+ var defer = new Zotero.Promise.defer();
+ this.httpd.stop(() => defer.resolve());
+ yield defer.promise;
+ win.close();
+ })
+ after(function* () {
+ this.timeout(60000);
+ //yield resetDB();
+ win.close();
+ })
+
+
+ describe("ZFS", function () {
+ describe("Syncing", function () {
+ it("should skip downloads if no last storage sync time", function* () {
+ var { engine, client, caller } = yield setup();
+
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 404
+ });
+ var result = yield engine.start();
+
+ assert.isFalse(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+
+ // Check last sync time
+ assert.isFalse(Zotero.Libraries.userLibrary.lastStorageSync);
+ })
+
+ it("should skip downloads if unchanged last storage sync time", function* () {
+ var { engine, client, caller } = yield setup();
+
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ var library = Zotero.Libraries.userLibrary;
+ library.lastStorageSync = newStorageSyncTime;
+ yield library.saveTx();
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+ var result = yield engine.start();
+
+ assert.isFalse(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+
+ // Check last sync time
+ assert.equal(library.lastStorageSync, newStorageSyncTime);
+ })
+
+ it("should ignore a remotely missing file", function* () {
+ var { engine, client, caller } = yield setup();
+
+ var item = new Zotero.Item("attachment");
+ item.attachmentLinkMode = 'imported_file';
+ item.attachmentPath = 'storage:test.txt';
+ yield item.saveTx();
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+ this.httpd.registerPathHandler(
+ `/users/1/items/${item.key}/file`,
+ {
+ handle: function (request, response) {
+ response.setStatusLine(null, 404, null);
+ }
+ }
+ );
+ var result = yield engine.start();
+
+ assert.isFalse(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+
+ // Check last sync time
+ assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
+ })
+
+ it("should handle a remotely failing file", function* () {
+ var { engine, client, caller } = yield setup();
+
+ var item = new Zotero.Item("attachment");
+ item.attachmentLinkMode = 'imported_file';
+ item.attachmentPath = 'storage:test.txt';
+ yield item.saveTx();
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+ this.httpd.registerPathHandler(
+ `/users/1/items/${item.key}/file`,
+ {
+ handle: function (request, response) {
+ response.setStatusLine(null, 500, null);
+ }
+ }
+ );
+ // TODO: In stopOnError mode, this the promise is rejected.
+ // This should probably test with stopOnError mode turned off instead.
+ var e = yield getPromiseError(engine.start());
+ assert.equal(e.message, Zotero.Sync.Storage.defaultError);
+ })
+
+ it("should download a missing file", function* () {
+ var { engine, client, caller } = yield setup();
+
+ var item = new Zotero.Item("attachment");
+ item.attachmentLinkMode = 'imported_file';
+ item.attachmentPath = 'storage:test.txt';
+ // TODO: Test binary data
+ var text = Zotero.Utilities.randomString();
+ yield item.saveTx();
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+
+ var mtime = "1441252524905";
+ var md5 = Zotero.Utilities.Internal.md5(text)
+
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+ var s3Path = `pretend-s3/${item.key}`;
+ this.httpd.registerPathHandler(
+ `/users/1/items/${item.key}/file`,
+ {
+ handle: function (request, response) {
+ if (!request.hasHeader('Zotero-API-Key')) {
+ response.setStatusLine(null, 403, "Forbidden");
+ return;
+ }
+ var key = request.getHeader('Zotero-API-Key');
+ if (key != apiKey) {
+ response.setStatusLine(null, 403, "Invalid key");
+ return;
+ }
+ response.setStatusLine(null, 302, "Found");
+ response.setHeader("Zotero-File-Modification-Time", mtime, false);
+ response.setHeader("Zotero-File-MD5", md5, false);
+ response.setHeader("Zotero-File-Compressed", "No", false);
+ response.setHeader("Location", baseURL + s3Path, false);
+ }
+ }
+ );
+ this.httpd.registerPathHandler(
+ "/" + s3Path,
+ {
+ handle: function (request, response) {
+ response.setStatusLine(null, 200, "OK");
+ response.write(text);
+ }
+ }
+ );
+ var result = yield engine.start();
+
+ assert.isTrue(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+
+ // Check last sync time
+ assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
+ var contents = yield Zotero.File.getContentsAsync(yield item.getFilePathAsync());
+ assert.equal(contents, text);
+ })
+
+ it("should upload new files", function* () {
+ var { engine, client, caller } = yield setup();
+
+ // Single file
+ var file1 = getTestDataDirectory();
+ file1.append('test.png');
+ var item1 = yield Zotero.Attachments.importFromFile({ file: file1 });
+ var mtime1 = yield item1.attachmentModificationTime;
+ var hash1 = yield item1.attachmentHash;
+ var path1 = item1.getFilePath();
+ var filename1 = 'test.png';
+ var size1 = (yield OS.File.stat(path1)).size;
+ var contentType1 = 'image/png';
+ var prefix1 = Zotero.Utilities.randomString();
+ var suffix1 = Zotero.Utilities.randomString();
+ var uploadKey1 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
+
+ // HTML file with auxiliary image
+ var file2 = OS.Path.join(getTestDataDirectory().path, 'snapshot', 'index.html');
+ var parentItem = yield createDataObject('item');
+ var item2 = yield Zotero.Attachments.importSnapshotFromFile({
+ file: file2,
+ url: 'http://example.com/',
+ parentItemID: parentItem.id,
+ title: 'Test',
+ contentType: 'text/html',
+ charset: 'utf-8'
+ });
+ var mtime2 = yield item2.attachmentModificationTime;
+ var hash2 = yield item2.attachmentHash;
+ var path2 = item2.getFilePath();
+ var filename2 = 'index.html';
+ var size2 = (yield OS.File.stat(path2)).size;
+ var contentType2 = 'text/html';
+ var charset2 = 'utf-8';
+ var prefix2 = Zotero.Utilities.randomString();
+ var suffix2 = Zotero.Utilities.randomString();
+ var uploadKey2 = Zotero.Utilities.randomString(32, 'abcdef0123456789');
+
+ var deferreds = [];
+
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 404
+ });
+ // https://github.com/cjohansen/Sinon.JS/issues/607
+ let fixSinonBug = ";charset=utf-8";
+ server.respond(function (req) {
+ // Get upload authorization for single file
+ if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item1.key}/file`
+ && req.requestBody.indexOf('upload=') == -1) {
+ assertAPIKey(req);
+ assert.equal(req.requestHeaders["If-None-Match"], "*");
+ assert.equal(
+ req.requestHeaders["Content-Type"],
+ "application/x-www-form-urlencoded" + fixSinonBug
+ );
+
+ let parts = req.requestBody.split('&');
+ let params = {};
+ for (let part of parts) {
+ let [key, val] = part.split('=');
+ params[key] = decodeURIComponent(val);
+ }
+ assert.equal(params.md5, hash1);
+ assert.equal(params.mtime, mtime1);
+ assert.equal(params.filename, filename1);
+ assert.equal(params.filesize, size1);
+ assert.equal(params.contentType, contentType1);
+
+ req.respond(
+ 200,
+ {
+ "Content-Type": "application/json"
+ },
+ JSON.stringify({
+ url: baseURL + "pretend-s3/1",
+ contentType: contentType1,
+ prefix: prefix1,
+ suffix: suffix1,
+ uploadKey: uploadKey1
+ })
+ );
+ }
+ // Get upload authorization for multi-file zip
+ else if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item2.key}/file`
+ && req.requestBody.indexOf('upload=') == -1) {
+ assertAPIKey(req);
+ assert.equal(req.requestHeaders["If-None-Match"], "*");
+ assert.equal(
+ req.requestHeaders["Content-Type"],
+ "application/x-www-form-urlencoded" + fixSinonBug
+ );
+
+ // Verify ZIP hash
+ let tmpZipPath = OS.Path.join(
+ Zotero.getTempDirectory().path,
+ item2.key + '.zip'
+ );
+ deferreds.push({
+ promise: Zotero.Utilities.Internal.md5Async(tmpZipPath)
+ .then(function (md5) {
+ assert.equal(params.zipMD5, md5);
+ })
+ });
+
+ let parts = req.requestBody.split('&');
+ let params = {};
+ for (let part of parts) {
+ let [key, val] = part.split('=');
+ params[key] = decodeURIComponent(val);
+ }
+ Zotero.debug(params);
+ assert.equal(params.md5, hash2);
+ assert.notEqual(params.zipMD5, hash2);
+ assert.equal(params.mtime, mtime2);
+ assert.equal(params.filename, filename2);
+ assert.equal(params.zipFilename, item2.key + ".zip");
+ assert.isTrue(parseInt(params.filesize) == params.filesize);
+ assert.equal(params.contentType, contentType2);
+ assert.equal(params.charset, charset2);
+
+ req.respond(
+ 200,
+ {
+ "Content-Type": "application/json"
+ },
+ JSON.stringify({
+ url: baseURL + "pretend-s3/2",
+ contentType: 'application/zip',
+ prefix: prefix2,
+ suffix: suffix2,
+ uploadKey: uploadKey2
+ })
+ );
+ }
+ // Upload single file to S3
+ else if (req.method == "POST" && req.url == baseURL + "pretend-s3/1") {
+ assert.equal(req.requestHeaders["Content-Type"], contentType1 + fixSinonBug);
+ assert.equal(req.requestBody.size, (new Blob([prefix1, File(file1), suffix1]).size));
+ req.respond(201, {}, "");
+ }
+ // Upload multi-file ZIP to S3
+ else if (req.method == "POST" && req.url == baseURL + "pretend-s3/2") {
+ assert.equal(req.requestHeaders["Content-Type"], "application/zip" + fixSinonBug);
+
+ // Verify uploaded ZIP file
+ let tmpZipPath = OS.Path.join(
+ Zotero.getTempDirectory().path,
+ Zotero.Utilities.randomString() + '.zip'
+ );
+
+ let deferred = Zotero.Promise.defer();
+ deferreds.push(deferred);
+ var reader = new FileReader();
+ reader.addEventListener("loadend", Zotero.Promise.coroutine(function* () {
+ try {
+
+ let file = yield OS.File.open(tmpZipPath, {
+ create: true
+ });
+
+ var contents = new Uint8Array(reader.result);
+ contents = contents.slice(prefix2.length, suffix2.length * -1);
+ yield file.write(contents);
+ yield file.close();
+
+ var zr = Components.classes["@mozilla.org/libjar/zip-reader;1"]
+ .createInstance(Components.interfaces.nsIZipReader);
+ zr.open(Zotero.File.pathToFile(tmpZipPath));
+ zr.test(null);
+ var entries = zr.findEntries('*');
+ var entryNames = [];
+ while (entries.hasMore()) {
+ entryNames.push(entries.getNext());
+ }
+ assert.equal(entryNames.length, 2);
+ assert.sameMembers(entryNames, ['index.html', 'img.gif']);
+ assert.equal(zr.getEntry('index.html').realSize, size2);
+ assert.equal(zr.getEntry('img.gif').realSize, 42);
+
+ deferred.resolve();
+ }
+ catch (e) {
+ deferred.reject(e);
+ }
+ }));
+ reader.readAsArrayBuffer(req.requestBody);
+
+ req.respond(201, {}, "");
+ }
+ // Register single-file upload
+ else if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item1.key}/file`
+ && req.requestBody.indexOf('upload=') != -1) {
+ assertAPIKey(req);
+ assert.equal(req.requestHeaders["If-None-Match"], "*");
+ assert.equal(
+ req.requestHeaders["Content-Type"],
+ "application/x-www-form-urlencoded" + fixSinonBug
+ );
+
+ let parts = req.requestBody.split('&');
+ let params = {};
+ for (let part of parts) {
+ let [key, val] = part.split('=');
+ params[key] = decodeURIComponent(val);
+ }
+ assert.equal(params.upload, uploadKey1);
+
+ req.respond(
+ 204,
+ {
+ "Last-Modified-Version": 10
+ },
+ ""
+ );
+ }
+ // Register multi-file upload
+ else if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item2.key}/file`
+ && req.requestBody.indexOf('upload=') != -1) {
+ assertAPIKey(req);
+ assert.equal(req.requestHeaders["If-None-Match"], "*");
+ assert.equal(
+ req.requestHeaders["Content-Type"],
+ "application/x-www-form-urlencoded" + fixSinonBug
+ );
+
+ let parts = req.requestBody.split('&');
+ let params = {};
+ for (let part of parts) {
+ let [key, val] = part.split('=');
+ params[key] = decodeURIComponent(val);
+ }
+ assert.equal(params.upload, uploadKey2);
+
+ req.respond(
+ 204,
+ {
+ "Last-Modified-Version": 15
+ },
+ ""
+ );
+ }
+ })
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ setResponse({
+ method: "POST",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+
+ // TODO: One-step uploads
+ /*// https://github.com/cjohansen/Sinon.JS/issues/607
+ let fixSinonBug = ";charset=utf-8";
+ server.respond(function (req) {
+ if (req.method == "POST" && req.url == `${baseURL}users/1/items/${item.key}/file`) {
+ assert.equal(req.requestHeaders["If-None-Match"], "*");
+ assert.equal(
+ req.requestHeaders["Content-Type"],
+ "application/json" + fixSinonBug
+ );
+
+ let params = JSON.parse(req.requestBody);
+ assert.equal(params.md5, hash);
+ assert.equal(params.mtime, mtime);
+ assert.equal(params.filename, filename);
+ assert.equal(params.size, size);
+ assert.equal(params.contentType, contentType);
+
+ req.respond(
+ 200,
+ {
+ "Content-Type": "application/json"
+ },
+ JSON.stringify({
+ url: baseURL + "pretend-s3",
+ headers: {
+ "Content-Type": contentType,
+ "Content-MD5": hash,
+ //"Content-Length": params.size, process but don't return
+ //"x-amz-meta-"
+ },
+ uploadKey
+ })
+ );
+ }
+ else if (req.method == "PUT" && req.url == baseURL + "pretend-s3") {
+ assert.equal(req.requestHeaders["Content-Type"], contentType + fixSinonBug);
+ assert.instanceOf(req.requestBody, File);
+ req.respond(201, {}, "");
+ }
+ })*/
+ var result = yield engine.start();
+
+ yield Zotero.Promise.all(deferreds.map(d => d.promise));
+
+ assert.isTrue(result.localChanges);
+ assert.isTrue(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+
+ // Check local objects
+ assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item1.id)), mtime1);
+ assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item1.id)), hash1);
+ assert.equal(item1.version, 10);
+ assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item2.id)), mtime2);
+ assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item2.id)), hash2);
+ assert.equal(item2.version, 15);
+
+ // Check last sync time
+ assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
+ })
+
+ it("should update local info for file that already exists on the server", function* () {
+ var { engine, client, caller } = yield setup();
+
+ var file = getTestDataDirectory();
+ file.append('test.png');
+ var item = yield Zotero.Attachments.importFromFile({ file: file });
+ item.version = 5;
+ yield item.saveTx();
+ var json = yield item.toJSON();
+ yield Zotero.Sync.Data.Local.saveCacheObject('item', item.libraryID, json);
+
+ var mtime = yield item.attachmentModificationTime;
+ var hash = yield item.attachmentHash;
+ var path = item.getFilePath();
+ var filename = 'test.png';
+ var size = (yield OS.File.stat(path)).size;
+ var contentType = 'image/png';
+
+ var newVersion = 10;
+ setResponse({
+ method: "POST",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + (Math.round(new Date().getTime() / 1000) - 50000)
+ });
+ // https://github.com/cjohansen/Sinon.JS/issues/607
+ let fixSinonBug = ";charset=utf-8";
+ server.respond(function (req) {
+ // Get upload authorization for single file
+ if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item.key}/file`
+ && req.requestBody.indexOf('upload=') == -1) {
+ assertAPIKey(req);
+ assert.equal(req.requestHeaders["If-None-Match"], "*");
+ assert.equal(
+ req.requestHeaders["Content-Type"],
+ "application/x-www-form-urlencoded" + fixSinonBug
+ );
+
+ req.respond(
+ 200,
+ {
+ "Content-Type": "application/json",
+ "Last-Modified-Version": newVersion
+ },
+ JSON.stringify({
+ exists: 1,
+ })
+ );
+ }
+ })
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ setResponse({
+ method: "POST",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+
+ // TODO: One-step uploads
+ var result = yield engine.start();
+
+ assert.isTrue(result.localChanges);
+ assert.isTrue(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+
+ // Check local objects
+ assert.equal((yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id)), mtime);
+ assert.equal((yield Zotero.Sync.Storage.Local.getSyncedHash(item.id)), hash);
+ assert.equal(item.version, newVersion);
+
+ // Check last sync time
+ assert.equal(Zotero.Libraries.userLibrary.lastStorageSync, newStorageSyncTime);
+ })
+ })
+
+ describe("#_processUploadFile()", function () {
+ it("should handle 412 with matching version and hash matching local file", function* () {
+ var { engine, client, caller } = yield setup();
+ var zfs = new Zotero.Sync.Storage.ZFS_Module({
+ apiClient: client
+ })
+
+ var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
+ var item = yield Zotero.Attachments.importFromFile({ file: filePath });
+ item.version = 5;
+ item.synced = true;
+ yield item.saveTx();
+
+ var itemJSON = yield item.toResponseJSON();
+
+ // Set saved hash to a different value, which should be overwritten
+ //
+ // We're also testing cases where a hash isn't set for a file (e.g., if the
+ // storage directory was transferred, the mtime doesn't match, but the file was
+ // never downloaded), but there's no difference in behavior
+ var dbHash = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa';
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, dbHash)
+ });
+
+ server.respond(function (req) {
+ if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item.key}/file`
+ && req.requestBody.indexOf('upload=') == -1
+ && req.requestHeaders["If-Match"] == dbHash) {
+ req.respond(
+ 412,
+ {
+ "Content-Type": "application/json",
+ "Last-Modified-Version": 5
+ },
+ "ETag does not match current version of file"
+ );
+ }
+ })
+ setResponse({
+ method: "GET",
+ url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
+ status: 200,
+ text: JSON.stringify([itemJSON])
+ });
+
+ var result = yield zfs._processUploadFile({
+ name: item.libraryKey
+ });
+ yield assert.eventually.equal(
+ Zotero.Sync.Storage.Local.getSyncedHash(item.id), itemJSON.data.md5
+ );
+ assert.isFalse(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+ assert.isFalse(result.fileSyncRequired);
+ })
+
+ it("should handle 412 with matching version and hash not matching local file", function* () {
+ var { engine, client, caller } = yield setup();
+ var zfs = new Zotero.Sync.Storage.ZFS_Module({
+ apiClient: client
+ })
+
+ var filePath = OS.Path.join(getTestDataDirectory().path, 'test.png');
+ var item = yield Zotero.Attachments.importFromFile({ file: filePath });
+ item.version = 5;
+ item.synced = true;
+ yield item.saveTx();
+
+ var fileHash = yield item.attachmentHash;
+ var itemJSON = yield item.toResponseJSON();
+ itemJSON.data.md5 = 'aaaaaaaaaaaaaaaaaaaaaaaa'
+
+ server.respond(function (req) {
+ if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item.key}/file`
+ && req.requestBody.indexOf('upload=') == -1
+ && req.requestHeaders["If-None-Match"] == "*") {
+ req.respond(
+ 412,
+ {
+ "Content-Type": "application/json",
+ "Last-Modified-Version": 5
+ },
+ "If-None-Match: * set but file exists"
+ );
+ }
+ })
+ setResponse({
+ method: "GET",
+ url: `users/1/items?format=json&itemKey=${item.key}&includeTrashed=1`,
+ status: 200,
+ text: JSON.stringify([itemJSON])
+ });
+
+ var result = yield zfs._processUploadFile({
+ name: item.libraryKey
+ });
+ yield assert.eventually.isNull(Zotero.Sync.Storage.Local.getSyncedHash(item.id));
+ yield assert.eventually.equal(
+ Zotero.Sync.Storage.Local.getSyncState(item.id),
+ Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ );
+ assert.isFalse(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isFalse(result.syncRequired);
+ assert.isTrue(result.fileSyncRequired);
+ })
+
+ it("should handle 412 with greater version", function* () {
+ var { engine, client, caller } = yield setup();
+ var zfs = new Zotero.Sync.Storage.ZFS_Module({
+ apiClient: client
+ })
+
+ var file = getTestDataDirectory();
+ file.append('test.png');
+ var item = yield Zotero.Attachments.importFromFile({ file });
+ item.version = 5;
+ item.synced = true;
+ yield item.saveTx();
+
+ server.respond(function (req) {
+ if (req.method == "POST"
+ && req.url == `${baseURL}users/1/items/${item.key}/file`
+ && req.requestBody.indexOf('upload=') == -1
+ && req.requestHeaders["If-None-Match"] == "*") {
+ req.respond(
+ 412,
+ {
+ "Content-Type": "application/json",
+ "Last-Modified-Version": 10
+ },
+ "If-None-Match: * set but file exists"
+ );
+ }
+ })
+
+ var result = yield zfs._processUploadFile({
+ name: item.libraryKey
+ });
+ assert.equal(item.version, 5);
+ assert.equal(item.synced, true);
+ assert.isFalse(result.localChanges);
+ assert.isFalse(result.remoteChanges);
+ assert.isTrue(result.syncRequired);
+ })
+ })
+ })
+})
diff --git a/test/tests/storageLocalTest.js b/test/tests/storageLocalTest.js
new file mode 100644
index 000000000..31694feed
--- /dev/null
+++ b/test/tests/storageLocalTest.js
@@ -0,0 +1,329 @@
+"use strict";
+
+describe("Zotero.Sync.Storage.Local", function () {
+ var win;
+
+ before(function* () {
+ win = yield loadBrowserWindow();
+ });
+ beforeEach(function* () {
+ yield resetDB({
+ thisArg: this
+ })
+ })
+ after(function () {
+ if (win) {
+ win.close();
+ }
+ });
+
+ describe("#checkForUpdatedFiles()", function () {
+ it("should flag modified file for upload and return it", function* () {
+ // Create attachment
+ let item = yield importFileAttachment('test.txt')
+ var hash = yield item.attachmentHash;
+ // Set file mtime to the past (without milliseconds, which aren't used on OS X)
+ var mtime = (Math.floor(new Date().getTime() / 1000) * 1000) - 1000;
+ yield OS.File.setDates((yield item.getFilePathAsync()), null, mtime);
+
+ // Mark as synced, so it will be checked
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+
+ // Update mtime and contents
+ var path = yield item.getFilePathAsync();
+ yield OS.File.setDates(path);
+ yield Zotero.File.putContentsAsync(path, Zotero.Utilities.randomString());
+
+ // File should be returned
+ var libraryID = Zotero.Libraries.userLibraryID;
+ var changed = yield Zotero.Sync.Storage.Local.checkForUpdatedFiles(libraryID);
+
+ yield item.eraseTx();
+
+ assert.equal(changed, true);
+ assert.equal(
+ (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
+ Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD
+ );
+ })
+
+ it("should skip a file if mod time hasn't changed", function* () {
+ // Create attachment
+ let item = yield importFileAttachment('test.txt')
+ var hash = yield item.attachmentHash;
+ var mtime = yield item.attachmentModificationTime;
+
+ // Mark as synced, so it will be checked
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+
+ var libraryID = Zotero.Libraries.userLibraryID;
+ var changed = yield Zotero.Sync.Storage.Local.checkForUpdatedFiles(libraryID);
+ var syncState = yield Zotero.Sync.Storage.Local.getSyncState(item.id);
+
+ yield item.eraseTx();
+
+ assert.isFalse(changed);
+ assert.equal(syncState, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
+ })
+
+ it("should skip a file if mod time has changed but contents haven't", function* () {
+ // Create attachment
+ let item = yield importFileAttachment('test.txt')
+ var hash = yield item.attachmentHash;
+ // Set file mtime to the past (without milliseconds, which aren't used on OS X)
+ var mtime = (Math.floor(new Date().getTime() / 1000) * 1000) - 1000;
+ yield OS.File.setDates((yield item.getFilePathAsync()), null, mtime);
+
+ // Mark as synced, so it will be checked
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedHash(item.id, hash);
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(item.id, mtime);
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+
+ // Update mtime, but not contents
+ var path = yield item.getFilePathAsync();
+ yield OS.File.setDates(path);
+
+ var libraryID = Zotero.Libraries.userLibraryID;
+ var changed = yield Zotero.Sync.Storage.Local.checkForUpdatedFiles(libraryID);
+ var syncState = yield Zotero.Sync.Storage.Local.getSyncState(item.id);
+ var syncedModTime = yield Zotero.Sync.Storage.Local.getSyncedModificationTime(item.id);
+ var newModTime = yield item.attachmentModificationTime;
+
+ yield item.eraseTx();
+
+ assert.isFalse(changed);
+ assert.equal(syncState, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
+ assert.equal(syncedModTime, newModTime);
+ })
+ })
+
+ describe("#processDownload()", function () {
+ var file1Name = 'index.html';
+ var file1Contents = '<html><body>Test</body></html>';
+ var file2Name = 'test.txt';
+ var file2Contents = 'Test';
+
+ var createZIP = Zotero.Promise.coroutine(function* (zipFile) {
+ var tmpDir = Zotero.getTempDirectory().path;
+ var zipDir = OS.Path.join(tmpDir, Zotero.Utilities.randomString());
+ yield OS.File.makeDir(zipDir);
+
+ yield Zotero.File.putContentsAsync(OS.Path.join(zipDir, file1Name), file1Contents);
+ yield Zotero.File.putContentsAsync(OS.Path.join(zipDir, file2Name), file2Contents);
+
+ yield Zotero.File.zipDirectory(zipDir, zipFile);
+ yield OS.File.removeDir(zipDir);
+ });
+
+ it("should download and extract a ZIP file into the attachment directory", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+ var parentItem = yield createDataObject('item');
+ var key = Zotero.DataObjectUtilities.generateKey();
+
+ var tmpDir = Zotero.getTempDirectory().path;
+ var zipFile = OS.Path.join(tmpDir, key + '.tmp');
+ yield createZIP(zipFile);
+
+ var md5 = Zotero.Utilities.Internal.md5(Zotero.File.pathToFile(zipFile));
+ var mtime = 1445667239000;
+
+ var json = {
+ key,
+ version: 10,
+ itemType: 'attachment',
+ linkMode: 'imported_url',
+ url: 'https://example.com',
+ filename: file1Name,
+ contentType: 'text/html',
+ charset: 'utf-8',
+ md5,
+ mtime
+ };
+ yield Zotero.Sync.Data.Local.saveCacheObjects(
+ 'item', libraryID, [json]
+ );
+ yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
+ libraryID, 'item', { stopOnError: true }
+ );
+ var item = yield Zotero.Items.getByLibraryAndKeyAsync(libraryID, key);
+ yield Zotero.Sync.Storage.Local.processDownload({
+ item,
+ md5,
+ mtime,
+ compressed: true
+ });
+ yield OS.File.remove(zipFile);
+
+ yield assert.eventually.equal(
+ item.attachmentHash, Zotero.Utilities.Internal.md5(file1Contents)
+ );
+ yield assert.eventually.equal(item.attachmentModificationTime, mtime);
+ })
+ })
+
+ describe("#_deleteExistingAttachmentFiles()", function () {
+ it("should delete all files", function* () {
+ var item = yield importFileAttachment('test.html');
+ var path = OS.Path.dirname(item.getFilePath());
+ var files = ['a', 'b', 'c', 'd'];
+ for (let file of files) {
+ yield Zotero.File.putContentsAsync(OS.Path.join(path, file), file);
+ }
+ yield Zotero.Sync.Storage.Local._deleteExistingAttachmentFiles(item);
+ for (let file of files) {
+ assert.isFalse(
+ (yield OS.File.exists(OS.Path.join(path, file))),
+ `File '${file}' doesn't exist`
+ );
+ }
+ })
+ })
+
+ describe("#getConflicts()", function () {
+ it("should return an array of objects for attachments in conflict", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+
+ var item1 = yield importFileAttachment('test.png');
+ item1.version = 10;
+ yield item1.saveTx();
+ var item2 = yield importFileAttachment('test.txt');
+ var item3 = yield importFileAttachment('test.html');
+ item3.version = 11;
+ yield item3.saveTx();
+
+ var json1 = yield item1.toJSON();
+ var json3 = yield item3.toJSON();
+ // Change remote mtimes
+ // Round to nearest second because OS X doesn't support ms resolution
+ var now = Math.round(new Date().getTime() / 1000) * 1000;
+ json1.mtime = now - 10000;
+ json3.mtime = now - 20000;
+ yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]);
+
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item1.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ );
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item3.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ );
+
+ var conflicts = yield Zotero.Sync.Storage.Local.getConflicts(libraryID);
+ assert.lengthOf(conflicts, 2);
+
+ var item1Conflict = conflicts.find(x => x.left.key == item1.key);
+ assert.equal(
+ item1Conflict.left.dateModified,
+ Zotero.Date.dateToISO(new Date(yield item1.attachmentModificationTime))
+ );
+ assert.equal(
+ item1Conflict.right.dateModified,
+ Zotero.Date.dateToISO(new Date(json1.mtime))
+ );
+
+ var item3Conflict = conflicts.find(x => x.left.key == item3.key);
+ assert.equal(
+ item3Conflict.left.dateModified,
+ Zotero.Date.dateToISO(new Date(yield item3.attachmentModificationTime))
+ );
+ assert.equal(
+ item3Conflict.right.dateModified,
+ Zotero.Date.dateToISO(new Date(json3.mtime))
+ );
+ })
+ })
+
+ describe("#resolveConflicts()", function () {
+ it("should show the conflict resolution window on attachment conflicts", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+
+ var item1 = yield importFileAttachment('test.png');
+ item1.version = 10;
+ yield item1.saveTx();
+ var item2 = yield importFileAttachment('test.txt');
+ var item3 = yield importFileAttachment('test.html');
+ item3.version = 11;
+ yield item3.saveTx();
+
+ var json1 = yield item1.toJSON();
+ var json3 = yield item3.toJSON();
+ // Change remote mtimes
+ json1.mtime = new Date().getTime() + 10000;
+ json3.mtime = new Date().getTime() - 10000;
+ yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json1, json3]);
+
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item1.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ );
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item3.id, Zotero.Sync.Storage.SYNC_STATE_IN_CONFLICT
+ );
+
+ var promise = waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
+ var doc = dialog.document;
+ var wizard = doc.documentElement;
+ var mergeGroup = wizard.getElementsByTagName('zoteromergegroup')[0];
+
+ // 1 (remote)
+ // Later remote version should be selected
+ assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
+
+ // Check checkbox text
+ assert.equal(
+ doc.getElementById('resolve-all').label,
+ Zotero.getString('sync.conflict.resolveAllRemote')
+ );
+
+ // Select local object
+ mergeGroup.leftpane.click();
+ assert.equal(mergeGroup.leftpane.getAttribute('selected'), 'true');
+
+ wizard.getButton('next').click();
+
+ // 2 (local)
+ // Later local version should be selected
+ assert.equal(mergeGroup.leftpane.getAttribute('selected'), 'true');
+ // Select remote object
+ mergeGroup.rightpane.click();
+ assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
+
+ if (Zotero.isMac) {
+ assert.isTrue(wizard.getButton('next').hidden);
+ assert.isFalse(wizard.getButton('finish').hidden);
+ }
+ else {
+ // TODO
+ }
+ wizard.getButton('finish').click();
+ })
+ yield Zotero.Sync.Storage.Local.resolveConflicts(libraryID);
+ yield promise;
+
+ yield assert.eventually.equal(
+ Zotero.Sync.Storage.Local.getSyncState(item1.id),
+ Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD
+ );
+ yield assert.eventually.equal(
+ Zotero.Sync.Storage.Local.getSyncState(item3.id),
+ Zotero.Sync.Storage.SYNC_STATE_FORCE_DOWNLOAD
+ );
+ })
+ })
+
+
+})
diff --git a/test/tests/storageRequestTest.js b/test/tests/storageRequestTest.js
new file mode 100644
index 000000000..9ab0b0c2d
--- /dev/null
+++ b/test/tests/storageRequestTest.js
@@ -0,0 +1,22 @@
+"use strict";
+
+describe("Zotero.Sync.Storage.Request", function () {
+ describe("#run()", function () {
+ it("should run a request and wait for it to complete", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+ var count = 0;
+ var request = new Zotero.Sync.Storage.Request({
+ type: 'download',
+ libraryID,
+ name: "1/AAAAAAAA",
+ onStart: Zotero.Promise.coroutine(function* () {
+ yield Zotero.Promise.delay(25);
+ count++;
+ return new Zotero.Sync.Storage.Result;
+ })
+ });
+ var results = yield request.start();
+ assert.equal(count, 1);
+ })
+ })
+})
diff --git a/test/tests/syncEngineTest.js b/test/tests/syncEngineTest.js
index 09d602aed..80ac18970 100644
--- a/test/tests/syncEngineTest.js
+++ b/test/tests/syncEngineTest.js
@@ -19,28 +19,20 @@ describe("Zotero.Sync.Data.Engine", function () {
var caller = new ConcurrentCaller(1);
caller.setLogger(msg => Zotero.debug(msg));
caller.stopOnError = true;
- caller.onError = function (e) {
- Zotero.logError(e);
- if (options.onError) {
- options.onError(e);
- }
- if (e.fatal) {
- caller.stop();
- throw e;
- }
- };
+ Components.utils.import("resource://zotero/config.js");
var client = new Zotero.Sync.APIClient({
- baseURL: baseURL,
+ baseURL,
apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
- apiKey: apiKey,
- concurrentCaller: caller,
+ apiKey,
+ caller,
background: options.background || true
});
var engine = new Zotero.Sync.Data.Engine({
apiClient: client,
- libraryID: options.libraryID || Zotero.Libraries.userLibraryID
+ libraryID: options.libraryID || Zotero.Libraries.userLibraryID,
+ stopOnError: true
});
return { engine, client, caller };
diff --git a/test/tests/syncLocalTest.js b/test/tests/syncLocalTest.js
index 22fec6806..cecee39aa 100644
--- a/test/tests/syncLocalTest.js
+++ b/test/tests/syncLocalTest.js
@@ -4,7 +4,7 @@ describe("Zotero.Sync.Data.Local", function() {
describe("#processSyncCacheForObjectType()", function () {
var types = Zotero.DataObjectUtilities.getTypes();
- it("should update local version number if remote version is identical", function* () {
+ it("should update local version number and mark as synced if remote version is identical", function* () {
var libraryID = Zotero.Libraries.userLibraryID;
for (let type of types) {
@@ -24,11 +24,167 @@ describe("Zotero.Sync.Data.Local", function() {
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
);
- assert.equal(
- objectsClass.getByLibraryAndKey(libraryID, obj.key).version, 10
- );
+ let localObj = objectsClass.getByLibraryAndKey(libraryID, obj.key);
+ assert.equal(localObj.version, 10);
+ assert.isTrue(localObj.synced);
}
})
+
+ it("should keep local item changes while applying non-conflicting remote changes", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+
+ var type = 'item';
+ let objectsClass = Zotero.DataObjectUtilities.getObjectsClassForObjectType(type);
+ let obj = yield createDataObject(type, { version: 5 });
+ let data = yield obj.toJSON();
+ yield Zotero.Sync.Data.Local.saveCacheObjects(
+ type, libraryID, [data]
+ );
+
+ // Change local title
+ yield modifyDataObject(obj)
+ var changedTitle = obj.getField('title');
+
+ // Save remote version to cache without title but with changed place
+ data.key = obj.key;
+ data.version = 10;
+ var changedPlace = data.place = 'New York';
+ let json = {
+ key: obj.key,
+ version: 10,
+ data: data
+ };
+ yield Zotero.Sync.Data.Local.saveCacheObjects(
+ type, libraryID, [json]
+ );
+
+ yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
+ libraryID, type, { stopOnError: true }
+ );
+ assert.equal(obj.version, 10);
+ assert.equal(obj.getField('title'), changedTitle);
+ assert.equal(obj.getField('place'), changedPlace);
+ })
+
+ it("should mark new attachment items for download", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+ Zotero.Sync.Storage.Local.setModeForLibrary(libraryID, 'zfs');
+
+ var key = Zotero.DataObjectUtilities.generateKey();
+ var version = 10;
+ var json = {
+ key,
+ version,
+ data: {
+ key,
+ version,
+ itemType: 'attachment',
+ linkMode: 'imported_file',
+ md5: '57f8a4fda823187b91e1191487b87fe6',
+ mtime: 1442261130615
+ }
+ };
+
+ yield Zotero.Sync.Data.Local.saveCacheObjects(
+ 'item', Zotero.Libraries.userLibraryID, [json]
+ );
+ yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
+ libraryID, 'item', { stopOnError: true }
+ );
+ var id = Zotero.Items.getIDFromLibraryAndKey(libraryID, key);
+ assert.equal(
+ (yield Zotero.Sync.Storage.Local.getSyncState(id)),
+ Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+ })
+
+ it("should mark updated attachment items for download", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+ Zotero.Sync.Storage.Local.setModeForLibrary(libraryID, 'zfs');
+
+ var item = yield importFileAttachment('test.png');
+ item.version = 5;
+ item.synced = true;
+ yield item.saveTx();
+
+ // Set file as synced
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
+ item.id, (yield item.attachmentModificationTime)
+ );
+ yield Zotero.Sync.Storage.Local.setSyncedHash(
+ item.id, (yield item.attachmentHash)
+ );
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+
+ // Simulate download of version with updated attachment
+ var json = yield item.toResponseJSON();
+ json.version = 10;
+ json.data.version = 10;
+ json.data.md5 = '57f8a4fda823187b91e1191487b87fe6';
+ json.data.mtime = new Date().getTime() + 10000;
+ yield Zotero.Sync.Data.Local.saveCacheObjects(
+ 'item', Zotero.Libraries.userLibraryID, [json]
+ );
+
+ yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
+ libraryID, 'item', { stopOnError: true }
+ );
+
+ assert.equal(
+ (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
+ Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+ })
+
+ it("should ignore attachment metadata when resolving metadata conflict", function* () {
+ var libraryID = Zotero.Libraries.userLibraryID;
+ Zotero.Sync.Storage.Local.setModeForLibrary(libraryID, 'zfs');
+
+ var item = yield importFileAttachment('test.png');
+ item.version = 5;
+ yield item.saveTx();
+ var json = yield item.toResponseJSON();
+ yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json]);
+
+ // Set file as synced
+ yield Zotero.DB.executeTransaction(function* () {
+ yield Zotero.Sync.Storage.Local.setSyncedModificationTime(
+ item.id, (yield item.attachmentModificationTime)
+ );
+ yield Zotero.Sync.Storage.Local.setSyncedHash(
+ item.id, (yield item.attachmentHash)
+ );
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
+ );
+ });
+
+ // Modify title locally, leaving item unsynced
+ var newTitle = Zotero.Utilities.randomString();
+ item.setField('title', newTitle);
+ yield item.saveTx();
+
+ // Simulate download of version with original title but updated attachment
+ json.version = 10;
+ json.data.version = 10;
+ json.data.md5 = '57f8a4fda823187b91e1191487b87fe6';
+ json.data.mtime = new Date().getTime() + 10000;
+ yield Zotero.Sync.Data.Local.saveCacheObjects('item', libraryID, [json]);
+
+ yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
+ libraryID, 'item', { stopOnError: true }
+ );
+
+ assert.equal(item.getField('title'), newTitle);
+ assert.equal(
+ (yield Zotero.Sync.Storage.Local.getSyncState(item.id)),
+ Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+ })
})
describe("Conflict Resolution", function () {
@@ -232,7 +388,10 @@ describe("Zotero.Sync.Data.Local", function() {
jsonData.title = Zotero.Utilities.randomString();
yield Zotero.Sync.Data.Local.saveCacheObjects(type, libraryID, [json]);
+ var windowOpened = false;
waitForWindow('chrome://zotero/content/merge.xul', function (dialog) {
+ windowOpened = true;
+
var doc = dialog.document;
var wizard = doc.documentElement;
var mergeGroup = wizard.getElementsByTagName('zoteromergegroup')[0];
@@ -240,12 +399,14 @@ describe("Zotero.Sync.Data.Local", function() {
// Remote version should be selected by default
assert.equal(mergeGroup.rightpane.getAttribute('selected'), 'true');
assert.ok(mergeGroup.leftpane.pane.onclick);
+ // Select local deleted version
mergeGroup.leftpane.pane.click();
wizard.getButton('finish').click();
})
yield Zotero.Sync.Data.Local.processSyncCacheForObjectType(
libraryID, type, { stopOnError: true }
);
+ assert.isTrue(windowOpened);
obj = objectsClass.getByLibraryAndKey(libraryID, key);
assert.isFalse(obj);
@@ -825,15 +986,28 @@ describe("Zotero.Sync.Data.Local", function() {
assert.sameDeepMembers(
result.conflicts,
[
- {
- field: "place",
- op: "delete"
- },
- {
- field: "date",
- op: "add",
- value: "2015-05-15"
- }
+ [
+ {
+ field: "place",
+ op: "add",
+ value: "Place"
+ },
+ {
+ field: "place",
+ op: "delete"
+ }
+ ],
+ [
+ {
+ field: "date",
+ op: "delete"
+ },
+ {
+ field: "date",
+ op: "add",
+ value: "2015-05-15"
+ }
+ ]
]
);
})
@@ -1296,4 +1470,68 @@ describe("Zotero.Sync.Data.Local", function() {
})
})
})
+
+
+ describe("#reconcileChangesWithoutCache()", function () {
+ it("should return conflict for conflicting fields", function () {
+ var json1 = {
+ key: "AAAAAAAA",
+ version: 1234,
+ title: "Title 1",
+ pages: 10,
+ dateModified: "2015-05-14 14:12:34"
+ };
+ var json2 = {
+ key: "AAAAAAAA",
+ version: 1235,
+ title: "Title 2",
+ place: "New York",
+ dateModified: "2015-05-14 13:45:12"
+ };
+ var ignoreFields = ['dateAdded', 'dateModified'];
+ var result = Zotero.Sync.Data.Local._reconcileChangesWithoutCache(
+ 'item', json1, json2, ignoreFields
+ );
+ assert.lengthOf(result.changes, 0);
+ assert.sameDeepMembers(
+ result.conflicts,
+ [
+ [
+ {
+ field: "title",
+ op: "add",
+ value: "Title 1"
+ },
+ {
+ field: "title",
+ op: "add",
+ value: "Title 2"
+ }
+ ],
+ [
+ {
+ field: "pages",
+ op: "add",
+ value: 10
+ },
+ {
+ field: "pages",
+ op: "delete"
+ }
+ ],
+ [
+ {
+ field: "place",
+ op: "delete"
+ },
+ {
+ field: "place",
+ op: "add",
+ value: "New York"
+ }
+ ]
+ ]
+ );
+ })
+ })
})
diff --git a/test/tests/syncRunnerTest.js b/test/tests/syncRunnerTest.js
index 6f505afad..4e725f618 100644
--- a/test/tests/syncRunnerTest.js
+++ b/test/tests/syncRunnerTest.js
@@ -5,7 +5,7 @@ describe("Zotero.Sync.Runner", function () {
var apiKey = Zotero.Utilities.randomString(24);
var baseURL = "http://local.zotero/";
- var userLibraryID, publicationsLibraryID, runner, caller, server, client, stub, spy;
+ var userLibraryID, publicationsLibraryID, runner, caller, server, stub, spy;
var responses = {
keyInfo: {
@@ -129,15 +129,7 @@ describe("Zotero.Sync.Runner", function () {
}
};
- var client = new Zotero.Sync.APIClient({
- baseURL: baseURL,
- apiVersion: options.apiVersion || ZOTERO_CONFIG.API_VERSION,
- apiKey: apiKey,
- concurrentCaller: caller,
- background: options.background || true
- });
-
- return { runner, caller, client };
+ return { runner, caller };
})
function setResponse(response) {
@@ -160,7 +152,7 @@ describe("Zotero.Sync.Runner", function () {
server = sinon.fakeServer.create();
server.autoRespond = true;
- ({ runner, caller, client } = yield setup());
+ ({ runner, caller } = yield setup());
yield Zotero.Users.setCurrentUserID(1);
yield Zotero.Users.setCurrentUsername("A");
@@ -180,7 +172,7 @@ describe("Zotero.Sync.Runner", function () {
it("should check key access", function* () {
spy = sinon.spy(runner, "checkUser");
setResponse('keyInfo.fullAccess');
- var json = yield runner.checkAccess(client);
+ var json = yield runner.checkAccess(runner.getAPIClient());
sinon.assert.calledWith(spy, 1, "Username");
var compare = {};
Object.assign(compare, responses.keyInfo.fullAccess.json);
@@ -216,7 +208,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('userGroups.groupVersions');
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 4);
assert.sameMembers(
@@ -240,19 +232,25 @@ describe("Zotero.Sync.Runner", function () {
setResponse('userGroups.groupVersions');
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json, [userLibraryID]
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json, [userLibraryID]
);
assert.lengthOf(libraries, 1);
assert.sameMembers(libraries, [userLibraryID]);
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json, [userLibraryID, publicationsLibraryID]
+ runner.getAPIClient(),
+ false,
+ responses.keyInfo.fullAccess.json,
+ [userLibraryID, publicationsLibraryID]
);
assert.lengthOf(libraries, 2);
assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID]);
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json, [group1.libraryID]
+ runner.getAPIClient(),
+ false,
+ responses.keyInfo.fullAccess.json,
+ [group1.libraryID]
);
assert.lengthOf(libraries, 1);
assert.sameMembers(libraries, [group1.libraryID]);
@@ -277,7 +275,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('groups.ownerGroup');
setResponse('groups.memberGroup');
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 4);
assert.sameMembers(
@@ -318,7 +316,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('groups.ownerGroup');
setResponse('groups.memberGroup');
var libraries = yield runner.checkLibraries(
- client,
+ runner.getAPIClient(),
false,
responses.keyInfo.fullAccess.json,
[group1.libraryID, group2.libraryID]
@@ -339,7 +337,7 @@ describe("Zotero.Sync.Runner", function () {
setResponse('groups.ownerGroup');
setResponse('groups.memberGroup');
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 4);
var groupData1 = responses.groups.ownerGroup;
@@ -370,7 +368,7 @@ describe("Zotero.Sync.Runner", function () {
assert.include(text, group1.name);
});
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 3);
assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID, group2.libraryID]);
@@ -388,7 +386,7 @@ describe("Zotero.Sync.Runner", function () {
assert.include(text, group.name);
}, "extra1");
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 3);
assert.sameMembers(libraries, [userLibraryID, publicationsLibraryID, group.libraryID]);
@@ -405,7 +403,7 @@ describe("Zotero.Sync.Runner", function () {
assert.include(text, group.name);
}, "cancel");
var libraries = yield runner.checkLibraries(
- client, false, responses.keyInfo.fullAccess.json
+ runner.getAPIClient(), false, responses.keyInfo.fullAccess.json
);
assert.lengthOf(libraries, 0);
assert.isTrue(Zotero.Groups.exists(groupData.json.id));
@@ -656,6 +654,11 @@ describe("Zotero.Sync.Runner", function () {
Zotero.Libraries.getVersion(Zotero.Groups.getLibraryIDFromGroupID(2694172)),
20
);
+
+ // Last sync time should be within the last second
+ var lastSyncTime = Zotero.Sync.Data.Local.getLastSyncTime();
+ assert.isAbove(lastSyncTime, new Date().getTime() - 1000);
+ assert.isBelow(lastSyncTime, new Date().getTime());
})
})
})
diff --git a/test/tests/zoteroPaneTest.js b/test/tests/zoteroPaneTest.js
index 8d93b4e8d..26968d0e4 100644
--- a/test/tests/zoteroPaneTest.js
+++ b/test/tests/zoteroPaneTest.js
@@ -1,3 +1,5 @@
+"use strict";
+
describe("ZoteroPane", function() {
var win, doc, zp;
@@ -90,4 +92,96 @@ describe("ZoteroPane", function() {
);
})
})
+
+ describe("#viewAttachment", function () {
+ Components.utils.import("resource://zotero-unit/httpd.js");
+ var apiKey = Zotero.Utilities.randomString(24);
+ var port = 16213;
+ var baseURL = `http://localhost:${port}/`;
+ var server;
+ var responses = {};
+
+ var setup = Zotero.Promise.coroutine(function* (options = {}) {
+ server = sinon.fakeServer.create();
+ server.autoRespond = true;
+ });
+
+ function setResponse(response) {
+ setHTTPResponse(server, baseURL, response, responses);
+ }
+
+ before(function () {
+ Zotero.HTTP.mock = sinon.FakeXMLHttpRequest;
+
+ Zotero.Sync.Runner.apiKey = apiKey;
+ Zotero.Sync.Runner.baseURL = baseURL;
+ })
+ beforeEach(function* () {
+ this.httpd = new HttpServer();
+ this.httpd.start(port);
+
+ yield Zotero.Users.setCurrentUserID(1);
+ yield Zotero.Users.setCurrentUsername("testuser");
+ })
+ afterEach(function* () {
+ var defer = new Zotero.Promise.defer();
+ this.httpd.stop(() => defer.resolve());
+ yield defer.promise;
+ })
+
+ it("should download an attachment on-demand", function* () {
+ yield setup();
+ Zotero.Sync.Storage.Local.downloadAsNeeded(Zotero.Libraries.userLibraryID, true);
+
+ var item = new Zotero.Item("attachment");
+ item.attachmentLinkMode = 'imported_file';
+ item.attachmentPath = 'storage:test.txt';
+ // TODO: Test binary data
+ var text = Zotero.Utilities.randomString();
+ yield item.saveTx();
+ yield Zotero.Sync.Storage.Local.setSyncState(
+ item.id, Zotero.Sync.Storage.SYNC_STATE_TO_DOWNLOAD
+ );
+
+ var mtime = "1441252524000";
+ var md5 = Zotero.Utilities.Internal.md5(text)
+
+ var newStorageSyncTime = Math.round(new Date().getTime() / 1000);
+ setResponse({
+ method: "GET",
+ url: "users/1/laststoragesync",
+ status: 200,
+ text: "" + newStorageSyncTime
+ });
+ var s3Path = `pretend-s3/${item.key}`;
+ this.httpd.registerPathHandler(
+ `/users/1/items/${item.key}/file`,
+ {
+ handle: function (request, response) {
+ response.setStatusLine(null, 302, "Found");
+ response.setHeader("Zotero-File-Modification-Time", mtime, false);
+ response.setHeader("Zotero-File-MD5", md5, false);
+ response.setHeader("Zotero-File-Compressed", "No", false);
+ response.setHeader("Location", baseURL + s3Path, false);
+ }
+ }
+ );
+ this.httpd.registerPathHandler(
+ "/" + s3Path,
+ {
+ handle: function (request, response) {
+ response.setStatusLine(null, 200, "OK");
+ response.write(text);
+ }
+ }
+ );
+
+ yield zp.viewAttachment(item.id);
+
+ assert.equal((yield item.attachmentHash), md5);
+ assert.equal((yield item.attachmentModificationTime), mtime);
+ var path = yield item.getFilePathAsync();
+ assert.equal((yield Zotero.File.getContentsAsync(path)), text);
+ })
+ })
})