diff --git a/chrome/content/zotero/xpcom/schema.js b/chrome/content/zotero/xpcom/schema.js index 270659f32..e2e9f9465 100644 --- a/chrome/content/zotero/xpcom/schema.js +++ b/chrome/content/zotero/xpcom/schema.js @@ -1056,10 +1056,6 @@ Zotero.Schema = new function(){ Zotero.DB.query(sql); var sql = "INSERT INTO itemData VALUES (1, 1, 2)"; Zotero.DB.query(sql); - var sql = "INSERT INTO itemDataValues VALUES (3, CURRENT_TIMESTAMP)"; - Zotero.DB.query(sql); - var sql = "INSERT INTO itemData VALUES (1, 27, 3)"; - Zotero.DB.query(sql); // CHNM as creator var sql = "INSERT INTO creatorData VALUES (1, '', 'Center for History and New Media', '', 1, NULL)"; @@ -2793,6 +2789,10 @@ Zotero.Schema = new function(){ Zotero.DB.query("UPDATE itemData SET fieldID=121 WHERE itemID IN (SELECT itemID FROM items WHERE itemTypeID=19) AND fieldID=14"); } + if (i==71) { + Zotero.DB.query("UPDATE itemAttachments SET storageModTime=storageModTime*1000 WHERE storageModTime<10000000000"); + } + Zotero.wait(); } diff --git a/chrome/content/zotero/xpcom/storage.js b/chrome/content/zotero/xpcom/storage.js index b4a098df4..0bb0ad42e 100644 --- a/chrome/content/zotero/xpcom/storage.js +++ b/chrome/content/zotero/xpcom/storage.js @@ -218,7 +218,7 @@ Zotero.Sync.Storage = new function () { /** * @param {Integer} itemID - * @return {Integer|NULL} Mod time as Unix timestamp, + * @return {Integer|NULL} Mod time as timestamp in ms, * or NULL if never synced */ this.getSyncedModificationTime = function (itemID) { @@ -235,7 +235,7 @@ Zotero.Sync.Storage = new function () { /** * @param {Integer} itemID * @param {Integer} mtime File modification time as - * Unix timestamp + * timestamp in ms * @param {Boolean} [updateItem=FALSE] Update dateModified field of * attachment item */ @@ -459,10 +459,10 @@ Zotero.Sync.Storage = new function () { continue; } - var fileModTime = Math.round(file.lastModifiedTime / 1000); + var fmtime = file.lastModifiedTime; //Zotero.debug("Stored mtime is " + attachmentData[item.id].mtime); - //Zotero.debug("File mtime is " + fileModTime); + //Zotero.debug("File mtime is " + fmtime); // Download-marking mode if (itemModTimes) { @@ -481,8 +481,29 @@ Zotero.Sync.Storage = new function () { continue; } + var mtime = attachmentData[item.id].mtime; + // If stored time matches file, it hasn't changed - if (attachmentData[item.id].mtime == fileModTime) { + if (mtime == fmtime) { + continue; + } + + // Allow floored timestamps for filesystems that don't support + // millisecond precision (e.g., HFS+) + if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) { + Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") " + + "for " + file.leafName + " -- ignoring"); + continue; + } + + // Allow timestamp to be exactly one hour off to get around + // time zone issues -- there may be a proper way to fix this + if (Math.abs(fmtime - mtime) == 3600000 + // And check with one-second precision as well + || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000 + || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) { + Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") " + + "-- assuming time zone issue and skipping upload"); continue; } @@ -501,13 +522,13 @@ Zotero.Sync.Storage = new function () { } var fileHash = item.attachmentHash; if (attachmentData[item.id].hash && attachmentData[item.id].hash == fileHash) { - Zotero.debug("Mod time didn't match (" + fileModTime + "!=" + attachmentData[item.id].mtime + ") " + Zotero.debug("Mod time didn't match (" + fmtime + "!=" + mtime + ") " + "but hash did for " + file.leafName + " -- ignoring"); continue; } Zotero.debug("Marking attachment " + item.id + " as changed (" - + attachmentData[item.id].mtime + " != " + fileModTime + ")"); + + mtime + " != " + fmtime + ")"); updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD; } @@ -603,7 +624,7 @@ Zotero.Sync.Storage = new function () { } if (!data.compressed && !data.syncHash) { - _error("|data.storageHash| is required if |data.compressed| is false in " + funcName); + _error("|data.syncHash| is required if |data.compressed| is false in " + funcName); } var item = data.item; @@ -658,12 +679,12 @@ Zotero.Sync.Storage = new function () { Zotero.Sync.Storage.resyncOnFinish = true; } else { - file.lastModifiedTime = syncModTime * 1000; - - // Only save hash if file isn't compressed - if (!data.compressed) { - Zotero.Sync.Storage.setSyncedHash(item.id, syncHash); + file.lastModifiedTime = syncModTime; + // If hash not provided (e.g., WebDAV), calculate it now + if (!syncHash) { + syncHash = item.attachmentHash; } + Zotero.Sync.Storage.setSyncedHash(item.id, syncHash); Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); } @@ -817,23 +838,16 @@ Zotero.Sync.Storage = new function () { throw ("Empty path for item " + item.key + " in " + funcName); } - var itemFileName = item.getFilename(); var fileName = file.leafName; var renamed = false; - // If file doesn't match the known filename, use that name - if (itemFileName != fileName) { - Zotero.debug("Renaming file '" + fileName + "' to known filename '" + itemFileName + "'"); - fileName = itemFileName; - renamed = true; - } - // Make sure the new filename is valid, in case an invalid character made it over // (e.g., from before we checked for them) var filteredName = Zotero.File.getValidFileName(fileName); if (filteredName != fileName) { Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'"); fileName = filteredName; + file.leafName = fileName; renamed = true; } @@ -842,9 +856,7 @@ Zotero.Sync.Storage = new function () { tempFile.moveTo(parentDir, fileName); } catch (e) { - var destFile = parentDir.clone(); - destFile.QueryInterface(Components.interfaces.nsILocalFile); - destFile.setRelativeDescriptor(parentDir, fileName); + var destFile = file.clone(); var windowsLength = false; var nameLength = false; @@ -910,10 +922,7 @@ Zotero.Sync.Storage = new function () { var returnFile = null; // processDownload() needs to know that we're renaming the file if (renamed) { - destFile = parentDir.clone(); - destFile.QueryInterface(Components.interfaces.nsILocalFile); - destFile.setRelativeDescriptor(parentDir, fileName); - returnFile = destFile; + var returnFile = file.clone(); } return returnFile; @@ -966,7 +975,6 @@ Zotero.Sync.Storage = new function () { var entries = zipReader.findEntries(null); while (entries.hasMore()) { - var renamed = false; count++; var entryName = entries.getNext(); var b64re = /%ZB64$/; @@ -984,35 +992,54 @@ Zotero.Sync.Storage = new function () { continue; } + Zotero.debug("Extracting " + fileName); + + var primaryFile = false; + var filtered = false; + var renamed = false; + + // Get the old filename var itemFileName = item.getFilename(); + // Make sure the new filename is valid, in case an invalid character + // somehow make it into the ZIP (e.g., from before we checked for them) + // + // Do this before trying to use the relative descriptor, since otherwise + // it might fail silently and select the parent directory + var filteredName = Zotero.File.getValidFileName(fileName); + if (filteredName != fileName) { + Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'"); + fileName = filteredName; + filtered = true; + } + + // Name in ZIP is a relative descriptor, so file has to be reconstructed + // using setRelativeDescriptor() + var destFile = parentDir.clone(); + destFile.QueryInterface(Components.interfaces.nsILocalFile); + destFile.setRelativeDescriptor(parentDir, fileName); + + fileName = destFile.leafName; + // If only one file in zip and it doesn't match the known filename, // take our chances and use that name if (count == 1 && !entries.hasMore()) { + // May not be necessary, but let's be safe + itemFileName = Zotero.File.getValidFileName(itemFileName); if (itemFileName != fileName) { - Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'"); + Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'", 2); + Components.utils.reportError("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'"); fileName = itemFileName; + destFile.leafName = fileName; renamed = true; } } var primaryFile = itemFileName == fileName; - - // Make sure the new filename is valid, in case an invalid character - // somehow make it into the ZIP (e.g., from before we checked for them) - var filteredName = Zotero.File.getValidFileName(fileName); - if (filteredName != fileName) { - Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'"); - fileName = filteredName; - if (primaryFile) { - renamed = true; - } + if (primaryFile && filtered) { + renamed = true; } - Zotero.debug("Extracting " + fileName); - var destFile = parentDir.clone(); - destFile.QueryInterface(Components.interfaces.nsILocalFile); - destFile.setRelativeDescriptor(parentDir, fileName); if (destFile.exists()) { var msg = "ZIP entry '" + fileName + "' " + "already exists"; Zotero.debug(msg, 2); @@ -1680,10 +1707,10 @@ Zotero.Sync.Storage.QueueManager = new function () { var item = Zotero.Sync.Storage.getItemFromRequestName(conflict.name); var item1 = item.clone(false, false, true); item1.setField('dateModified', - Zotero.Date.dateToSQL(new Date(conflict.localData.modTime * 1000), true)); + Zotero.Date.dateToSQL(new Date(conflict.localData.modTime), true)); var item2 = item.clone(false, false, true); item2.setField('dateModified', - Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime * 1000), true)); + Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime), true)); objectPairs.push([item1, item2]); } diff --git a/chrome/content/zotero/xpcom/storage/webdav.js b/chrome/content/zotero/xpcom/storage/webdav.js index 8cb6479d3..00c94bf37 100644 --- a/chrome/content/zotero/xpcom/storage/webdav.js +++ b/chrome/content/zotero/xpcom/storage/webdav.js @@ -241,27 +241,52 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._getStorageModificationTime = funct var xml = new XML(req.responseText); } catch (e) { + Zotero.debug(e); var xml = null; } - if (xml && xml.childNodes.length()) { + + if (xml) { + Zotero.debug(xml.children().length()); + } + + if (xml && xml.children().length()) { // TODO: other stuff, but this makes us forward-compatible mtime = xml.mtime.toString(); + var seconds = false; } else { mtime = req.responseText; + var seconds = true; + } + + var invalid = false; + + // Unix timestamps need to be converted to ms-based timestamps + if (seconds) { + if (mtime.match(/^[0-9]{1,10}$/)) { + Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds"); + mtime = mtime * 1000; + } + else { + invalid = true; + } + } + else if (!mtime.match(/^[0-9]{1,13}$/)) { + invalid = true; } // Delete invalid .prop files - if (!(mtime + '').match(/^[0-9]{10}$/)) { - var msg = "Invalid mod date '" + Zotero.Utilities.prototype.ellipsize(mtime, 20) - + "' for item " + Zotero.Items.getLibraryKeyHash(item) + " in " + funcName; + if (invalid) { + var msg = "An error occurred during file syncing. Try the sync again.\n\n" + + "Invalid mod date '" + Zotero.Utilities.prototype.ellipsize(mtime, 20) + + "' for item " + Zotero.Items.getLibraryKeyHash(item); Zotero.debug(msg, 1); self._deleteStorageFiles([item.key + ".prop"], null, self); self.onError(msg); return; } - var mdate = new Date(mtime * 1000); + var mdate = new Date(parseInt(mtime)); callback(item, mdate); }); } @@ -271,12 +296,20 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._getStorageModificationTime = funct * Set mod time of file on storage server * * @param {Zotero.Item} item - * @param {Function} callback Callback f(item, mtime) + * @param {Function} callback Callback f(item, props) */ Zotero.Sync.Storage.Session.WebDAV.prototype._setStorageModificationTime = function (item, callback) { var uri = this._getItemPropertyURI(item); - Zotero.Utilities.HTTP.WebDAV.doPut(uri, item.attachmentModificationTime + '', function (req) { + var mtime = item.attachmentModificationTime; + var hash = item.attachmentHash; + + var prop = + {mtime} + {hash} + ; + + Zotero.Utilities.HTTP.WebDAV.doPut(uri, prop.toXMLString(), function (req) { switch (req.status) { case 200: case 201: @@ -288,7 +321,7 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._setStorageModificationTime = funct throw ("Unexpected status code " + req.status + " in " + "Zotero.Sync.Storage._setStorageModificationTime()"); } - callback(item, item.attachmentModificationTime); + callback(item, { mtime: mtime, hash: hash }); }); } @@ -324,12 +357,11 @@ Zotero.Sync.Storage.Session.WebDAV.prototype.downloadFile = function (request) { } try { - var syncModTime = Zotero.Date.toUnixTimestamp(mdate); + var syncModTime = mdate.getTime(); // Skip download if local file exists and matches mod time var file = item.getFile(); - if (file && file.exists() - && syncModTime == Math.round(file.lastModifiedTime / 1000)) { + if (file && file.exists() && syncModTime == file.lastModifiedTime) { Zotero.debug("File mod time matches remote file -- skipping download"); Zotero.DB.beginTransaction(); @@ -461,21 +493,35 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._processUploadFile = function (data != Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) { if (mdate) { // Remote prop time - var mtime = Zotero.Date.toUnixTimestamp(mdate); + var mtime = mdate.getTime(); + // Local file time var fmtime = item.attachmentModificationTime; + var same = false; + if (fmtime == mtime) { + same = true; + Zotero.debug("File mod time matches remote file -- skipping upload"); + } + // Allow floored timestamps for filesystems that don't support + // millisecond precision (e.g., HFS+) + else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) { + same = true; + Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") " + + "-- skipping upload"); + } // Allow timestamp to be exactly one hour off to get around // time zone issues -- there may be a proper way to fix this - if (fmtime == mtime || Math.abs(fmtime - mtime) == 3600) { - if (fmtime == mtime) { - Zotero.debug("File mod time matches remote file -- skipping upload"); - } - else { - Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") " - + "-- assuming time zone issue and skipping upload"); - } - + else if (Math.abs(fmtime - mtime) == 3600000 + // And check with one-second precision as well + || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000 + || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) { + same = true; + Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") " + + "-- assuming time zone issue and skipping upload"); + } + + if (same) { Zotero.DB.beginTransaction(); var syncState = Zotero.Sync.Storage.getSyncState(item.id); Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true); @@ -592,7 +638,7 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._onUploadComplete = function (httpR var self = this; - this._setStorageModificationTime(item, function (item, mtime) { + this._setStorageModificationTime(item, function (item, props) { if (!request.isRunning()) { Zotero.debug("Upload request '" + request.name + "' is no longer running after getting mod time"); @@ -602,10 +648,8 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._onUploadComplete = function (httpR Zotero.DB.beginTransaction(); Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC); - Zotero.Sync.Storage.setSyncedModificationTime(item.id, mtime, true); - - var hash = item.attachmentHash; - Zotero.Sync.Storage.setSyncedHash(item.id, hash); + Zotero.Sync.Storage.setSyncedModificationTime(item.id, props.mtime, true); + Zotero.Sync.Storage.setSyncedHash(item.id, props.hash); Zotero.DB.commitTransaction(); diff --git a/chrome/content/zotero/xpcom/storage/zfs.js b/chrome/content/zotero/xpcom/storage/zfs.js index 8f74f9c70..3aad8259e 100644 --- a/chrome/content/zotero/xpcom/storage/zfs.js +++ b/chrome/content/zotero/xpcom/storage/zfs.js @@ -130,7 +130,8 @@ Zotero.Sync.Storage.Session.ZFS.prototype._getStorageFileInfo = function (item, var info = {}; info.hash = req.getResponseHeader('ETag'); info.filename = req.getResponseHeader('X-Zotero-Filename'); - info.mtime = req.getResponseHeader('X-Zotero-Modification-Time'); + var mtime = req.getResponseHeader('X-Zotero-Modification-Time'); + info.mtime = parseInt(mtime); info.compressed = req.getResponseHeader('X-Zotero-Compressed') == 'Yes'; Zotero.debug(info); @@ -184,7 +185,7 @@ Zotero.Sync.Storage.Session.ZFS.prototype.downloadFile = function (request) { var file = item.getFile(); // Skip download if local file exists and matches mod time if (file && file.exists()) { - if (syncModTime == Math.round(file.lastModifiedTime / 1000)) { + if (syncModTime == file.lastModifiedTime) { Zotero.debug("File mod time matches remote file -- skipping download"); Zotero.DB.beginTransaction(); @@ -346,20 +347,34 @@ Zotero.Sync.Storage.Session.ZFS.prototype._processUploadFile = function (data) { if (info) { // Remote mod time var mtime = info.mtime; + // Local file time var fmtime = item.attachmentModificationTime; + var same = false; + if (fmtime == mtime) { + same = true; + Zotero.debug("File mod time matches remote file -- skipping upload"); + } + // Allow floored timestamps for filesystems that don't support + // millisecond precision (e.g., HFS+) + else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) { + same = true; + Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") " + + "-- skipping upload"); + } // Allow timestamp to be exactly one hour off to get around // time zone issues -- there may be a proper way to fix this - if (fmtime == mtime || Math.abs(fmtime - mtime) == 3600) { - if (fmtime == mtime) { - Zotero.debug("File mod time matches remote file -- skipping upload"); - } - else { - Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") " - + "-- assuming time zone issue and skipping upload"); - } - + else if (Math.abs(fmtime - mtime) == 3600000 + // And check with one-second precision as well + || Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000 + || Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) { + same = true; + Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") " + + "-- assuming time zone issue and skipping upload"); + } + + if (same) { Zotero.debug(Zotero.Sync.Storage.getSyncedModificationTime(item.id)); Zotero.DB.beginTransaction(); @@ -804,7 +819,7 @@ Zotero.Sync.Storage.Session.ZFS.prototype.getLastSyncTime = function (callback) if (req.status == 200) { var ts = req.responseText; - var date = new Date(req.responseText * 1000); + var date = new Date(ts * 1000); Zotero.debug("Last successful storage sync was " + date); self._lastSyncTime = ts; } @@ -934,7 +949,7 @@ Zotero.Sync.Storage.Session.ZFS.prototype.purgeDeletedStorageFiles = function (c */ Zotero.Sync.Storage.Session.ZFS.prototype._getItemURI = function (item) { var uri = this.rootURI; - uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1'; + uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1&version=1'; return uri; } diff --git a/chrome/content/zotero/xpcom/sync.js b/chrome/content/zotero/xpcom/sync.js index a2e0d664b..1c7af5747 100644 --- a/chrome/content/zotero/xpcom/sync.js +++ b/chrome/content/zotero/xpcom/sync.js @@ -1133,7 +1133,7 @@ Zotero.Sync.Server = new function () { this.canAutoResetClient = true; this.manualSyncRequired = false; this.nextLocalSyncDate = false; - this.apiVersion = 7; + this.apiVersion = 8; default xml namespace = ''; @@ -2450,7 +2450,8 @@ Zotero.Sync.Server.Data = new function() { continue; case 'item': - var diff = obj.diff(remoteObj, false, ["dateModified"]); + var diff = obj.diff(remoteObj, false, ["dateAdded", "dateModified"]); + Zotero.debug('Diff:'); Zotero.debug(diff); if (!diff) { // Check if creators changed @@ -2655,6 +2656,11 @@ Zotero.Sync.Server.Data = new function() { var mtime = xmlNode.@storageModTime.toString(); if (mtime) { var lk = Zotero.Items.getLibraryKeyHash(obj) + // Convert previously used Unix timestamps to ms-based timestamps + if (mtime < 10000000000) { + Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds"); + mtime = mtime * 1000; + } itemStorageModTimes[lk] = parseInt(mtime); } } diff --git a/userdata.sql b/userdata.sql index d534299b2..4987430de 100644 --- a/userdata.sql +++ b/userdata.sql @@ -1,4 +1,4 @@ --- 70 +-- 71 -- Copyright (c) 2009 Center for History and New Media -- George Mason University, Fairfax, Virginia, USA