- Use millisecond resolution for file sync timestamps (fixes problem with slow sync initialization on some systems)

- Fix errors syncing files with filenames containing extended characters

This will not sync with the server until a server update is pushed.

chrome/content/zotero/xpcom/schema.js

@@ -1056,10 +1056,6 @@ Zotero.Schema = new function(){
 				Zotero.DB.query(sql);
 				var sql = "INSERT INTO itemData VALUES (1, 1, 2)";
 				Zotero.DB.query(sql);
-				var sql = "INSERT INTO itemDataValues VALUES (3, CURRENT_TIMESTAMP)";
-				Zotero.DB.query(sql);
-				var sql = "INSERT INTO itemData VALUES (1, 27, 3)";
-				Zotero.DB.query(sql);
 				
 				// CHNM as creator
 				var sql = "INSERT INTO creatorData VALUES (1, '', 'Center for History and New Media', '', 1, NULL)";
@@ -2793,6 +2789,10 @@ Zotero.Schema = new function(){
 					Zotero.DB.query("UPDATE itemData SET fieldID=121 WHERE itemID IN (SELECT itemID FROM items WHERE itemTypeID=19) AND fieldID=14");
 				}
 				
+				if (i==71) {
+					Zotero.DB.query("UPDATE itemAttachments SET storageModTime=storageModTime*1000 WHERE storageModTime<10000000000");
+				}
+				
 				Zotero.wait();
 			}
 			

chrome/content/zotero/xpcom/storage.js

@@ -218,7 +218,7 @@ Zotero.Sync.Storage = new function () {
 	
 	/**
 	 * @param	{Integer}			itemID
-	 * @return	{Integer|NULL}					Mod time as Unix timestamp,
+	 * @return	{Integer|NULL}					Mod time as timestamp in ms,
 	 *												or NULL if never synced
 	 */
 	this.getSyncedModificationTime = function (itemID) {
@@ -235,7 +235,7 @@ Zotero.Sync.Storage = new function () {
 	/**
 	 * @param	{Integer}	itemID
 	 * @param	{Integer}	mtime				File modification time as
-	 *												Unix timestamp
+	 *												timestamp in ms
 	 * @param	{Boolean}	[updateItem=FALSE]	Update dateModified field of
 	 *												attachment item
 	 */
@@ -459,10 +459,10 @@ Zotero.Sync.Storage = new function () {
 				continue;
 			}
 			
-			var fileModTime = Math.round(file.lastModifiedTime / 1000);
+			var fmtime = file.lastModifiedTime;
 			
 			//Zotero.debug("Stored mtime is " + attachmentData[item.id].mtime);
-			//Zotero.debug("File mtime is " + fileModTime);
+			//Zotero.debug("File mtime is " + fmtime);
 			
 			// Download-marking mode
 			if (itemModTimes) {
@@ -481,8 +481,29 @@ Zotero.Sync.Storage = new function () {
 				continue;
 			}
 			
+			var mtime = attachmentData[item.id].mtime;
+			
 			// If stored time matches file, it hasn't changed
-			if (attachmentData[item.id].mtime == fileModTime) {
+			if (mtime == fmtime) {
+				continue;
+			}
+			
+			// Allow floored timestamps for filesystems that don't support
+			// millisecond precision (e.g., HFS+)
+			if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
+				Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
+					+ "for " + file.leafName + " -- ignoring");
+				continue;
+			}
+			
+			// Allow timestamp to be exactly one hour off to get around
+			// time zone issues -- there may be a proper way to fix this
+			if (Math.abs(fmtime - mtime) == 3600000
+					// And check with one-second precision as well
+					|| Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
+					|| Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
+				Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
+					+ "-- assuming time zone issue and skipping upload");
 				continue;
 			}
 			
@@ -501,13 +522,13 @@ Zotero.Sync.Storage = new function () {
 			}
 			var fileHash = item.attachmentHash;
 			if (attachmentData[item.id].hash && attachmentData[item.id].hash == fileHash) {
-				Zotero.debug("Mod time didn't match (" + fileModTime + "!=" + attachmentData[item.id].mtime + ") "
+				Zotero.debug("Mod time didn't match (" + fmtime + "!=" + mtime + ") "
 					+ "but hash did for " + file.leafName + " -- ignoring");
 				continue;
 			}
 			
 			Zotero.debug("Marking attachment " + item.id + " as changed ("
-				+ attachmentData[item.id].mtime + " != " + fileModTime + ")");
+				+ mtime + " != " + fmtime + ")");
 			updatedStates[item.id] = Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD;
 		}
 		
@@ -603,7 +624,7 @@ Zotero.Sync.Storage = new function () {
 		}
 		
 		if (!data.compressed && !data.syncHash) {
-			_error("|data.storageHash| is required if  |data.compressed| is false in " + funcName);
+			_error("|data.syncHash| is required if  |data.compressed| is false in " + funcName);
 		}
 		
 		var item = data.item;
@@ -658,12 +679,12 @@ Zotero.Sync.Storage = new function () {
 			Zotero.Sync.Storage.resyncOnFinish = true;
 		}
 		else {
-			file.lastModifiedTime = syncModTime * 1000;
+			file.lastModifiedTime = syncModTime;
-			
+			// If hash not provided (e.g., WebDAV), calculate it now
-			// Only save hash if file isn't compressed
+			if (!syncHash) {
-			if (!data.compressed) {
+				syncHash = item.attachmentHash;
-				Zotero.Sync.Storage.setSyncedHash(item.id, syncHash);
 			}
+			Zotero.Sync.Storage.setSyncedHash(item.id, syncHash);
 			Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
 		}
 		
@@ -817,23 +838,16 @@ Zotero.Sync.Storage = new function () {
 			throw ("Empty path for item " + item.key + " in " + funcName);
 		}
 		
-		var itemFileName = item.getFilename();
 		var fileName = file.leafName;
 		var renamed = false;
 		
-		// If file doesn't match the known filename, use that name
-		if (itemFileName != fileName) {
-			Zotero.debug("Renaming file '" + fileName + "' to known filename '" + itemFileName + "'");
-			fileName = itemFileName;
-			renamed = true;
-		}
-		
 		// Make sure the new filename is valid, in case an invalid character made it over
 		// (e.g., from before we checked for them)
 		var filteredName = Zotero.File.getValidFileName(fileName);
 		if (filteredName != fileName) {
 			Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
 			fileName = filteredName;
+			file.leafName = fileName;
 			renamed = true;
 		}
 		
@@ -842,9 +856,7 @@ Zotero.Sync.Storage = new function () {
 			tempFile.moveTo(parentDir, fileName);
 		}
 		catch (e) {
-			var destFile = parentDir.clone();
+			var destFile = file.clone();
-			destFile.QueryInterface(Components.interfaces.nsILocalFile);
-			destFile.setRelativeDescriptor(parentDir, fileName);
 			
 			var windowsLength = false;
 			var nameLength = false;
@@ -910,10 +922,7 @@ Zotero.Sync.Storage = new function () {
 		var returnFile = null;
 		// processDownload() needs to know that we're renaming the file
 		if (renamed) {
-			destFile = parentDir.clone();
+			var returnFile = file.clone();
-			destFile.QueryInterface(Components.interfaces.nsILocalFile);
-			destFile.setRelativeDescriptor(parentDir, fileName);
-			returnFile = destFile;
 		}
 		
 		return returnFile;
@@ -966,7 +975,6 @@ Zotero.Sync.Storage = new function () {
 		
 		var entries = zipReader.findEntries(null);
 		while (entries.hasMore()) {
-			var renamed = false;
 			count++;
 			var entryName = entries.getNext();
 			var b64re = /%ZB64$/;
@@ -984,35 +992,54 @@ Zotero.Sync.Storage = new function () {
 				continue;
 			}
 			
+			Zotero.debug("Extracting " + fileName);
+			
+			var primaryFile = false;
+			var filtered = false;
+			var renamed = false;
+			
+			// Get the old filename
 			var itemFileName = item.getFilename();
 			
+			// Make sure the new filename is valid, in case an invalid character
+			// somehow make it into the ZIP (e.g., from before we checked for them)
+			//
+			// Do this before trying to use the relative descriptor, since otherwise
+			// it might fail silently and select the parent directory
+			var filteredName = Zotero.File.getValidFileName(fileName);
+			if (filteredName != fileName) {
+				Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
+				fileName = filteredName;
+				filtered = true;
+			}
+			
+			// Name in ZIP is a relative descriptor, so file has to be reconstructed
+			// using setRelativeDescriptor()
+			var destFile = parentDir.clone();
+			destFile.QueryInterface(Components.interfaces.nsILocalFile);
+			destFile.setRelativeDescriptor(parentDir, fileName);
+			
+			fileName = destFile.leafName;
+			
 			// If only one file in zip and it doesn't match the known filename,
 			// take our chances and use that name
 			if (count == 1 && !entries.hasMore()) {
+				// May not be necessary, but let's be safe
+				itemFileName = Zotero.File.getValidFileName(itemFileName);
 				if (itemFileName != fileName) {
-					Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'");
+					Zotero.debug("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'", 2);
+					Components.utils.reportError("Renaming single file '" + fileName + "' in ZIP to known filename '" + itemFileName + "'");
 					fileName = itemFileName;
+					destFile.leafName = fileName;
 					renamed = true;
 				}
 			}
 			
 			var primaryFile = itemFileName == fileName;
-			
+			if (primaryFile && filtered) {
-			// Make sure the new filename is valid, in case an invalid character
-			// somehow make it into the ZIP (e.g., from before we checked for them)
-			var filteredName = Zotero.File.getValidFileName(fileName);
-			if (filteredName != fileName) {
-				Zotero.debug("Filtering filename '" + fileName + "' to '" + filteredName + "'");
-				fileName = filteredName;
-				if (primaryFile) {
 				renamed = true;
 			}
-			}
 			
-			Zotero.debug("Extracting " + fileName);
-			var destFile = parentDir.clone();
-			destFile.QueryInterface(Components.interfaces.nsILocalFile);
-			destFile.setRelativeDescriptor(parentDir, fileName);
 			if (destFile.exists()) {
 				var msg = "ZIP entry '" + fileName + "' " + "already exists";
 				Zotero.debug(msg, 2);
@@ -1680,10 +1707,10 @@ Zotero.Sync.Storage.QueueManager = new function () {
 			var item = Zotero.Sync.Storage.getItemFromRequestName(conflict.name);
 			var item1 = item.clone(false, false, true);
 			item1.setField('dateModified',
-				Zotero.Date.dateToSQL(new Date(conflict.localData.modTime * 1000), true));
+				Zotero.Date.dateToSQL(new Date(conflict.localData.modTime), true));
 			var item2 = item.clone(false, false, true);
 			item2.setField('dateModified',
-				Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime * 1000), true));
+				Zotero.Date.dateToSQL(new Date(conflict.remoteData.modTime), true));
 			objectPairs.push([item1, item2]);
 		}
 		

chrome/content/zotero/xpcom/storage/webdav.js

@@ -241,27 +241,52 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._getStorageModificationTime = funct
 			var xml = new XML(req.responseText);
 		}
 		catch (e) {
+			Zotero.debug(e);
 			var xml = null;
 		}
-		if (xml && xml.childNodes.length()) {
+		
+		if (xml) {
+			Zotero.debug(xml.children().length());
+		}
+		
+		if (xml && xml.children().length()) {
 			// TODO: other stuff, but this makes us forward-compatible
 			mtime = xml.mtime.toString();
+			var seconds = false;
 		}
 		else {
 			mtime = req.responseText;
+			var seconds = true;
+		}
+		
+		var invalid = false;
+		
+		// Unix timestamps need to be converted to ms-based timestamps
+		if (seconds) {
+			if (mtime.match(/^[0-9]{1,10}$/)) {
+				Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds");
+				mtime = mtime * 1000;
+			}
+			else {
+				invalid = true;
+			}
+		}
+		else if (!mtime.match(/^[0-9]{1,13}$/)) {
+			invalid = true;
 		}
 		
 		// Delete invalid .prop files
-		if (!(mtime + '').match(/^[0-9]{10}$/)) {
+		if (invalid) {
-			var msg = "Invalid mod date '" + Zotero.Utilities.prototype.ellipsize(mtime, 20)
+			var msg = "An error occurred during file syncing. Try the sync again.\n\n"
-				+ "' for item " + Zotero.Items.getLibraryKeyHash(item) + " in " + funcName;
+				+ "Invalid mod date '" + Zotero.Utilities.prototype.ellipsize(mtime, 20)
+				+ "' for item " + Zotero.Items.getLibraryKeyHash(item);
 			Zotero.debug(msg, 1);
 			self._deleteStorageFiles([item.key + ".prop"], null, self);
 			self.onError(msg);
 			return;
 		}
 		
-		var mdate = new Date(mtime * 1000);
+		var mdate = new Date(parseInt(mtime));
 		callback(item, mdate);
 	});
 }
@@ -271,12 +296,20 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._getStorageModificationTime = funct
  * Set mod time of file on storage server
  *
  * @param	{Zotero.Item}	item
- * @param	{Function}		callback		Callback f(item, mtime)
+ * @param	{Function}		callback		Callback f(item, props)
  */
 Zotero.Sync.Storage.Session.WebDAV.prototype._setStorageModificationTime = function (item, callback) {
 	var uri = this._getItemPropertyURI(item);
 	
-	Zotero.Utilities.HTTP.WebDAV.doPut(uri, item.attachmentModificationTime + '', function (req) {
+	var mtime = item.attachmentModificationTime;
+	var hash = item.attachmentHash;
+	
+	var prop = <properties version="1">
+		<mtime>{mtime}</mtime>
+		<hash>{hash}</hash>
+	</properties>;
+	
+	Zotero.Utilities.HTTP.WebDAV.doPut(uri, prop.toXMLString(), function (req) {
 		switch (req.status) {
 			case 200:
 			case 201:
@@ -288,7 +321,7 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._setStorageModificationTime = funct
 				throw ("Unexpected status code " + req.status + " in "
 					+ "Zotero.Sync.Storage._setStorageModificationTime()");
 		}
-		callback(item, item.attachmentModificationTime);
+		callback(item, { mtime: mtime, hash: hash });
 	});
 }
 
@@ -324,12 +357,11 @@ Zotero.Sync.Storage.Session.WebDAV.prototype.downloadFile = function (request) {
 		}
 		
 		try {
-			var syncModTime = Zotero.Date.toUnixTimestamp(mdate);
+			var syncModTime = mdate.getTime();
 			
 			// Skip download if local file exists and matches mod time
 			var file = item.getFile();
-			if (file && file.exists()
+			if (file && file.exists() && syncModTime == file.lastModifiedTime) {
-					&& syncModTime == Math.round(file.lastModifiedTime / 1000)) {
 				Zotero.debug("File mod time matches remote file -- skipping download");
 				
 				Zotero.DB.beginTransaction();
@@ -461,21 +493,35 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._processUploadFile = function (data
 					!= Zotero.Sync.Storage.SYNC_STATE_FORCE_UPLOAD) {
 				if (mdate) {
 					// Remote prop time
-					var mtime = Zotero.Date.toUnixTimestamp(mdate);
+					var mtime = mdate.getTime();
+					
 					// Local file time
 					var fmtime = item.attachmentModificationTime;
 					
-					// Allow timestamp to be exactly one hour off to get around
+					var same = false;
-					// time zone issues -- there may be a proper way to fix this
-					if (fmtime == mtime || Math.abs(fmtime - mtime) == 3600) {
 					if (fmtime == mtime) {
+						same = true;
 						Zotero.debug("File mod time matches remote file -- skipping upload");
 					}
-						else {
+					// Allow floored timestamps for filesystems that don't support
+					// millisecond precision (e.g., HFS+)
+					else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
+						same = true;
+						Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
+							+ "-- skipping upload");
+					}
+					// Allow timestamp to be exactly one hour off to get around
+					// time zone issues -- there may be a proper way to fix this
+					else if (Math.abs(fmtime - mtime) == 3600000
+							// And check with one-second precision as well
+							|| Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
+							|| Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
+						same = true;
 						Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
 							+ "-- assuming time zone issue and skipping upload");
 					}
 					
+					if (same) {
 						Zotero.DB.beginTransaction();
 						var syncState = Zotero.Sync.Storage.getSyncState(item.id);
 						Zotero.Sync.Storage.setSyncedModificationTime(item.id, fmtime, true);
@@ -592,7 +638,7 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._onUploadComplete = function (httpR
 	
 	var self = this;
 	
-	this._setStorageModificationTime(item, function (item, mtime) {
+	this._setStorageModificationTime(item, function (item, props) {
 		if (!request.isRunning()) {
 			Zotero.debug("Upload request '" + request.name
 				+ "' is no longer running after getting mod time");
@@ -602,10 +648,8 @@ Zotero.Sync.Storage.Session.WebDAV.prototype._onUploadComplete = function (httpR
 		Zotero.DB.beginTransaction();
 		
 		Zotero.Sync.Storage.setSyncState(item.id, Zotero.Sync.Storage.SYNC_STATE_IN_SYNC);
-		Zotero.Sync.Storage.setSyncedModificationTime(item.id, mtime, true);
+		Zotero.Sync.Storage.setSyncedModificationTime(item.id, props.mtime, true);
-		
+		Zotero.Sync.Storage.setSyncedHash(item.id, props.hash);
-		var hash = item.attachmentHash;
-		Zotero.Sync.Storage.setSyncedHash(item.id, hash);
 		
 		Zotero.DB.commitTransaction();
 		

chrome/content/zotero/xpcom/storage/zfs.js

@@ -130,7 +130,8 @@ Zotero.Sync.Storage.Session.ZFS.prototype._getStorageFileInfo = function (item,
 		var info = {};
 		info.hash = req.getResponseHeader('ETag');
 		info.filename = req.getResponseHeader('X-Zotero-Filename');
-		info.mtime = req.getResponseHeader('X-Zotero-Modification-Time');
+		var mtime = req.getResponseHeader('X-Zotero-Modification-Time');
+		info.mtime = parseInt(mtime);
 		info.compressed = req.getResponseHeader('X-Zotero-Compressed') == 'Yes';
 		Zotero.debug(info);
 		
@@ -184,7 +185,7 @@ Zotero.Sync.Storage.Session.ZFS.prototype.downloadFile = function (request) {
 			var file = item.getFile();
 			// Skip download if local file exists and matches mod time
 			if (file && file.exists()) {
-				if (syncModTime == Math.round(file.lastModifiedTime / 1000)) {
+				if (syncModTime == file.lastModifiedTime) {
 					Zotero.debug("File mod time matches remote file -- skipping download");
 					
 					Zotero.DB.beginTransaction();
@@ -346,20 +347,34 @@ Zotero.Sync.Storage.Session.ZFS.prototype._processUploadFile = function (data) {
 				if (info) {
 					// Remote mod time
 					var mtime = info.mtime;
+					
 					// Local file time
 					var fmtime = item.attachmentModificationTime;
 					
-					// Allow timestamp to be exactly one hour off to get around
+					var same = false;
-					// time zone issues -- there may be a proper way to fix this
-					if (fmtime == mtime || Math.abs(fmtime - mtime) == 3600) {
 					if (fmtime == mtime) {
+						same = true;
 						Zotero.debug("File mod time matches remote file -- skipping upload");
 					}
-						else {
+					// Allow floored timestamps for filesystems that don't support
+					// millisecond precision (e.g., HFS+)
+					else if (Math.floor(mtime / 1000) * 1000 == fmtime || Math.floor(fmtime / 1000) * 1000 == mtime) {
+						same = true;
+						Zotero.debug("File mod times are within one-second precision (" + fmtime + " ≅ " + mtime + ") "
+							+ "-- skipping upload");
+					}
+					// Allow timestamp to be exactly one hour off to get around
+					// time zone issues -- there may be a proper way to fix this
+					else if (Math.abs(fmtime - mtime) == 3600000
+							// And check with one-second precision as well
+							|| Math.abs(fmtime - Math.floor(mtime / 1000) * 1000) == 3600000
+							|| Math.abs(Math.floor(fmtime / 1000) * 1000 - mtime) == 3600000) {
+						same = true;
 						Zotero.debug("File mod time (" + fmtime + ") is exactly one hour off remote file (" + mtime + ") "
 							+ "-- assuming time zone issue and skipping upload");
 					}
 					
+					if (same) {
 						Zotero.debug(Zotero.Sync.Storage.getSyncedModificationTime(item.id));
 						
 						Zotero.DB.beginTransaction();
@@ -804,7 +819,7 @@ Zotero.Sync.Storage.Session.ZFS.prototype.getLastSyncTime = function (callback)
 		
 		if (req.status == 200) {
 			var ts = req.responseText;
-			var date = new Date(req.responseText * 1000);
+			var date = new Date(ts * 1000);
 			Zotero.debug("Last successful storage sync was " + date);
 			self._lastSyncTime = ts;
 		}
@@ -934,7 +949,7 @@ Zotero.Sync.Storage.Session.ZFS.prototype.purgeDeletedStorageFiles = function (c
  */
 Zotero.Sync.Storage.Session.ZFS.prototype._getItemURI = function (item) {
 	var uri = this.rootURI;
-	uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1';
+	uri.spec += Zotero.URI.getItemPath(item) + '/file?auth=1&iskey=1&version=1';
 	return uri;
 }
 

chrome/content/zotero/xpcom/sync.js

@@ -1133,7 +1133,7 @@ Zotero.Sync.Server = new function () {
 	this.canAutoResetClient = true;
 	this.manualSyncRequired = false;
 	this.nextLocalSyncDate = false;
-	this.apiVersion = 7;
+	this.apiVersion = 8;
 	
 	default xml namespace = '';
 	
@@ -2450,7 +2450,8 @@ Zotero.Sync.Server.Data = new function() {
 									continue;
 									
 								case 'item':
-									var diff = obj.diff(remoteObj, false, ["dateModified"]);
+									var diff = obj.diff(remoteObj, false, ["dateAdded", "dateModified"]);
+									Zotero.debug('Diff:');
 									Zotero.debug(diff);
 									if (!diff) {
 										// Check if creators changed
@@ -2655,6 +2656,11 @@ Zotero.Sync.Server.Data = new function() {
 							var mtime = xmlNode.@storageModTime.toString();
 							if (mtime) {
 								var lk = Zotero.Items.getLibraryKeyHash(obj)
+								// Convert previously used Unix timestamps to ms-based timestamps
+								if (mtime < 10000000000) {
+									Zotero.debug("Converting Unix timestamp '" + mtime + "' to milliseconds");
+									mtime = mtime * 1000;
+								}
 								itemStorageModTimes[lk] = parseInt(mtime);
 							}
 						}

userdata.sql

@@ -1,4 +1,4 @@
--- 70
+-- 71
 
 -- Copyright (c) 2009 Center for History and New Media
 --                    George Mason University, Fairfax, Virginia, USA