Optimize local file modification checks during file syncs

- On manual sync or the first auto-sync of a session, check all files
- During other auto-syncs, check only files previously modified or
  opened externally via Zotero (including Show File) within the last 3
  hours
- Every 3 hours, do a full check of all files even if it's an auto-sync
- Spin event loop during synchronous file checks to avoid hanging the UI
- Zotero.Sync.Runner.sync() now takes an options object (e.g.,
  options.background)

Todo:

- Provide feedback on last full check in sync icon tooltip?
- Auto-sync on app focus, if this speeds up syncs enough?

chrome/content/zotero/xpcom/notifier.js

@@ -27,7 +27,7 @@ Zotero.Notifier = new function(){
 	var _observers = {};
 	var _disabled = false;
 	var _types = [
-		'collection', 'creator', 'search', 'share', 'share-items', 'item',
+		'collection', 'creator', 'search', 'share', 'share-items', 'item', 'file',
 		'collection-item', 'item-tag', 'tag', 'setting', 'group', 'trash', 'bucket', 'relation'
 	];
 	var _inTransaction;

chrome/content/zotero/xpcom/storage.js

@@ -75,17 +75,23 @@ Zotero.Sync.Storage = new function () {
 		}
 	}
 	
+	Zotero.Notifier.registerObserver(this, ['file']);
+	
+	
 	//
 	// Private properties
 	//
+	var _maxCheckAgeInSeconds = 10800; // maximum age for upload modification check (3 hours)
 	var _syncInProgress;
 	var _updatesInProgress;
 	var _itemDownloadPercentages = {};
+	var _uploadCheckFiles = [];
+	var _lastFullFileCheck = {};
 	
 	
-	this.sync = function (libraries) {
+	this.sync = function (options) {
-		if (libraries) {
+		if (options.libraries) {
-			Zotero.debug("Starting file sync for libraries " + libraries);
+			Zotero.debug("Starting file sync for libraries " + options.libraries);
 		}
 		else {
 			Zotero.debug("Starting file sync");
@@ -100,7 +106,7 @@ Zotero.Sync.Storage = new function () {
 		return Q.fcall(function () {
 			// TODO: Make sure modes are active
 			
-			if (libraries && libraries.indexOf(0) == -1) {
+			if (options.libraries && options.libraries.indexOf(0) == -1) {
 				return;
 			}
 			
@@ -116,7 +122,7 @@ Zotero.Sync.Storage = new function () {
 			if (Zotero.Sync.Storage.ZFS.includeGroupFiles) {
 				var groups = Zotero.Groups.getAll();
 				for each(var group in groups) {
-					if (libraries && libraries.indexOf(group.libraryID) == -1) {
+					if (options.libraries && options.libraries.indexOf(group.libraryID) == -1) {
 						continue;
 					}
 					// TODO: if library file syncing enabled
@@ -214,8 +220,37 @@ Zotero.Sync.Storage = new function () {
 			});
 			
 			// Check for updated files to upload in each library
-			return Q.all([self.checkForUpdatedFiles(null, parseInt(libraryID))
+			var promises = [];
-				for (libraryID in librarySyncTimes)])
+			for (let libraryID in librarySyncTimes) {
+				let promise;
+				libraryID = parseInt(libraryID);
+				
+				if (!Zotero.Libraries.isFilesEditable(libraryID)) {
+					Zotero.debug("No file editing access -- skipping file "
+						+ "modification check for library " + libraryID);
+					continue;
+				}
+				// If this is a background sync, it's not the first sync of
+				// the session, the library has had at least one full check
+				// this session, and it's been less than _maxCheckAgeInSeconds
+				// since the last full check of this library, check only files
+				// that were previously modified or opened recently
+				else if (options.background
+						&& !options.firstInSession
+						&& _lastFullFileCheck[libraryID]
+						&& (_lastFullFileCheck[libraryID] + (_maxCheckAgeInSeconds * 1000))
+							> new Date().getTime()) {
+					let itemIDs = _getFilesToCheck(libraryID);
+					promise = self.checkForUpdatedFiles(libraryID, itemIDs);
+				}
+				// Otherwise check all files in the library
+				else {
+					_lastFullFileCheck[libraryID] = new Date().getTime();
+					promise = self.checkForUpdatedFiles(libraryID);
+				}
+				promises.push(promise);
+			}
+			return Q.all(promises)
 			.then(function () {
 				// Queue files to download and upload from each library
 				for (let libraryID in librarySyncTimes) {
@@ -646,18 +681,25 @@ Zotero.Sync.Storage = new function () {
 	 * Scans local files and marks any that have changed for uploading
 	 * and any that are missing for downloading
 	 *
+	 * @param {Integer} [libraryID]
+	 * @param {Integer[]} [itemIDs]
 	 * @param {Object} [itemModTimes]  Item mod times indexed by item ids;
 	 *                                 items with stored mod times
 	 *                                 that differ from the provided
 	 *                                 time but file mod times
 	 *                                 matching the stored time will
 	 *                                 be marked for download
-	 * @param	{Boolean}	[includePersonalItems=false]
-	 * @param	{Boolean}	[includeGroupItems=false]
 	 * @return {Promise} Promise resolving to TRUE if any items changed state,
 	 *                   FALSE otherwise
 	 */
-	this.checkForUpdatedFiles = function (itemModTimes, libraryID) {
+	this.checkForUpdatedFiles = function (libraryID, itemIDs, itemModTimes) {
+		libraryID = parseInt(libraryID);
+		if (isNaN(libraryID)) {
+			libraryID = false;
+		}
+		
+		Components.utils.import("resource://gre/modules/Task.jsm");
+		return Q(Task.spawn(function () {
 			var msg = "Checking for locally changed attachment files";
 			
 			var memmgr = Components.classes["@mozilla.org/memory-reporter-manager;1"]
@@ -665,22 +707,39 @@ Zotero.Sync.Storage = new function () {
 			memmgr.init();
 			Zotero.debug("Memory usage: " + memmgr.resident);
 			
-		if (typeof libraryID != 'undefined') {
+			if (libraryID !== false) {
-			msg += " in library " + libraryID;
+				if (itemIDs) {
-			if (itemModTimes) {
+					if (!itemIDs.length) {
-				throw new Error("libraryID is not allowed when itemModTimes is set");
+						var msg = "No files to check for local changes in library " + libraryID;
+						Zotero.debug(msg);
+						throw new Task.Result(false);
 					}
 				}
+				if (itemModTimes) {
+					throw new Error("itemModTimes is not allowed when libraryID is set");
+				}
+				
+				msg += " in library " + libraryID;
+			}
+			else if (itemIDs) {
+				throw new Error("libraryID not provided");
+			}
+			else if (itemModTimes) {
+				if (!Object.keys(itemModTimes).length) {
+					throw new Task.Result(false);
+				}
+				msg += " in download-marking mode";
+			}
 			else {
-			if (!itemModTimes) {
+				throw new Error("libraryID, itemIDs, or itemModTimes must be provided");
-				return Q(false);
-			}
 			}
 			Zotero.debug(msg);
 			
 			var changed = false;
 			
-		var itemIDs = Object.keys(itemModTimes ? itemModTimes : {});
+			if (!itemIDs) {
+				itemIDs = Object.keys(itemModTimes ? itemModTimes : {});
+			}
 			
 			// Can only handle 999 bound parameters at a time
 			var numIDs = itemIDs.length;
@@ -702,7 +761,7 @@ Zotero.Sync.Storage = new function () {
 					Zotero.Sync.Storage.SYNC_STATE_TO_UPLOAD,
 					Zotero.Sync.Storage.SYNC_STATE_IN_SYNC
 				);
-			if (typeof libraryID != 'undefined') {
+				if (libraryID !== false) {
 					sql += " AND libraryID=?";
 					params.push(libraryID == 0 ? null : libraryID);
 				}
@@ -724,15 +783,15 @@ Zotero.Sync.Storage = new function () {
 			// we don't need to do anything
 			if (!rows.length) {
 				var msg = "No in-sync or to-upload files found";
-			if (typeof libraryID != 'undefined') {
+				if (libraryID !== false) {
 					msg += " in library " + libraryID;
 				}
 				Zotero.debug(msg);
-			return Q(changed);
+				throw new Task.Result(changed);
 			}
 			
 			// Index attachment data by item id
-		var itemIDs = [];
+			itemIDs = [];
 			var attachmentData = {};
 			for each(let row in rows) {
 				var id = row.itemID;
@@ -747,11 +806,19 @@ Zotero.Sync.Storage = new function () {
 			}
 			rows = null;
 			
+			var t = new Date();
+			var items = Zotero.Items.get(itemIDs);
+			var numItems = items.length;
+			var updatedStates = {};
+			
 			// OS.File didn't work reliably before Firefox 23, so use the old code
 			if (Zotero.platformMajorVersion < 23) {
-			var updatedStates = {};
+				Zotero.debug("Performing synchronous file update check");
-			var items = Zotero.Items.get(itemIDs);
+				
 				for each(var item in items) {
+					// Spin the event loop during synchronous file access
+					yield Q.delay(1);
+					
 					Zotero.debug("Memory usage: " + memmgr.resident);
 					
 					let row = attachmentData[item.id];
@@ -858,14 +925,13 @@ Zotero.Sync.Storage = new function () {
 					Zotero.debug("No synced files have changed locally");
 				}
 				
-			return Q(changed);
+				Zotero.debug("Checked " + numItems + " files in " + (new Date() - t) + "ms");
+				
+				throw new Task.Result(changed);
 			}
 			
 			Components.utils.import("resource://gre/modules/osfile.jsm")
 			
-		var updatedStates = {};
-		var items = Zotero.Items.get(itemIDs);
-		
 			let checkItems = function () {
 				if (!items.length) return;
 				
@@ -997,7 +1063,7 @@ Zotero.Sync.Storage = new function () {
 				});
 			};
 			
-		return checkItems()
+			throw new Task.Result(checkItems()
 			.then(function () {
 				for (let itemID in updatedStates) {
 					Zotero.Sync.Storage.setSyncState(itemID, updatedStates[itemID]);
@@ -1008,9 +1074,12 @@ Zotero.Sync.Storage = new function () {
 					Zotero.debug("No synced files have changed locally");
 				}
 				
+				Zotero.debug("Checked " + numItems + " files in " + (new Date() - t) + "ms");
+				
 				return changed;
-		});
+			}));
-	}
+		}));
+	};
 	
 	
 	/**
@@ -1310,6 +1379,20 @@ Zotero.Sync.Storage = new function () {
 	}
 	
 	
+	this.notify = function(event, type, ids, extraData) {
+		if (event == 'open' && type == 'file') {
+			let timestamp = new Date().getTime();
+			
+			for each(let id in ids) {
+				_uploadCheckFiles.push({
+					itemID: id,
+					timestamp: timestamp
+				});
+			}
+		}
+	}
+	
+	
 	//
 	// Private methods
 	//
@@ -1940,6 +2023,36 @@ Zotero.Sync.Storage = new function () {
 	}
 	
 	
+	/**
+	 * Get files to check for local modifications for uploading
+	 *
+	 * This includes files previously modified and files opened externally
+	 * via Zotero within _maxCheckAgeInSeconds.
+	 */
+	function _getFilesToCheck(libraryID) {
+		var minTime = new Date().getTime() - (_maxCheckAgeInSeconds * 1000);
+		
+		// Get files by modification time
+		var sql = "SELECT itemID FROM itemAttachments JOIN items USING (itemID) "
+			+ "WHERE libraryID=? AND linkMode IN (?,?) AND syncState IN (?) AND "
+			+ "storageModTime>=?";
+		var params = [
+			libraryID == 0 ? null : libraryID,
+			Zotero.Attachments.LINK_MODE_IMPORTED_FILE,
+			Zotero.Attachments.LINK_MODE_IMPORTED_URL,
+			Zotero.Sync.Storage.SYNC_STATE_IN_SYNC,
+			minTime
+		];
+		var itemIDs = Zotero.DB.columnQuery(sql, params) || [];
+		
+		// Get files by open time
+		_uploadCheckFiles.filter(function (x) x.timestamp >= minTime);
+		itemIDs = itemIDs.concat([x.itemID for each(x in _uploadCheckFiles)])
+		
+		return Zotero.Utilities.arrayUnique(itemIDs);
+	}
+	
+	
 	/**
 	 * @inner
 	 * @return	{String[]|FALSE}			Array of keys, or FALSE if none

chrome/content/zotero/xpcom/sync.js

@@ -520,6 +520,7 @@ Zotero.Sync.Runner = new function () {
 	var _autoSyncTimer;
 	var _queue;
 	var _background;
+	var _firstInSession = true;
 	
 	var _lastSyncStatus;
 	var _currentSyncStatusLabel;
@@ -533,7 +534,13 @@ Zotero.Sync.Runner = new function () {
 		this.IdleListener.init();
 	}
 	
-	this.sync = function (background) {
+	this.sync = function (options) {
+		if (!options) options = {};
+		if (_firstInSession) {
+			options.firstInSession = true;
+			_firstInSession = false;
+		}
+		
 		_warning = null;
 		
 		if (Zotero.HTTP.browserIsOffline()){
@@ -549,7 +556,7 @@ Zotero.Sync.Runner = new function () {
 		// Purge deleted objects so they don't cause sync errors (e.g., long tags)
 		Zotero.purgeDataObjects(true);
 		
-		_background = !!background;
+		_background = !!options.background;
 		this.setSyncIcon('animate');
 		
 		var finalCallbacks = {
@@ -562,7 +569,7 @@ Zotero.Sync.Runner = new function () {
 		var storageSync = function () {
 			Zotero.Sync.Runner.setSyncStatus(Zotero.getString('sync.status.syncingFiles'));
 			
-			Zotero.Sync.Storage.sync()
+			Zotero.Sync.Storage.sync(options)
 			.then(function (results) {
 				Zotero.debug("File sync is finished");
 				
@@ -692,7 +699,9 @@ Zotero.Sync.Runner = new function () {
 					return;
 				}
 				
-				Zotero.Sync.Runner.sync(background);
+				Zotero.Sync.Runner.sync({
+					background: background
+				});
 			}
 		}
 		
@@ -1143,8 +1152,10 @@ Zotero.Sync.Runner.IdleListener = {
 		
 		Zotero.debug("Beginning idle sync");
 		
-		Zotero.Sync.Runner.sync(true);
+		Zotero.Sync.Runner.sync({
-		Zotero.Sync.Runner.setSyncTimeout(this._idleTimeout, true);
+			background: true
+		});
+		Zotero.Sync.Runner.setSyncTimeout(this._idleTimeout, true, true);
 	},
 	
 	_backObserver: {
@@ -1160,7 +1171,9 @@ Zotero.Sync.Runner.IdleListener = {
 				return;
 			}
 			Zotero.debug("Beginning return-from-idle sync");
-			Zotero.Sync.Runner.sync(true);
+			Zotero.Sync.Runner.sync({
+				background: true
+			});
 		}
 	},
 	
@@ -2021,7 +2034,9 @@ Zotero.Sync.Server = new function () {
 						
 						Zotero.Sync.Server.resetClient();
 						Zotero.Sync.Server.canAutoResetClient = false;
-						Zotero.Sync.Runner.sync(background);
+						Zotero.Sync.Runner.sync({
+							background: background
+						});
 					}, 1);
 					break;
 				
@@ -2124,7 +2139,9 @@ Zotero.Sync.Server = new function () {
 							Zotero.Sync.Server.canAutoResetClient = false;
 						}
 						
-						Zotero.Sync.Runner.sync(background);
+						Zotero.Sync.Runner.sync({
+							background: background
+						});
 					}, 1);
 					break;
 				
@@ -2359,7 +2376,9 @@ Zotero.Sync.Server = new function () {
 						}
 						Zotero.Sync.Server.resetClient();
 						Zotero.Sync.Server.canAutoResetClient = false;
-						Zotero.Sync.Runner.sync(background);
+						Zotero.Sync.Runner.sync({
+							background: background
+						});
 					}, 1);
 					break;
 				
@@ -3430,7 +3449,7 @@ Zotero.Sync.Server.Data = new function() {
 			// Check mod times and hashes of updated items against stored values to see
 			// if they've been updated elsewhere and mark for download if so
 			if (type == 'item' && Object.keys(itemStorageModTimes).length) {
-				yield Zotero.Sync.Storage.checkForUpdatedFiles(itemStorageModTimes);
+				yield Zotero.Sync.Storage.checkForUpdatedFiles(null, null, itemStorageModTimes);
 			}
 		}
 		

chrome/content/zotero/zoteroPane.js

@@ -433,7 +433,9 @@ var ZoteroPane = new function()
 					return;
 				}
 				
-				Zotero.Sync.Runner.sync(true);
+				Zotero.Sync.Runner.sync({
+					background: true
+				});
 			})
 			.done();
 		}
@@ -3497,6 +3499,7 @@ var ZoteroPane = new function()
 					this.loadURI(url, event);
 				}
 				else {
+					Zotero.Notifier.trigger('open', 'file', itemID);
 					Zotero.launchFile(file);
 				}
 			}
@@ -3618,6 +3621,7 @@ var ZoteroPane = new function()
 					var parent = file.parent.QueryInterface(Components.interfaces.nsILocalFile);
 					Zotero.launchFile(parent);
 				}
+				Zotero.Notifier.trigger('open', 'file', attachment.id);
 			}
 			else {
 				this.showAttachmentNotFoundDialog(attachment.id, noLocateOnMissing)