Full-text content syncing

Full-text content is now synced to the server and other clients. Up to
~500K of content is transferred with each sync, so multiple syncs may be
necessary. Downloaded content is written to temporary files and
processed on idle (currently 5 seconds), so synced content may not be
immediately available.

Performance will improve with API syncing and async DB in 4.1.

This also includes some other full-text tweaks, such as storing
text converted from HTML in cache files, which may speed up phrase
searching.

chrome/content/zotero/xpcom/attachments.js

@@ -567,11 +567,9 @@ Zotero.Attachments = new function(){
 			var f = function() {
 				if (mimeType == 'application/pdf') {
 					Zotero.Fulltext.indexPDF(file, itemID);
-					Zotero.Notifier.trigger('refresh', 'item', itemID);
 				}
-				if (Zotero.MIME.isTextType(mimeType)) {
+				else if (Zotero.MIME.isTextType(mimeType)) {
 					Zotero.Fulltext.indexDocument(document, itemID);
-					Zotero.Notifier.trigger('refresh', 'item', itemID);
 				}
 				if (callback) {
 					callback(attachmentItem);
@@ -981,7 +979,7 @@ Zotero.Attachments = new function(){
 	
 	function getStorageDirectory(itemID) {
 		if (!itemID) {
-			throw ("itemID not provided in Zotero.Attachments.getStorageDirectory()");
+			throw new Error("itemID not provided in Zotero.Attachments.getStorageDirectory()");
 		}
 		var item = Zotero.Items.get(itemID);
 		if (!item) {

chrome/content/zotero/xpcom/fulltext.js

@@ -24,7 +24,6 @@
 */
 
 Zotero.Fulltext = new function(){
-	const FULLTEXT_VERSION = 1;
 	const CACHE_FILE = '.zotero-ft-cache';
 	
 	this.init = init;
@@ -38,7 +37,6 @@ Zotero.Fulltext = new function(){
 	this.indexFile = indexFile;
 	this.indexPDF = indexPDF;
 	this.indexItems = indexItems;
-	this.findTextInFile = findTextInFile;
 	this.findTextInItems = findTextInItems;
 	this.clearItemWords = clearItemWords;
 	this.getPages = getPages;
@@ -72,6 +70,7 @@ Zotero.Fulltext = new function(){
 	this.__defineGetter__("INDEX_STATE_PARTIAL", function () { return 2; });
 	this.__defineGetter__("INDEX_STATE_INDEXED", function () { return 3; });
 	
+	const _processorCacheFile = '.zotero-ft-unprocessed';
 	
 	var _pdfConverterVersion = null;
 	var _pdfConverterFileName = null;
@@ -80,6 +79,16 @@ Zotero.Fulltext = new function(){
 	var _pdfInfoFileName = null;
 	var _pdfInfo = null; // nsIFile to executable
 	
+	var _idleObserverIsRegistered = false;
+	var _idleObserverDelay = 5;
+	var _processorTimer = null;
+	var _upgradeCheck = true;
+	
+	const SYNC_STATE_UNSYNCED = 0;
+	const SYNC_STATE_IN_SYNC = 1;
+	const SYNC_STATE_TO_PROCESS = 2;
+	const SYNC_STATE_TO_DOWNLOAD = 3;
+	
 	var self = this;
 	
 	function init() {
@@ -98,6 +107,18 @@ Zotero.Fulltext = new function(){
 		
 		this.registerPDFTool('converter');
 		this.registerPDFTool('info');
+		
+		// TEMP: Remove after 4.1 DB schema change
+		var cols = Zotero.DB.getColumns('fulltextItems');
+		if (cols.indexOf("synced") == -1) {
+			Zotero.DB.beginTransaction();
+			Zotero.DB.query("ALTER TABLE fulltextItems ADD COLUMN synced INT DEFAULT 0");
+			Zotero.DB.query("REPLACE INTO settings (setting, key, value) VALUES ('fulltext', 'downloadAll', 1)");
+			Zotero.DB.commitTransaction();
+		}
+		
+		this.startContentProcessor();
+		Zotero.addShutdownListener(this.stopContentProcessor);
 	}
 	
 	
@@ -192,7 +213,7 @@ Zotero.Fulltext = new function(){
 	/*
 	 * Index multiple words at once
 	 */
-	function indexWords(itemID, words){
+	function indexWords(itemID, words) {
 		if (!words || !words.length || !itemID){
 			return false;
 		}
@@ -231,9 +252,10 @@ Zotero.Fulltext = new function(){
 		}
 		while (done < numWords);
 		
-		Zotero.DB.query("REPLACE INTO fulltextItems (itemID, version) VALUES (?,?)",
+		if (!Zotero.DB.valueQuery("SELECT COUNT(*) FROM fulltextItems WHERE itemID=?", itemID)) {
-			[itemID, FULLTEXT_VERSION]);
+			let sql = "INSERT INTO fulltextItems (itemID, version) VALUES (?,?)";
-		
+			Zotero.DB.query(sql, [itemID, 0]);
+		}
 		
 		// Handle bound parameters manually for optimal speed
 		var statement1 = Zotero.DB.getStatement("INSERT INTO fulltextWords (word) VALUES (?)");
@@ -266,7 +288,7 @@ Zotero.Fulltext = new function(){
 	}
 	
 	
-	function indexString(text, charset, itemID){
+	function indexString(text, charset, itemID, stats, version, synced) {
 		try {
 			Zotero.UnresponsiveScriptIndicator.disable();
 			
@@ -275,7 +297,23 @@ Zotero.Fulltext = new function(){
 			Zotero.DB.beginTransaction();
 			
 			this.clearItemWords(itemID, true);
-			this.indexWords(itemID, words);
+			this.indexWords(itemID, words, stats, version, synced);
+			
+			var sql = "UPDATE fulltextItems SET synced=?";
+			var params = [synced ? parseInt(synced) : SYNC_STATE_UNSYNCED];
+			if (stats) {
+				for (let stat in stats) {
+					sql += ", " + stat + "=?";
+					params.push(stats[stat] ? parseInt(stats[stat]) : null);
+				}
+			}
+			if (version) {
+				sql += ", version=?";
+				params.push(parseInt(version));
+			}
+			sql += " WHERE itemID=?";
+			params.push(itemID);
+			Zotero.DB.query(sql, params);
 			
 			/*
 			var sql = "REPLACE INTO fulltextContent (itemID, textContent) VALUES (?,?)";
@@ -283,6 +321,14 @@ Zotero.Fulltext = new function(){
 			*/
 			
 			Zotero.DB.commitTransaction();
+			
+			// If there's a processor cache file, delete it (whether or not we just used it)
+			var cacheFile = this.getItemProcessorCacheFile(itemID);
+			if (cacheFile.exists()) {
+				cacheFile.remove(false);
+			}
+			
+			Zotero.Notifier.trigger('refresh', 'item', itemID);
 		}
 		finally {
 			Zotero.UnresponsiveScriptIndicator.enable();
@@ -499,7 +545,7 @@ Zotero.Fulltext = new function(){
 	
 	
 	function indexItems(items, complete, ignoreErrors) {
-		if (items.constructor.name != 'Array') {
+		if (Array.isArray(items)) {
 			items = [items];
 		}
 		var items = Zotero.Items.get(items);
@@ -507,20 +553,22 @@ Zotero.Fulltext = new function(){
 		
 		Zotero.DB.beginTransaction();
 		
-		for each(var i in items){
+		for each (let item in items) {
-			if (!i.isAttachment()){
+			if (!item.isAttachment()) {
 				continue;
 			}
 			
-			var file = i.getFile();
+			let itemID = item.id;
+			
+			var file = item.getFile();
 			if (!file){
-				Zotero.debug("No file to index for item " + i.id + " in Fulltext.indexItems()");
+				Zotero.debug("No file to index for item " + itemID + " in Fulltext.indexItems()");
 				continue;
 			}
 			
 			if (ignoreErrors) {
 				try {
-					this.indexFile(file, i.attachmentMIMEType, i.attachmentCharset, i.id, !complete);
+					this.indexFile(file, item.attachmentMIMEType, item.attachmentCharset, itemID, !complete);
 				}
 				catch (e) {
 					Zotero.debug(e, 1);
@@ -529,7 +577,7 @@ Zotero.Fulltext = new function(){
 				}
 			}
 			else {
-				this.indexFile(file, i.attachmentMIMEType, i.attachmentCharset, i.id, !complete);
+				this.indexFile(file, item.attachmentMIMEType, item.attachmentCharset, itemID, !complete);
 			}
 		}
 		
@@ -537,8 +585,356 @@ Zotero.Fulltext = new function(){
 	}
 	
 	
+	//
+	// Full-text content syncing
+	//
+	/**
+	 * Get content and stats that haven't yet been synced
+	 *
+	 * @param {Integer} maxChars  Maximum total characters to include.
+	 *                            The total can go over this if there's a
+	 *                            single large item.
+	 * @return {Array<Object>}
+	 */
+	this.getUnsyncedContent = function (maxChars) {
+		var first = true;
+		var chars = 0;
+		var contentItems = [];
+		var sql = "SELECT itemID, indexedChars, totalChars, indexedPages, totalPages "
+			+ "FROM fulltextItems WHERE synced=" + SYNC_STATE_UNSYNCED;
+		var rows = Zotero.DB.query(sql) || [];
+		for each (let row in rows) {
+			let text;
+			let itemID = row.itemID;
+			let item = Zotero.Items.get(itemID);
+			let libraryKey = item.libraryID + "/" + item.key;
+			let mimeType = item.attachmentMIMEType;
+			if (isCachedMIMEType(mimeType) || Zotero.MIME.isTextType(mimeType)) {
+				try {
+					let cacheFile = this.getItemCacheFile(itemID);
+					if (cacheFile.exists()) {
+						Zotero.debug("Adding full-text content from cache "
+							+ "file for item " + libraryKey);
+						text = Zotero.File.getContents(cacheFile);
+					}
+					else {
+						if (!Zotero.MIME.isTextType(mimeType)) {
+							Zotero.debug("Full-text content cache file doesn't exist for item "
+								+ libraryKey, 2);
+							continue;
+						}
+						
+						let file = item.getFile();
+						if (!file) {
+							Zotero.debug("File doesn't exist getting full-text content for item "
+								+ libraryKey, 2);
+							continue;
+						}
+						
+						Zotero.debug("Adding full-text content from file for item " + libraryKey);
+						text = Zotero.File.getContents(
+							file, item.attachmentCharset, row.indexedChars
+						);
+						
+						// Split elements to avoid word concatentation
+						if (item.attachmentMIMEType == 'text/html') {
+							text = text.replace(/(>)/g, '$1 ');
+							text = this.HTMLToText(text);
+							
+							// Write the converted text to a cache file
+							Zotero.debug("Writing converted full-text HTML content to "
+								+ cacheFile.path);
+							if (!cacheFile.parent.exists()) {
+								Zotero.Attachments.createDirectoryForItem(itemID);
+							}
+							Zotero.File.putContentsAsync(cacheFile, text)
+							.catch(function (e) {
+								Zotero.debug(e, 1);
+								Components.utils.reportError(e);
+							})
+						}
+					}
+				}
+				catch (e) {
+					Zotero.debug(e, 1);
+					Components.utils.reportError(e);
+					continue;
+				}
+			}
+			else {
+				Zotero.debug("Skipping non-text file getting full-text content for item "
+					+ libraryKey, 2);
+				
+				// Delete rows for items that weren't supposed to be indexed
+				this.clearItemWords(itemID);
+				continue;
+			}
+			
+			// If this isn't the first item and it would put us over the limit,
+			// skip it
+			if (!first && maxChars && ((chars + text.length) > maxChars)) {
+				continue;
+			}
+			chars += text.length;
+			first = false;
+			
+			contentItems.push({
+				libraryID: item.libraryID,
+				key: item.key,
+				text: text,
+				indexedChars: row.indexedChars ? row.indexedChars : 0,
+				totalChars: row.totalChars ? row.totalChars : 0,
+				indexedPages: row.indexedPages ? row.indexedPages : 0,
+				totalPages: row.totalPages ? row.totalPages : 0
+			});
+			
+			if (maxChars && chars > maxChars) {
+				break;
+			}
+		}
+		return contentItems;
+	}
+	
+	
+	/**
+	 * @return {String}  PHP-formatted POST data for items not yet downloaded
+	 */
+	this.getUndownloadedPostData = function () {
+		// On upgrade, get all content
+		var sql = "SELECT value FROM settings WHERE setting='fulltext' AND key='downloadAll'";
+		if (Zotero.DB.valueQuery(sql)) {
+			return "&ftkeys=all";
+		}
+		
+		var sql = "SELECT itemID FROM fulltextItems WHERE synced="
+			+ SYNC_STATE_TO_DOWNLOAD;
+		var itemIDs = Zotero.DB.columnQuery(sql);
+		if (!itemIDs) {
+			return "";
+		}
+		var undownloaded = {};
+		for each (let itemID in itemIDs) {
+			let item = Zotero.Items.get(itemID);
+			let libraryID = item.libraryID
+			libraryID = libraryID ? libraryID : Zotero.libraryID;
+			if (!undownloaded[libraryID]) {
+				undownloaded[libraryID] = [];
+			}
+			undownloaded[libraryID].push(item.key);
+		}
+		var data = "";
+		for (let libraryID in undownloaded) {
+			for (let i = 0; i < undownloaded[libraryID].length; i++) {
+				data += "&" + encodeURIComponent("ftkeys[" + libraryID + "][" + i + "]")
+					+ "=" + undownloaded[libraryID][i];
+			}
+		}
+		return data;
+	}
+	
+	
+	/**
+	 * Save full-text content and stats to a cache file
+	 */
+	this.setItemContent = function (libraryID, key, text, stats, version) {
+		var item = Zotero.Items.getByLibraryAndKey(libraryID, key);
+		if (!item) {
+			let msg = "Item not found setting full-text content";
+			Zotero.debug(msg, 1);
+			Components.utils.reportError(msg);
+			return;
+		}
+		var itemID = item.id;
+		
+		if (text !== '') {
+			var cacheFile = this.getItemProcessorCacheFile(itemID);
+			
+			// If a storage directory doesn't exist, create it
+			if (!cacheFile.parent.exists()) {
+				Zotero.Attachments.createDirectoryForItem(itemID);
+			}
+			
+			Zotero.debug("Writing full-text content and data to " + cacheFile.path);
+			Zotero.File.putContents(cacheFile, JSON.stringify({
+				indexedChars: stats.indexedChars,
+				totalChars: stats.totalChars,
+				indexedPages: stats.indexedPages,
+				totalPages: stats.totalPages,
+				version: version,
+				text: text
+			}));
+			var synced = SYNC_STATE_TO_PROCESS;
+		}
+		else {
+			Zotero.debug("Marking full-text content for download");
+			var synced = SYNC_STATE_TO_DOWNLOAD;
+		}
+		
+		// Mark the item as unprocessed
+		if (Zotero.DB.valueQuery("SELECT COUNT(*) FROM fulltextItems WHERE itemID=?", itemID)) {
+			Zotero.DB.query("UPDATE fulltextItems SET synced=? WHERE itemID=?", [synced, itemID]);
+		}
+		// If not yet indexed, add an empty row
+		else {
+			Zotero.DB.query(
+				"REPLACE INTO fulltextItems (itemID, version, synced) VALUES (?, 0, ?)",
+				[itemID, synced]
+			);
+		}
+		
+		if (_upgradeCheck) {
+			Zotero.DB.query("DELETE FROM settings WHERE setting='fulltext' AND key='downloadAll'");
+			_upgradeCheck = false;
+		}
+		
+		this.startContentProcessor();
+	}
+	
+	
+	/**
+	 * Start the idle observer for the background content processor
+	 */
+	this.startContentProcessor = function () {
+		if (!_idleObserverIsRegistered) {
+			Zotero.debug("Initializing full-text content ingester idle observer");
+			var idleService = Components.classes["@mozilla.org/widget/idleservice;1"]
+					.getService(Components.interfaces.nsIIdleService);
+			idleService.addIdleObserver(this.idleObserver, _idleObserverDelay);
+			_idleObserverIsRegistered = true;
+		}
+	}
+	
+	/**
+	 * Stop the idle observer and a running timer, if there is one
+	 */
+	this.stopContentProcessor = function () {
+		if (_idleObserverIsRegistered) {
+			var idleService = Components.classes["@mozilla.org/widget/idleservice;1"]
+				.getService(Components.interfaces.nsIIdleService);
+			idleService.removeIdleObserver(this.idleObserver, _idleObserverDelay);
+			_idleObserverIsRegistered = false;
+		}
+		
+		if (_processorTimer) {
+			_processorTimer.cancel();
+			_processorTimer = null;
+		}
+	}
+	
+	/**
+	 *
+	 * @param {Array<Integer>} itemIDs  An array of itemIDs to process; if this
+	 *                                  is omitted, a database query is made
+	 *                                  to find unprocessed content
+	 * @return {Boolean}  TRUE if there's more content to process; FALSE otherwise
+	 */
+	this.processUnprocessedContent = function (itemIDs) {
+		if (!itemIDs) {
+			Zotero.debug("Checking for unprocessed full-text content");
+			let sql = "SELECT itemID FROM fulltextItems WHERE synced="
+				+ SYNC_STATE_TO_PROCESS;
+			itemIDs = Zotero.DB.columnQuery(sql) || [];
+		}
+		// If there's no more unprocessed content, stop the idle observer
+		if (!itemIDs.length) {
+			Zotero.debug("No unprocessed full-text content found");
+			this.stopContentProcessor();
+			return;
+		}
+		
+		let itemID = itemIDs.shift();
+		let item = Zotero.Items.get(itemID);
+		
+		Zotero.debug("Processing full-text content for item " + item.libraryKey);
+		
+		Zotero.Fulltext.indexFromProcessorCache(itemID)
+		.then(function () {
+			if (itemIDs.length) {
+				if (!_processorTimer) {
+					_processorTimer = Components.classes["@mozilla.org/timer;1"]
+						.createInstance(Components.interfaces.nsITimer);
+				}
+				_processorTimer.initWithCallback(
+					function () {
+						Zotero.Fulltext.processUnprocessedContent(itemIDs);
+					},
+					100,
+					Components.interfaces.nsITimer.TYPE_ONE_SHOT
+				);
+			}
+		})
+		.done();
+	}
+	
+	this.idleObserver = {
+		observe: function (subject, topic, data) {
+			// On idle, start the background processor
+			if (topic == 'idle') {
+				Zotero.Fulltext.processUnprocessedContent();
+			}
+			// When back from idle, stop the processor (but keep the idle
+			// observer registered)
+			else if (topic == 'active') {
+				if (_processorTimer) {
+					Zotero.debug("Stopping full-text content processor");
+					_processorTimer.cancel();
+				}
+			}
+		}
+	};
+	
+	
+	this.indexFromProcessorCache = function (itemID) {
+		var self = this;
+		return Q.fcall(function () {
+			var cacheFile = self.getItemProcessorCacheFile(itemID);
+			if (!cacheFile.exists())  {
+				Zotero.debug("Full-text content processor cache file doesn't exist for item " + itemID);
+				return false;
+			}
+			
+			let data;
+			
+			return Zotero.File.getContentsAsync(cacheFile)
+			.then(function (json) {
+				data = JSON.parse(json);
+				
+				// Write the text content to the regular cache file
+				cacheFile = self.getItemCacheFile(itemID);
+				
+				Zotero.debug("Writing full-text content to " + cacheFile.path);
+				return Zotero.File.putContentsAsync(cacheFile, data.text, "UTF-8");
+			})
+			.then(function () {
+				Zotero.Fulltext.indexString(
+					data.text,
+					"UTF-8",
+					itemID,
+					{
+						indexedChars: data.indexedChars,
+						totalChars: data.totalChars,
+						indexedPages: data.indexedPages,
+						totalPages: data.totalPages
+					},
+					data.version,
+					1
+				);
+			});
+		})
+		.catch(function (e) {
+			Components.utils.reportError(e);
+			Zotero.debug(e, 1);
+			return false;
+		});
+	}
+	
+	//
+	// End full-text content syncing
+	//
+	
+	
 	/*
-	 * Scan a file for a text string
+	 * Scan a string for another string
 	 *
 	 * _items_ -- one or more attachment items to search
 	 * _searchText_ -- text pattern to search for
@@ -548,21 +944,7 @@ Zotero.Fulltext = new function(){
 	 *
 	 * - Slashes in regex are optional
 	 */
-	function findTextInFile(file, charset, searchText, mode){
+	this.findTextInString = function (content, searchText, mode) {
-		Zotero.debug("Searching for text '" + searchText + "' in " + file.path);
-		
-		var maxLength = Zotero.Prefs.get('fulltext.textMaxLength');
-		var str = Zotero.File.getContents(file, charset, maxLength);
-		
-		// If not binary mode, convert HTML to text
-		if (!mode || mode.indexOf('Binary')==-1){
-			// Split elements to avoid word concatentation
-			str = str.replace(/(>)/g, '$1 ');
-			
-			// Parse to avoid searching on HTML
-			str = this.HTMLToText(str);
-		}
-		
 		switch (mode){
 			case 'regexp':
 			case 'regexpCS':
@@ -583,7 +965,7 @@ Zotero.Fulltext = new function(){
 				
 				try {
 					var re = new RegExp(searchText, flags);
-					var matches = re.exec(str);
+					var matches = re.exec(content);
 				}
 				catch (e) {
 					Zotero.debug(e, 1);
@@ -591,7 +973,7 @@ Zotero.Fulltext = new function(){
 				}
 				if (matches){
 					Zotero.debug("Text found");
-					return str.substr(matches.index, 50);
+					return content.substr(matches.index, 50);
 				}
 				
 				break;
@@ -599,12 +981,12 @@ Zotero.Fulltext = new function(){
 			default:
 				// Case-insensitive
 				searchText = searchText.toLowerCase();
-				str = str.toLowerCase();
+				content = content.toLowerCase();
 				
-				var pos = str.indexOf(searchText);
+				var pos = content.indexOf(searchText);
 				if (pos!=-1){
 					Zotero.debug('Text found');
-					return str.substr(pos, 50);
+					return content.substr(pos, 50);
 				}
 		}
 		
@@ -633,42 +1015,78 @@ Zotero.Fulltext = new function(){
 		var items = Zotero.Items.get(items);
 		var found = [];
 		
-		for each(var i in items){
+		for each (let item in items) {
-			if (!i.isAttachment()){
+			if (!item.isAttachment()) {
 				continue;
 			}
 			
-			var file = i.getFile();
+			let itemID = item.id;
-			if (!file){
+			let content;
-				continue;
+			let mimeType = item.attachmentMIMEType;
-			}
+			let maxLength = Zotero.Prefs.get('fulltext.textMaxLength');
+			let binaryMode = mode && mode.indexOf('Binary') != -1;
 			
-			var mimeType = i.attachmentMIMEType;
 			if (isCachedMIMEType(mimeType)) {
-				var file = this.getItemCacheFile(i.id);
+				let file = this.getItemCacheFile(itemID);
 				if (!file.exists()) {
 					continue;
 				}
 				
-				mimeType = 'text/plain';
+				Zotero.debug("Searching for text '" + searchText + "' in " + file.path);
-				var charset = 'utf-8';
+				content = Zotero.File.getContents(file, 'utf-8', maxLength);
 			}
 			else {
 				// If not binary mode, only scan plaintext files
-				if (!mode || mode.indexOf('Binary') == -1) {
+				if (!binaryMode) {
 					if (!Zotero.MIME.isTextType(mimeType)) {
 						Zotero.debug('Not scanning MIME type ' + mimeType, 4);
 						continue;
 					}
 				}
 				
-				var charset = i.attachmentCharset;
+				// Check for a cache file
+				let cacheFile = this.getItemCacheFile(itemID);
+				if (cacheFile.exists()) {
+					Zotero.debug("Searching for text '" + searchText + "' in " + cacheFile.path);
+					content = Zotero.File.getContents(cacheFile, 'utf-8', maxLength);
+				}
+				else {
+					// If that doesn't exist, check for the actual file
+					let file = item.getFile();
+					if (!file) {
+						continue;
+					}
+					
+					Zotero.debug("Searching for text '" + searchText + "' in " + file.path);
+					content = Zotero.File.getContents(file, item.attachmentCharset, maxLength);
+					
+					// If HTML and not binary mode, convert to text
+					if (mimeType == 'text/html' && !binaryMode) {
+						// Split elements to avoid word concatentation
+						content = content.replace(/(>)/g, '$1 ');
+						
+						content = this.HTMLToText(content);
+						
+						// Write the converted text to a cache file for future searches
+						Zotero.debug("Writing converted full-text content to " + cacheFile.path);
+						if (!cacheFile.parent.exists()) {
+							Zotero.Attachments.createDirectoryForItem(itemID);
+						}
+						Zotero.File.putContentsAsync(cacheFile, content)
+						.catch(function (e) {
+							Zotero.debug(e, 1);
+							Components.utils.reportError(e);
+						})
+					}
+				}
 			}
 			
-			var match = this.findTextInFile(file, charset, searchText, mode);
+			let match = this.findTextInString(content, searchText, mode);
-			
+			if (match != -1) {
-			if (match != -1){
+				found.push({
-				found.push({id:i.getID(), match:match});
+					id: itemID,
+					match: match
+				});
 			}
 		}
 		
@@ -681,8 +1099,8 @@ Zotero.Fulltext = new function(){
 		var sql = "SELECT rowid FROM fulltextItems WHERE itemID=? LIMIT 1";
 		var indexed = Zotero.DB.valueQuery(sql, itemID);
 		if (indexed) {
-			Zotero.DB.query("DELETE FROM fulltextItems WHERE itemID=?", itemID);
 			Zotero.DB.query("DELETE FROM fulltextItemWords WHERE itemID=?", itemID);
+			Zotero.DB.query("DELETE FROM fulltextItems WHERE itemID=?", itemID);
 		}
 		Zotero.DB.commitTransaction();
 		
@@ -760,15 +1178,27 @@ Zotero.Fulltext = new function(){
 	
 	function setPages(itemID, obj) {
 		var sql = "UPDATE fulltextItems SET indexedPages=?, totalPages=? WHERE itemID=?";
-		Zotero.DB.query(sql, [obj.indexed ? obj.indexed : null,
+		Zotero.DB.query(
-			obj.total ? obj.total : null, itemID]);
+			sql,
+			[
+				obj.indexed ? parseInt(obj.indexed) : null,
+				obj.total ? parseInt(obj.total) : null,
+				itemID
+			]
+		);
 	}
 	
 	
 	function setChars(itemID, obj) {
 		var sql = "UPDATE fulltextItems SET indexedChars=?, totalChars=? WHERE itemID=?";
-		Zotero.DB.query(sql, [obj.indexed ? obj.indexed : null,
+		Zotero.DB.query(
-			obj.total ? obj.total : null, itemID]);
+			sql,
+			[
+				obj.indexed ? parseInt(obj.indexed) : null,
+				obj.total ? parseInt(obj.total) : null,
+				itemID
+			]
+		);
 	}
 	
 	
@@ -878,6 +1308,13 @@ Zotero.Fulltext = new function(){
 	}
 	
 	
+	this.getItemProcessorCacheFile = function (itemID) {
+		var cacheFile = Zotero.Attachments.getStorageDirectory(itemID);
+		cacheFile.append(_processorCacheFile);
+		return cacheFile;
+	}
+	
+	
 	/*
 	 * Returns true if an item can be reindexed
 	 *
@@ -971,18 +1408,14 @@ Zotero.Fulltext = new function(){
 		}
 		
 		Zotero.debug('Clearing full-text cache file for item ' + itemID);
-		switch (item.attachmentMIMEType) {
+		var cacheFile = this.getItemCacheFile(itemID);
-			case 'application/pdf':
+		if (cacheFile.exists()) {
-				var cacheFile = this.getItemCacheFile(itemID);
+			try {
-				if (cacheFile.exists()) {
+				cacheFile.remove(false);
-					try {
+			}
-						cacheFile.remove(false);
+			catch (e) {
-					}
+				Zotero.File.checkFileAccessError(e, cacheFile, 'delete');
-					catch (e) {
+			}
-						Zotero.File.checkFileAccessError(e, cacheFile, 'delete');
-					}
-				}
-				break;
 		}
 	}
 	

chrome/content/zotero/xpcom/sync.js

@@ -55,6 +55,10 @@ Zotero.Sync = new function() {
 			setting: {
 				singular: 'Setting',
 				plural: 'Settings'
+			},
+			fulltext: {
+				singular: 'Fulltext',
+				plural: 'Fulltexts'
 			}
 		};
 	});
@@ -132,7 +136,7 @@ Zotero.Sync = new function() {
 		}
 		
 		for (var type in this.syncObjects) {
-			if (type == 'setting') {
+			if (type == 'setting' || type == 'fulltext') {
 				continue;
 			}
 			
@@ -451,6 +455,9 @@ Zotero.Sync.EventListener = new function () {
 				if (type == 'setting') {
 					[libraryID, key] = ids[i].split("/");
 				}
+				else if (type == 'fulltext') {
+					continue;
+				}
 				else {
 					var oldItem = extraData[ids[i]].old;
 					libraryID = oldItem.primary.libraryID;
@@ -1462,6 +1469,8 @@ Zotero.Sync.Server = new function () {
 			body += '&upload=1';
 		}
 		
+		body += Zotero.Fulltext.getUndownloadedPostData();
+		
 		Zotero.Sync.Runner.setSyncStatus(Zotero.getString('sync.status.gettingUpdatedData'));
 		
 		Zotero.HTTP.doPost(url, body, function (xmlhttp) {
@@ -1694,6 +1703,14 @@ Zotero.Sync.Server = new function () {
 							var sql = "UPDATE syncedSettings SET synced=1";
 							Zotero.DB.query(sql);
 							
+							if (syncSession.fulltextItems && syncSession.fulltextItems.length) {
+								let sql = "UPDATE fulltextItems SET synced=1 WHERE itemID=?";
+								for each (let lk in syncSession.fulltextItems) {
+									let item = Zotero.Items.getByLibraryAndKey(lk.libraryID, lk.key);
+									Zotero.DB.query(sql, item.id);
+								}
+							}
+							
 							//throw('break2');
 							
 							Zotero.DB.commitTransaction();
@@ -2825,6 +2842,26 @@ Zotero.Sync.Server.Data = new function() {
 					Zotero.SyncedSettings.setSynchronous(libraryID, name, value, version, true);
 					continue;
 				}
+				else if (type == 'fulltext') {
+					if (!libraryID) {
+						libraryID = 0;
+					}
+					let key = objectNode.getAttribute('key');
+					Zotero.debug("Processing remote full-text content for item " + libraryID + "/" + key);
+					Zotero.Fulltext.setItemContent(
+						libraryID,
+						key,
+						objectNode.textContent,
+						{
+							indexedChars: parseInt(objectNode.getAttribute('indexedChars')),
+							totalChars: parseInt(objectNode.getAttribute('totalChars')),
+							indexedPages: parseInt(objectNode.getAttribute('indexedPages')),
+							totalPages: parseInt(objectNode.getAttribute('totalPages'))
+						},
+						parseInt(objectNode.getAttribute('version'))
+					);
+					continue;
+				}
 				
 				var key = objectNode.getAttribute('key');
 				var objLibraryKeyHash = Zotero[Types].makeLibraryKeyHash(libraryID, key);
@@ -3537,6 +3574,36 @@ Zotero.Sync.Server.Data = new function() {
 			docElem.appendChild(settingsNode);
 		}
 		
+		// Add up to 500K characters of full-text content
+		try {
+			var rows = Zotero.Fulltext.getUnsyncedContent(500000);
+		}
+		catch (e) {
+			Zotero.debug(e, 1);
+			Components.utils.reportError(e);
+			var rows = [];
+		}
+		if (rows.length) {
+			let fulltextsNode = doc.createElement('fulltexts');
+			syncSession.fulltextItems = [];
+			for (let i=0; i<rows.length; i++) {
+				syncSession.fulltextItems.push({
+					libraryID: rows[i].libraryID,
+					key: rows[i].key
+				})
+				let node = doc.createElement('fulltext');
+				node.setAttribute('libraryID', rows[i].libraryID ? rows[i].libraryID : Zotero.libraryID);
+				node.setAttribute('key', rows[i].key);
+				node.setAttribute('indexedChars', rows[i].indexedChars);
+				node.setAttribute('totalChars', rows[i].totalChars);
+				node.setAttribute('indexedPages', rows[i].indexedPages);
+				node.setAttribute('totalPages', rows[i].totalPages);
+				node.appendChild(doc.createTextNode(_xmlize(rows[i].text)));
+				fulltextsNode.appendChild(node);
+			}
+			docElem.appendChild(fulltextsNode);
+		}
+		
 		// Deletions
 		var deletedNode = doc.createElement('deleted');
 		var inserted = false;

resource/schema/userdata.sql

@@ -271,6 +271,7 @@ CREATE TABLE fulltextItems (
     totalPages INT,
     indexedChars INT,
     totalChars INT,
+    synced INT DEFAULT 0,
     FOREIGN KEY (itemID) REFERENCES items(itemID)
 );
 CREATE INDEX fulltextItems_version ON fulltextItems(version);