From 62840f3b9370941b7c7d04f3385898141cc89db5 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Sun, 3 Nov 2013 19:14:10 -0500 Subject: [PATCH] Full-text content syncing Full-text content is now synced to the server and other clients. Up to ~500K of content is transferred with each sync, so multiple syncs may be necessary. Downloaded content is written to temporary files and processed on idle (currently 5 seconds), so synced content may not be immediately available. Performance will improve with API syncing and async DB in 4.1. This also includes some other full-text tweaks, such as storing text converted from HTML in cache files, which may speed up phrase searching. --- chrome/content/zotero/xpcom/attachments.js | 6 +- chrome/content/zotero/xpcom/fulltext.js | 571 ++++++++++++++++++--- chrome/content/zotero/xpcom/sync.js | 69 ++- resource/schema/userdata.sql | 1 + 4 files changed, 573 insertions(+), 74 deletions(-) diff --git a/chrome/content/zotero/xpcom/attachments.js b/chrome/content/zotero/xpcom/attachments.js index d62a7579b..2cb574c5a 100644 --- a/chrome/content/zotero/xpcom/attachments.js +++ b/chrome/content/zotero/xpcom/attachments.js @@ -567,11 +567,9 @@ Zotero.Attachments = new function(){ var f = function() { if (mimeType == 'application/pdf') { Zotero.Fulltext.indexPDF(file, itemID); - Zotero.Notifier.trigger('refresh', 'item', itemID); } - if (Zotero.MIME.isTextType(mimeType)) { + else if (Zotero.MIME.isTextType(mimeType)) { Zotero.Fulltext.indexDocument(document, itemID); - Zotero.Notifier.trigger('refresh', 'item', itemID); } if (callback) { callback(attachmentItem); @@ -981,7 +979,7 @@ Zotero.Attachments = new function(){ function getStorageDirectory(itemID) { if (!itemID) { - throw ("itemID not provided in Zotero.Attachments.getStorageDirectory()"); + throw new Error("itemID not provided in Zotero.Attachments.getStorageDirectory()"); } var item = Zotero.Items.get(itemID); if (!item) { diff --git a/chrome/content/zotero/xpcom/fulltext.js b/chrome/content/zotero/xpcom/fulltext.js index 9a21aeb11..6b983649b 100644 --- a/chrome/content/zotero/xpcom/fulltext.js +++ b/chrome/content/zotero/xpcom/fulltext.js @@ -24,7 +24,6 @@ */ Zotero.Fulltext = new function(){ - const FULLTEXT_VERSION = 1; const CACHE_FILE = '.zotero-ft-cache'; this.init = init; @@ -38,7 +37,6 @@ Zotero.Fulltext = new function(){ this.indexFile = indexFile; this.indexPDF = indexPDF; this.indexItems = indexItems; - this.findTextInFile = findTextInFile; this.findTextInItems = findTextInItems; this.clearItemWords = clearItemWords; this.getPages = getPages; @@ -72,6 +70,7 @@ Zotero.Fulltext = new function(){ this.__defineGetter__("INDEX_STATE_PARTIAL", function () { return 2; }); this.__defineGetter__("INDEX_STATE_INDEXED", function () { return 3; }); + const _processorCacheFile = '.zotero-ft-unprocessed'; var _pdfConverterVersion = null; var _pdfConverterFileName = null; @@ -80,6 +79,16 @@ Zotero.Fulltext = new function(){ var _pdfInfoFileName = null; var _pdfInfo = null; // nsIFile to executable + var _idleObserverIsRegistered = false; + var _idleObserverDelay = 5; + var _processorTimer = null; + var _upgradeCheck = true; + + const SYNC_STATE_UNSYNCED = 0; + const SYNC_STATE_IN_SYNC = 1; + const SYNC_STATE_TO_PROCESS = 2; + const SYNC_STATE_TO_DOWNLOAD = 3; + var self = this; function init() { @@ -98,6 +107,18 @@ Zotero.Fulltext = new function(){ this.registerPDFTool('converter'); this.registerPDFTool('info'); + + // TEMP: Remove after 4.1 DB schema change + var cols = Zotero.DB.getColumns('fulltextItems'); + if (cols.indexOf("synced") == -1) { + Zotero.DB.beginTransaction(); + Zotero.DB.query("ALTER TABLE fulltextItems ADD COLUMN synced INT DEFAULT 0"); + Zotero.DB.query("REPLACE INTO settings (setting, key, value) VALUES ('fulltext', 'downloadAll', 1)"); + Zotero.DB.commitTransaction(); + } + + this.startContentProcessor(); + Zotero.addShutdownListener(this.stopContentProcessor); } @@ -192,7 +213,7 @@ Zotero.Fulltext = new function(){ /* * Index multiple words at once */ - function indexWords(itemID, words){ + function indexWords(itemID, words) { if (!words || !words.length || !itemID){ return false; } @@ -231,9 +252,10 @@ Zotero.Fulltext = new function(){ } while (done < numWords); - Zotero.DB.query("REPLACE INTO fulltextItems (itemID, version) VALUES (?,?)", - [itemID, FULLTEXT_VERSION]); - + if (!Zotero.DB.valueQuery("SELECT COUNT(*) FROM fulltextItems WHERE itemID=?", itemID)) { + let sql = "INSERT INTO fulltextItems (itemID, version) VALUES (?,?)"; + Zotero.DB.query(sql, [itemID, 0]); + } // Handle bound parameters manually for optimal speed var statement1 = Zotero.DB.getStatement("INSERT INTO fulltextWords (word) VALUES (?)"); @@ -266,7 +288,7 @@ Zotero.Fulltext = new function(){ } - function indexString(text, charset, itemID){ + function indexString(text, charset, itemID, stats, version, synced) { try { Zotero.UnresponsiveScriptIndicator.disable(); @@ -275,7 +297,23 @@ Zotero.Fulltext = new function(){ Zotero.DB.beginTransaction(); this.clearItemWords(itemID, true); - this.indexWords(itemID, words); + this.indexWords(itemID, words, stats, version, synced); + + var sql = "UPDATE fulltextItems SET synced=?"; + var params = [synced ? parseInt(synced) : SYNC_STATE_UNSYNCED]; + if (stats) { + for (let stat in stats) { + sql += ", " + stat + "=?"; + params.push(stats[stat] ? parseInt(stats[stat]) : null); + } + } + if (version) { + sql += ", version=?"; + params.push(parseInt(version)); + } + sql += " WHERE itemID=?"; + params.push(itemID); + Zotero.DB.query(sql, params); /* var sql = "REPLACE INTO fulltextContent (itemID, textContent) VALUES (?,?)"; @@ -283,6 +321,14 @@ Zotero.Fulltext = new function(){ */ Zotero.DB.commitTransaction(); + + // If there's a processor cache file, delete it (whether or not we just used it) + var cacheFile = this.getItemProcessorCacheFile(itemID); + if (cacheFile.exists()) { + cacheFile.remove(false); + } + + Zotero.Notifier.trigger('refresh', 'item', itemID); } finally { Zotero.UnresponsiveScriptIndicator.enable(); @@ -499,7 +545,7 @@ Zotero.Fulltext = new function(){ function indexItems(items, complete, ignoreErrors) { - if (items.constructor.name != 'Array') { + if (Array.isArray(items)) { items = [items]; } var items = Zotero.Items.get(items); @@ -507,20 +553,22 @@ Zotero.Fulltext = new function(){ Zotero.DB.beginTransaction(); - for each(var i in items){ - if (!i.isAttachment()){ + for each (let item in items) { + if (!item.isAttachment()) { continue; } - var file = i.getFile(); + let itemID = item.id; + + var file = item.getFile(); if (!file){ - Zotero.debug("No file to index for item " + i.id + " in Fulltext.indexItems()"); + Zotero.debug("No file to index for item " + itemID + " in Fulltext.indexItems()"); continue; } if (ignoreErrors) { try { - this.indexFile(file, i.attachmentMIMEType, i.attachmentCharset, i.id, !complete); + this.indexFile(file, item.attachmentMIMEType, item.attachmentCharset, itemID, !complete); } catch (e) { Zotero.debug(e, 1); @@ -529,7 +577,7 @@ Zotero.Fulltext = new function(){ } } else { - this.indexFile(file, i.attachmentMIMEType, i.attachmentCharset, i.id, !complete); + this.indexFile(file, item.attachmentMIMEType, item.attachmentCharset, itemID, !complete); } } @@ -537,8 +585,356 @@ Zotero.Fulltext = new function(){ } + // + // Full-text content syncing + // + /** + * Get content and stats that haven't yet been synced + * + * @param {Integer} maxChars Maximum total characters to include. + * The total can go over this if there's a + * single large item. + * @return {Array} + */ + this.getUnsyncedContent = function (maxChars) { + var first = true; + var chars = 0; + var contentItems = []; + var sql = "SELECT itemID, indexedChars, totalChars, indexedPages, totalPages " + + "FROM fulltextItems WHERE synced=" + SYNC_STATE_UNSYNCED; + var rows = Zotero.DB.query(sql) || []; + for each (let row in rows) { + let text; + let itemID = row.itemID; + let item = Zotero.Items.get(itemID); + let libraryKey = item.libraryID + "/" + item.key; + let mimeType = item.attachmentMIMEType; + if (isCachedMIMEType(mimeType) || Zotero.MIME.isTextType(mimeType)) { + try { + let cacheFile = this.getItemCacheFile(itemID); + if (cacheFile.exists()) { + Zotero.debug("Adding full-text content from cache " + + "file for item " + libraryKey); + text = Zotero.File.getContents(cacheFile); + } + else { + if (!Zotero.MIME.isTextType(mimeType)) { + Zotero.debug("Full-text content cache file doesn't exist for item " + + libraryKey, 2); + continue; + } + + let file = item.getFile(); + if (!file) { + Zotero.debug("File doesn't exist getting full-text content for item " + + libraryKey, 2); + continue; + } + + Zotero.debug("Adding full-text content from file for item " + libraryKey); + text = Zotero.File.getContents( + file, item.attachmentCharset, row.indexedChars + ); + + // Split elements to avoid word concatentation + if (item.attachmentMIMEType == 'text/html') { + text = text.replace(/(>)/g, '$1 '); + text = this.HTMLToText(text); + + // Write the converted text to a cache file + Zotero.debug("Writing converted full-text HTML content to " + + cacheFile.path); + if (!cacheFile.parent.exists()) { + Zotero.Attachments.createDirectoryForItem(itemID); + } + Zotero.File.putContentsAsync(cacheFile, text) + .catch(function (e) { + Zotero.debug(e, 1); + Components.utils.reportError(e); + }) + } + } + } + catch (e) { + Zotero.debug(e, 1); + Components.utils.reportError(e); + continue; + } + } + else { + Zotero.debug("Skipping non-text file getting full-text content for item " + + libraryKey, 2); + + // Delete rows for items that weren't supposed to be indexed + this.clearItemWords(itemID); + continue; + } + + // If this isn't the first item and it would put us over the limit, + // skip it + if (!first && maxChars && ((chars + text.length) > maxChars)) { + continue; + } + chars += text.length; + first = false; + + contentItems.push({ + libraryID: item.libraryID, + key: item.key, + text: text, + indexedChars: row.indexedChars ? row.indexedChars : 0, + totalChars: row.totalChars ? row.totalChars : 0, + indexedPages: row.indexedPages ? row.indexedPages : 0, + totalPages: row.totalPages ? row.totalPages : 0 + }); + + if (maxChars && chars > maxChars) { + break; + } + } + return contentItems; + } + + + /** + * @return {String} PHP-formatted POST data for items not yet downloaded + */ + this.getUndownloadedPostData = function () { + // On upgrade, get all content + var sql = "SELECT value FROM settings WHERE setting='fulltext' AND key='downloadAll'"; + if (Zotero.DB.valueQuery(sql)) { + return "&ftkeys=all"; + } + + var sql = "SELECT itemID FROM fulltextItems WHERE synced=" + + SYNC_STATE_TO_DOWNLOAD; + var itemIDs = Zotero.DB.columnQuery(sql); + if (!itemIDs) { + return ""; + } + var undownloaded = {}; + for each (let itemID in itemIDs) { + let item = Zotero.Items.get(itemID); + let libraryID = item.libraryID + libraryID = libraryID ? libraryID : Zotero.libraryID; + if (!undownloaded[libraryID]) { + undownloaded[libraryID] = []; + } + undownloaded[libraryID].push(item.key); + } + var data = ""; + for (let libraryID in undownloaded) { + for (let i = 0; i < undownloaded[libraryID].length; i++) { + data += "&" + encodeURIComponent("ftkeys[" + libraryID + "][" + i + "]") + + "=" + undownloaded[libraryID][i]; + } + } + return data; + } + + + /** + * Save full-text content and stats to a cache file + */ + this.setItemContent = function (libraryID, key, text, stats, version) { + var item = Zotero.Items.getByLibraryAndKey(libraryID, key); + if (!item) { + let msg = "Item not found setting full-text content"; + Zotero.debug(msg, 1); + Components.utils.reportError(msg); + return; + } + var itemID = item.id; + + if (text !== '') { + var cacheFile = this.getItemProcessorCacheFile(itemID); + + // If a storage directory doesn't exist, create it + if (!cacheFile.parent.exists()) { + Zotero.Attachments.createDirectoryForItem(itemID); + } + + Zotero.debug("Writing full-text content and data to " + cacheFile.path); + Zotero.File.putContents(cacheFile, JSON.stringify({ + indexedChars: stats.indexedChars, + totalChars: stats.totalChars, + indexedPages: stats.indexedPages, + totalPages: stats.totalPages, + version: version, + text: text + })); + var synced = SYNC_STATE_TO_PROCESS; + } + else { + Zotero.debug("Marking full-text content for download"); + var synced = SYNC_STATE_TO_DOWNLOAD; + } + + // Mark the item as unprocessed + if (Zotero.DB.valueQuery("SELECT COUNT(*) FROM fulltextItems WHERE itemID=?", itemID)) { + Zotero.DB.query("UPDATE fulltextItems SET synced=? WHERE itemID=?", [synced, itemID]); + } + // If not yet indexed, add an empty row + else { + Zotero.DB.query( + "REPLACE INTO fulltextItems (itemID, version, synced) VALUES (?, 0, ?)", + [itemID, synced] + ); + } + + if (_upgradeCheck) { + Zotero.DB.query("DELETE FROM settings WHERE setting='fulltext' AND key='downloadAll'"); + _upgradeCheck = false; + } + + this.startContentProcessor(); + } + + + /** + * Start the idle observer for the background content processor + */ + this.startContentProcessor = function () { + if (!_idleObserverIsRegistered) { + Zotero.debug("Initializing full-text content ingester idle observer"); + var idleService = Components.classes["@mozilla.org/widget/idleservice;1"] + .getService(Components.interfaces.nsIIdleService); + idleService.addIdleObserver(this.idleObserver, _idleObserverDelay); + _idleObserverIsRegistered = true; + } + } + + /** + * Stop the idle observer and a running timer, if there is one + */ + this.stopContentProcessor = function () { + if (_idleObserverIsRegistered) { + var idleService = Components.classes["@mozilla.org/widget/idleservice;1"] + .getService(Components.interfaces.nsIIdleService); + idleService.removeIdleObserver(this.idleObserver, _idleObserverDelay); + _idleObserverIsRegistered = false; + } + + if (_processorTimer) { + _processorTimer.cancel(); + _processorTimer = null; + } + } + + /** + * + * @param {Array} itemIDs An array of itemIDs to process; if this + * is omitted, a database query is made + * to find unprocessed content + * @return {Boolean} TRUE if there's more content to process; FALSE otherwise + */ + this.processUnprocessedContent = function (itemIDs) { + if (!itemIDs) { + Zotero.debug("Checking for unprocessed full-text content"); + let sql = "SELECT itemID FROM fulltextItems WHERE synced=" + + SYNC_STATE_TO_PROCESS; + itemIDs = Zotero.DB.columnQuery(sql) || []; + } + // If there's no more unprocessed content, stop the idle observer + if (!itemIDs.length) { + Zotero.debug("No unprocessed full-text content found"); + this.stopContentProcessor(); + return; + } + + let itemID = itemIDs.shift(); + let item = Zotero.Items.get(itemID); + + Zotero.debug("Processing full-text content for item " + item.libraryKey); + + Zotero.Fulltext.indexFromProcessorCache(itemID) + .then(function () { + if (itemIDs.length) { + if (!_processorTimer) { + _processorTimer = Components.classes["@mozilla.org/timer;1"] + .createInstance(Components.interfaces.nsITimer); + } + _processorTimer.initWithCallback( + function () { + Zotero.Fulltext.processUnprocessedContent(itemIDs); + }, + 100, + Components.interfaces.nsITimer.TYPE_ONE_SHOT + ); + } + }) + .done(); + } + + this.idleObserver = { + observe: function (subject, topic, data) { + // On idle, start the background processor + if (topic == 'idle') { + Zotero.Fulltext.processUnprocessedContent(); + } + // When back from idle, stop the processor (but keep the idle + // observer registered) + else if (topic == 'active') { + if (_processorTimer) { + Zotero.debug("Stopping full-text content processor"); + _processorTimer.cancel(); + } + } + } + }; + + + this.indexFromProcessorCache = function (itemID) { + var self = this; + return Q.fcall(function () { + var cacheFile = self.getItemProcessorCacheFile(itemID); + if (!cacheFile.exists()) { + Zotero.debug("Full-text content processor cache file doesn't exist for item " + itemID); + return false; + } + + let data; + + return Zotero.File.getContentsAsync(cacheFile) + .then(function (json) { + data = JSON.parse(json); + + // Write the text content to the regular cache file + cacheFile = self.getItemCacheFile(itemID); + + Zotero.debug("Writing full-text content to " + cacheFile.path); + return Zotero.File.putContentsAsync(cacheFile, data.text, "UTF-8"); + }) + .then(function () { + Zotero.Fulltext.indexString( + data.text, + "UTF-8", + itemID, + { + indexedChars: data.indexedChars, + totalChars: data.totalChars, + indexedPages: data.indexedPages, + totalPages: data.totalPages + }, + data.version, + 1 + ); + }); + }) + .catch(function (e) { + Components.utils.reportError(e); + Zotero.debug(e, 1); + return false; + }); + } + + // + // End full-text content syncing + // + + /* - * Scan a file for a text string + * Scan a string for another string * * _items_ -- one or more attachment items to search * _searchText_ -- text pattern to search for @@ -548,21 +944,7 @@ Zotero.Fulltext = new function(){ * * - Slashes in regex are optional */ - function findTextInFile(file, charset, searchText, mode){ - Zotero.debug("Searching for text '" + searchText + "' in " + file.path); - - var maxLength = Zotero.Prefs.get('fulltext.textMaxLength'); - var str = Zotero.File.getContents(file, charset, maxLength); - - // If not binary mode, convert HTML to text - if (!mode || mode.indexOf('Binary')==-1){ - // Split elements to avoid word concatentation - str = str.replace(/(>)/g, '$1 '); - - // Parse to avoid searching on HTML - str = this.HTMLToText(str); - } - + this.findTextInString = function (content, searchText, mode) { switch (mode){ case 'regexp': case 'regexpCS': @@ -583,7 +965,7 @@ Zotero.Fulltext = new function(){ try { var re = new RegExp(searchText, flags); - var matches = re.exec(str); + var matches = re.exec(content); } catch (e) { Zotero.debug(e, 1); @@ -591,7 +973,7 @@ Zotero.Fulltext = new function(){ } if (matches){ Zotero.debug("Text found"); - return str.substr(matches.index, 50); + return content.substr(matches.index, 50); } break; @@ -599,12 +981,12 @@ Zotero.Fulltext = new function(){ default: // Case-insensitive searchText = searchText.toLowerCase(); - str = str.toLowerCase(); + content = content.toLowerCase(); - var pos = str.indexOf(searchText); + var pos = content.indexOf(searchText); if (pos!=-1){ Zotero.debug('Text found'); - return str.substr(pos, 50); + return content.substr(pos, 50); } } @@ -633,42 +1015,78 @@ Zotero.Fulltext = new function(){ var items = Zotero.Items.get(items); var found = []; - for each(var i in items){ - if (!i.isAttachment()){ + for each (let item in items) { + if (!item.isAttachment()) { continue; } - var file = i.getFile(); - if (!file){ - continue; - } + let itemID = item.id; + let content; + let mimeType = item.attachmentMIMEType; + let maxLength = Zotero.Prefs.get('fulltext.textMaxLength'); + let binaryMode = mode && mode.indexOf('Binary') != -1; - var mimeType = i.attachmentMIMEType; if (isCachedMIMEType(mimeType)) { - var file = this.getItemCacheFile(i.id); + let file = this.getItemCacheFile(itemID); if (!file.exists()) { continue; } - mimeType = 'text/plain'; - var charset = 'utf-8'; + Zotero.debug("Searching for text '" + searchText + "' in " + file.path); + content = Zotero.File.getContents(file, 'utf-8', maxLength); } else { // If not binary mode, only scan plaintext files - if (!mode || mode.indexOf('Binary') == -1) { + if (!binaryMode) { if (!Zotero.MIME.isTextType(mimeType)) { Zotero.debug('Not scanning MIME type ' + mimeType, 4); continue; } } - var charset = i.attachmentCharset; + // Check for a cache file + let cacheFile = this.getItemCacheFile(itemID); + if (cacheFile.exists()) { + Zotero.debug("Searching for text '" + searchText + "' in " + cacheFile.path); + content = Zotero.File.getContents(cacheFile, 'utf-8', maxLength); + } + else { + // If that doesn't exist, check for the actual file + let file = item.getFile(); + if (!file) { + continue; + } + + Zotero.debug("Searching for text '" + searchText + "' in " + file.path); + content = Zotero.File.getContents(file, item.attachmentCharset, maxLength); + + // If HTML and not binary mode, convert to text + if (mimeType == 'text/html' && !binaryMode) { + // Split elements to avoid word concatentation + content = content.replace(/(>)/g, '$1 '); + + content = this.HTMLToText(content); + + // Write the converted text to a cache file for future searches + Zotero.debug("Writing converted full-text content to " + cacheFile.path); + if (!cacheFile.parent.exists()) { + Zotero.Attachments.createDirectoryForItem(itemID); + } + Zotero.File.putContentsAsync(cacheFile, content) + .catch(function (e) { + Zotero.debug(e, 1); + Components.utils.reportError(e); + }) + } + } } - var match = this.findTextInFile(file, charset, searchText, mode); - - if (match != -1){ - found.push({id:i.getID(), match:match}); + let match = this.findTextInString(content, searchText, mode); + if (match != -1) { + found.push({ + id: itemID, + match: match + }); } } @@ -681,8 +1099,8 @@ Zotero.Fulltext = new function(){ var sql = "SELECT rowid FROM fulltextItems WHERE itemID=? LIMIT 1"; var indexed = Zotero.DB.valueQuery(sql, itemID); if (indexed) { - Zotero.DB.query("DELETE FROM fulltextItems WHERE itemID=?", itemID); Zotero.DB.query("DELETE FROM fulltextItemWords WHERE itemID=?", itemID); + Zotero.DB.query("DELETE FROM fulltextItems WHERE itemID=?", itemID); } Zotero.DB.commitTransaction(); @@ -760,15 +1178,27 @@ Zotero.Fulltext = new function(){ function setPages(itemID, obj) { var sql = "UPDATE fulltextItems SET indexedPages=?, totalPages=? WHERE itemID=?"; - Zotero.DB.query(sql, [obj.indexed ? obj.indexed : null, - obj.total ? obj.total : null, itemID]); + Zotero.DB.query( + sql, + [ + obj.indexed ? parseInt(obj.indexed) : null, + obj.total ? parseInt(obj.total) : null, + itemID + ] + ); } function setChars(itemID, obj) { var sql = "UPDATE fulltextItems SET indexedChars=?, totalChars=? WHERE itemID=?"; - Zotero.DB.query(sql, [obj.indexed ? obj.indexed : null, - obj.total ? obj.total : null, itemID]); + Zotero.DB.query( + sql, + [ + obj.indexed ? parseInt(obj.indexed) : null, + obj.total ? parseInt(obj.total) : null, + itemID + ] + ); } @@ -878,6 +1308,13 @@ Zotero.Fulltext = new function(){ } + this.getItemProcessorCacheFile = function (itemID) { + var cacheFile = Zotero.Attachments.getStorageDirectory(itemID); + cacheFile.append(_processorCacheFile); + return cacheFile; + } + + /* * Returns true if an item can be reindexed * @@ -971,18 +1408,14 @@ Zotero.Fulltext = new function(){ } Zotero.debug('Clearing full-text cache file for item ' + itemID); - switch (item.attachmentMIMEType) { - case 'application/pdf': - var cacheFile = this.getItemCacheFile(itemID); - if (cacheFile.exists()) { - try { - cacheFile.remove(false); - } - catch (e) { - Zotero.File.checkFileAccessError(e, cacheFile, 'delete'); - } - } - break; + var cacheFile = this.getItemCacheFile(itemID); + if (cacheFile.exists()) { + try { + cacheFile.remove(false); + } + catch (e) { + Zotero.File.checkFileAccessError(e, cacheFile, 'delete'); + } } } diff --git a/chrome/content/zotero/xpcom/sync.js b/chrome/content/zotero/xpcom/sync.js index bde044c4e..2aed5dc11 100644 --- a/chrome/content/zotero/xpcom/sync.js +++ b/chrome/content/zotero/xpcom/sync.js @@ -55,6 +55,10 @@ Zotero.Sync = new function() { setting: { singular: 'Setting', plural: 'Settings' + }, + fulltext: { + singular: 'Fulltext', + plural: 'Fulltexts' } }; }); @@ -132,7 +136,7 @@ Zotero.Sync = new function() { } for (var type in this.syncObjects) { - if (type == 'setting') { + if (type == 'setting' || type == 'fulltext') { continue; } @@ -451,6 +455,9 @@ Zotero.Sync.EventListener = new function () { if (type == 'setting') { [libraryID, key] = ids[i].split("/"); } + else if (type == 'fulltext') { + continue; + } else { var oldItem = extraData[ids[i]].old; libraryID = oldItem.primary.libraryID; @@ -1462,6 +1469,8 @@ Zotero.Sync.Server = new function () { body += '&upload=1'; } + body += Zotero.Fulltext.getUndownloadedPostData(); + Zotero.Sync.Runner.setSyncStatus(Zotero.getString('sync.status.gettingUpdatedData')); Zotero.HTTP.doPost(url, body, function (xmlhttp) { @@ -1694,6 +1703,14 @@ Zotero.Sync.Server = new function () { var sql = "UPDATE syncedSettings SET synced=1"; Zotero.DB.query(sql); + if (syncSession.fulltextItems && syncSession.fulltextItems.length) { + let sql = "UPDATE fulltextItems SET synced=1 WHERE itemID=?"; + for each (let lk in syncSession.fulltextItems) { + let item = Zotero.Items.getByLibraryAndKey(lk.libraryID, lk.key); + Zotero.DB.query(sql, item.id); + } + } + //throw('break2'); Zotero.DB.commitTransaction(); @@ -2825,6 +2842,26 @@ Zotero.Sync.Server.Data = new function() { Zotero.SyncedSettings.setSynchronous(libraryID, name, value, version, true); continue; } + else if (type == 'fulltext') { + if (!libraryID) { + libraryID = 0; + } + let key = objectNode.getAttribute('key'); + Zotero.debug("Processing remote full-text content for item " + libraryID + "/" + key); + Zotero.Fulltext.setItemContent( + libraryID, + key, + objectNode.textContent, + { + indexedChars: parseInt(objectNode.getAttribute('indexedChars')), + totalChars: parseInt(objectNode.getAttribute('totalChars')), + indexedPages: parseInt(objectNode.getAttribute('indexedPages')), + totalPages: parseInt(objectNode.getAttribute('totalPages')) + }, + parseInt(objectNode.getAttribute('version')) + ); + continue; + } var key = objectNode.getAttribute('key'); var objLibraryKeyHash = Zotero[Types].makeLibraryKeyHash(libraryID, key); @@ -3537,6 +3574,36 @@ Zotero.Sync.Server.Data = new function() { docElem.appendChild(settingsNode); } + // Add up to 500K characters of full-text content + try { + var rows = Zotero.Fulltext.getUnsyncedContent(500000); + } + catch (e) { + Zotero.debug(e, 1); + Components.utils.reportError(e); + var rows = []; + } + if (rows.length) { + let fulltextsNode = doc.createElement('fulltexts'); + syncSession.fulltextItems = []; + for (let i=0; i