suzanne.soy/zotero
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Full-text content syncing

Browse Source 
Full-text content is now synced to the server and other clients. Up to
~500K of content is transferred with each sync, so multiple syncs may be
necessary. Downloaded content is written to temporary files and
processed on idle (currently 5 seconds), so synced content may not be
immediately available.

Performance will improve with API syncing and async DB in 4.1.

This also includes some other full-text tweaks, such as storing
text converted from HTML in cache files, which may speed up phrase
searching.
This commit is contained in:
Dan Stillman 2013-11-03 19:14:10 -05:00
parent afd14ae4b9
commit 62840f3b93
4 changed files with 573 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
chrome/content/zotero/xpcom/attachments.js
View File

@ -567,11 +567,9 @@ Zotero.Attachments = new function(){
var f = function() {
if (mimeType == 'application/pdf') {
Zotero.Fulltext.indexPDF(file, itemID);
Zotero.Notifier.trigger('refresh', 'item', itemID);
}
if (Zotero.MIME.isTextType(mimeType)) {
else if (Zotero.MIME.isTextType(mimeType)) {
Zotero.Fulltext.indexDocument(document, itemID);
Zotero.Notifier.trigger('refresh', 'item', itemID);
}
if (callback) {
callback(attachmentItem);
@ -981,7 +979,7 @@ Zotero.Attachments = new function(){
function getStorageDirectory(itemID) {
if (!itemID) {
throw ("itemID not provided in Zotero.Attachments.getStorageDirectory()");
throw new Error("itemID not provided in Zotero.Attachments.getStorageDirectory()");
}
var item = Zotero.Items.get(itemID);
if (!item) {

571
chrome/content/zotero/xpcom/fulltext.js
View File

@ -24,7 +24,6 @@
*/
Zotero.Fulltext = new function(){
const FULLTEXT_VERSION = 1;
const CACHE_FILE = '.zotero-ft-cache';
this.init = init;
@ -38,7 +37,6 @@ Zotero.Fulltext = new function(){
this.indexFile = indexFile;
this.indexPDF = indexPDF;
this.indexItems = indexItems;
this.findTextInFile = findTextInFile;
this.findTextInItems = findTextInItems;
this.clearItemWords = clearItemWords;
this.getPages = getPages;
@ -72,6 +70,7 @@ Zotero.Fulltext = new function(){
this.__defineGetter__("INDEX_STATE_PARTIAL", function () { return 2; });
this.__defineGetter__("INDEX_STATE_INDEXED", function () { return 3; });
const _processorCacheFile = '.zotero-ft-unprocessed';
var _pdfConverterVersion = null;
var _pdfConverterFileName = null;
@ -80,6 +79,16 @@ Zotero.Fulltext = new function(){
var _pdfInfoFileName = null;
var _pdfInfo = null; // nsIFile to executable
var _idleObserverIsRegistered = false;
var _idleObserverDelay = 5;
var _processorTimer = null;
var _upgradeCheck = true;
const SYNC_STATE_UNSYNCED = 0;
const SYNC_STATE_IN_SYNC = 1;
const SYNC_STATE_TO_PROCESS = 2;
const SYNC_STATE_TO_DOWNLOAD = 3;
var self = this;
function init() {
@ -98,6 +107,18 @@ Zotero.Fulltext = new function(){
this.registerPDFTool('converter');
this.registerPDFTool('info');
// TEMP: Remove after 4.1 DB schema change
var cols = Zotero.DB.getColumns('fulltextItems');
if (cols.indexOf("synced") == -1) {
Zotero.DB.beginTransaction();
Zotero.DB.query("ALTER TABLE fulltextItems ADD COLUMN synced INT DEFAULT 0");
Zotero.DB.query("REPLACE INTO settings (setting, key, value) VALUES ('fulltext', 'downloadAll', 1)");
Zotero.DB.commitTransaction();
}
this.startContentProcessor();
Zotero.addShutdownListener(this.stopContentProcessor);
}
@ -192,7 +213,7 @@ Zotero.Fulltext = new function(){
/*
* Index multiple words at once
*/
function indexWords(itemID, words){
function indexWords(itemID, words) {
if (!words || !words.length || !itemID){
return false;
}
@ -231,9 +252,10 @@ Zotero.Fulltext = new function(){
}
while (done < numWords);
Zotero.DB.query("REPLACE INTO fulltextItems (itemID, version) VALUES (?,?)",
[itemID, FULLTEXT_VERSION]);
if (!Zotero.DB.valueQuery("SELECT COUNT(*) FROM fulltextItems WHERE itemID=?", itemID)) {
let sql = "INSERT INTO fulltextItems (itemID, version) VALUES (?,?)";
Zotero.DB.query(sql, [itemID, 0]);
}
// Handle bound parameters manually for optimal speed
var statement1 = Zotero.DB.getStatement("INSERT INTO fulltextWords (word) VALUES (?)");
@ -266,7 +288,7 @@ Zotero.Fulltext = new function(){
}
function indexString(text, charset, itemID){
function indexString(text, charset, itemID, stats, version, synced) {
try {
Zotero.UnresponsiveScriptIndicator.disable();
@ -275,7 +297,23 @@ Zotero.Fulltext = new function(){
Zotero.DB.beginTransaction();
this.clearItemWords(itemID, true);
this.indexWords(itemID, words);
this.indexWords(itemID, words, stats, version, synced);
var sql = "UPDATE fulltextItems SET synced=?";
var params = [synced ? parseInt(synced) : SYNC_STATE_UNSYNCED];
if (stats) {
for (let stat in stats) {
sql += ", " + stat + "=?";
params.push(stats[stat] ? parseInt(stats[stat]) : null);
}
}
if (version) {
sql += ", version=?";
params.push(parseInt(version));
}
sql += " WHERE itemID=?";
params.push(itemID);
Zotero.DB.query(sql, params);
/*
var sql = "REPLACE INTO fulltextContent (itemID, textContent) VALUES (?,?)";
@ -283,6 +321,14 @@ Zotero.Fulltext = new function(){
*/
Zotero.DB.commitTransaction();
// If there's a processor cache file, delete it (whether or not we just used it)
var cacheFile = this.getItemProcessorCacheFile(itemID);
if (cacheFile.exists()) {
cacheFile.remove(false);
}
Zotero.Notifier.trigger('refresh', 'item', itemID);
}
finally {
Zotero.UnresponsiveScriptIndicator.enable();
@ -499,7 +545,7 @@ Zotero.Fulltext = new function(){
function indexItems(items, complete, ignoreErrors) {
if (items.constructor.name != 'Array') {
if (Array.isArray(items)) {
items = [items];
}
var items = Zotero.Items.get(items);
@ -507,20 +553,22 @@ Zotero.Fulltext = new function(){
Zotero.DB.beginTransaction();
for each(var i in items){
if (!i.isAttachment()){
for each (let item in items) {
if (!item.isAttachment()) {
continue;
}
var file = i.getFile();
let itemID = item.id;
var file = item.getFile();
if (!file){
Zotero.debug("No file to index for item " + i.id + " in Fulltext.indexItems()");
Zotero.debug("No file to index for item " + itemID + " in Fulltext.indexItems()");
continue;
}
if (ignoreErrors) {
try {
this.indexFile(file, i.attachmentMIMEType, i.attachmentCharset, i.id, !complete);
this.indexFile(file, item.attachmentMIMEType, item.attachmentCharset, itemID, !complete);
}
catch (e) {
Zotero.debug(e, 1);
@ -529,7 +577,7 @@ Zotero.Fulltext = new function(){
}
}
else {
this.indexFile(file, i.attachmentMIMEType, i.attachmentCharset, i.id, !complete);
this.indexFile(file, item.attachmentMIMEType, item.attachmentCharset, itemID, !complete);
}
}
@ -537,8 +585,356 @@ Zotero.Fulltext = new function(){
}
//
// Full-text content syncing
//
/**
* Get content and stats that haven't yet been synced
*
* @param {Integer} maxChars Maximum total characters to include.
* The total can go over this if there's a
* single large item.
* @return {Array<Object>}
*/
this.getUnsyncedContent = function (maxChars) {
var first = true;
var chars = 0;
var contentItems = [];
var sql = "SELECT itemID, indexedChars, totalChars, indexedPages, totalPages "
+ "FROM fulltextItems WHERE synced=" + SYNC_STATE_UNSYNCED;
var rows = Zotero.DB.query(sql) || [];
for each (let row in rows) {
let text;
let itemID = row.itemID;
let item = Zotero.Items.get(itemID);
let libraryKey = item.libraryID + "/" + item.key;
let mimeType = item.attachmentMIMEType;
if (isCachedMIMEType(mimeType) || Zotero.MIME.isTextType(mimeType)) {
try {
let cacheFile = this.getItemCacheFile(itemID);
if (cacheFile.exists()) {
Zotero.debug("Adding full-text content from cache "
+ "file for item " + libraryKey);
text = Zotero.File.getContents(cacheFile);
}
else {
if (!Zotero.MIME.isTextType(mimeType)) {
Zotero.debug("Full-text content cache file doesn't exist for item "
+ libraryKey, 2);
continue;
}
let file = item.getFile();
if (!file) {
Zotero.debug("File doesn't exist getting full-text content for item "
+ libraryKey, 2);
continue;
}
Zotero.debug("Adding full-text content from file for item " + libraryKey);
text = Zotero.File.getContents(
file, item.attachmentCharset, row.indexedChars
);
// Split elements to avoid word concatentation
if (item.attachmentMIMEType == 'text/html') {
text = text.replace(/(>)/g, '$1 ');
text = this.HTMLToText(text);
// Write the converted text to a cache file
Zotero.debug("Writing converted full-text HTML content to "
+ cacheFile.path);
if (!cacheFile.parent.exists()) {
Zotero.Attachments.createDirectoryForItem(itemID);
}
Zotero.File.putContentsAsync(cacheFile, text)
.catch(function (e) {
Zotero.debug(e, 1);
Components.utils.reportError(e);
})
}
}
}
catch (e) {
Zotero.debug(e, 1);
Components.utils.reportError(e);
continue;
}
}
else {
Zotero.debug("Skipping non-text file getting full-text content for item "
+ libraryKey, 2);
// Delete rows for items that weren't supposed to be indexed
this.clearItemWords(itemID);
continue;
}
// If this isn't the first item and it would put us over the limit,
// skip it
if (!first && maxChars && ((chars + text.length) > maxChars)) {
continue;
}
chars += text.length;
first = false;
contentItems.push({
libraryID: item.libraryID,
key: item.key,
text: text,
indexedChars: row.indexedChars ? row.indexedChars : 0,
totalChars: row.totalChars ? row.totalChars : 0,
indexedPages: row.indexedPages ? row.indexedPages : 0,
totalPages: row.totalPages ? row.totalPages : 0
});
if (maxChars && chars > maxChars) {
break;
}
}
return contentItems;
}
/**
* @return {String} PHP-formatted POST data for items not yet downloaded
*/
this.getUndownloadedPostData = function () {
// On upgrade, get all content
var sql = "SELECT value FROM settings WHERE setting='fulltext' AND key='downloadAll'";
if (Zotero.DB.valueQuery(sql)) {
return "&ftkeys=all";
}
var sql = "SELECT itemID FROM fulltextItems WHERE synced="
+ SYNC_STATE_TO_DOWNLOAD;
var itemIDs = Zotero.DB.columnQuery(sql);
if (!itemIDs) {
return "";
}
var undownloaded = {};
for each (let itemID in itemIDs) {
let item = Zotero.Items.get(itemID);
let libraryID = item.libraryID
libraryID = libraryID ? libraryID : Zotero.libraryID;
if (!undownloaded[libraryID]) {
undownloaded[libraryID] = [];
}
undownloaded[libraryID].push(item.key);
}
var data = "";
for (let libraryID in undownloaded) {
for (let i = 0; i < undownloaded[libraryID].length; i++) {
data += "&" + encodeURIComponent("ftkeys[" + libraryID + "][" + i + "]")
+ "=" + undownloaded[libraryID][i];
}
}
return data;
}
/**
* Save full-text content and stats to a cache file
*/
this.setItemContent = function (libraryID, key, text, stats, version) {
var item = Zotero.Items.getByLibraryAndKey(libraryID, key);
if (!item) {
let msg = "Item not found setting full-text content";
Zotero.debug(msg, 1);
Components.utils.reportError(msg);
return;
}
var itemID = item.id;
if (text !== '') {
var cacheFile = this.getItemProcessorCacheFile(itemID);
// If a storage directory doesn't exist, create it
if (!cacheFile.parent.exists()) {
Zotero.Attachments.createDirectoryForItem(itemID);
}
Zotero.debug("Writing full-text content and data to " + cacheFile.path);
Zotero.File.putContents(cacheFile, JSON.stringify({
indexedChars: stats.indexedChars,
totalChars: stats.totalChars,
indexedPages: stats.indexedPages,
totalPages: stats.totalPages,
version: version,
text: text
}));
var synced = SYNC_STATE_TO_PROCESS;
}
else {
Zotero.debug("Marking full-text content for download");
var synced = SYNC_STATE_TO_DOWNLOAD;
}
// Mark the item as unprocessed
if (Zotero.DB.valueQuery("SELECT COUNT(*) FROM fulltextItems WHERE itemID=?", itemID)) {
Zotero.DB.query("UPDATE fulltextItems SET synced=? WHERE itemID=?", [synced, itemID]);
}
// If not yet indexed, add an empty row
else {
Zotero.DB.query(
"REPLACE INTO fulltextItems (itemID, version, synced) VALUES (?, 0, ?)",
[itemID, synced]
);
}
if (_upgradeCheck) {
Zotero.DB.query("DELETE FROM settings WHERE setting='fulltext' AND key='downloadAll'");
_upgradeCheck = false;
}
this.startContentProcessor();
}
/**
* Start the idle observer for the background content processor
*/
this.startContentProcessor = function () {
if (!_idleObserverIsRegistered) {
Zotero.debug("Initializing full-text content ingester idle observer");
var idleService = Components.classes["@mozilla.org/widget/idleservice;1"]
.getService(Components.interfaces.nsIIdleService);
idleService.addIdleObserver(this.idleObserver, _idleObserverDelay);
_idleObserverIsRegistered = true;
}
}
/**
* Stop the idle observer and a running timer, if there is one
*/
this.stopContentProcessor = function () {
if (_idleObserverIsRegistered) {
var idleService = Components.classes["@mozilla.org/widget/idleservice;1"]
.getService(Components.interfaces.nsIIdleService);
idleService.removeIdleObserver(this.idleObserver, _idleObserverDelay);
_idleObserverIsRegistered = false;
}
if (_processorTimer) {
_processorTimer.cancel();
_processorTimer = null;
}
}
/**
*
* @param {Array<Integer>} itemIDs An array of itemIDs to process; if this
* is omitted, a database query is made
* to find unprocessed content
* @return {Boolean} TRUE if there's more content to process; FALSE otherwise
*/
this.processUnprocessedContent = function (itemIDs) {
if (!itemIDs) {
Zotero.debug("Checking for unprocessed full-text content");
let sql = "SELECT itemID FROM fulltextItems WHERE synced="
+ SYNC_STATE_TO_PROCESS;
itemIDs = Zotero.DB.columnQuery(sql) || [];
}
// If there's no more unprocessed content, stop the idle observer
if (!itemIDs.length) {
Zotero.debug("No unprocessed full-text content found");
this.stopContentProcessor();
return;
}
let itemID = itemIDs.shift();
let item = Zotero.Items.get(itemID);
Zotero.debug("Processing full-text content for item " + item.libraryKey);
Zotero.Fulltext.indexFromProcessorCache(itemID)
.then(function () {
if (itemIDs.length) {
if (!_processorTimer) {
_processorTimer = Components.classes["@mozilla.org/timer;1"]
.createInstance(Components.interfaces.nsITimer);
}
_processorTimer.initWithCallback(
function () {
Zotero.Fulltext.processUnprocessedContent(itemIDs);
},
100,
Components.interfaces.nsITimer.TYPE_ONE_SHOT
);
}
})
.done();
}
this.idleObserver = {
observe: function (subject, topic, data) {
// On idle, start the background processor
if (topic == 'idle') {
Zotero.Fulltext.processUnprocessedContent();
}
// When back from idle, stop the processor (but keep the idle
// observer registered)
else if (topic == 'active') {
if (_processorTimer) {
Zotero.debug("Stopping full-text content processor");
_processorTimer.cancel();
}
}
}
};
this.indexFromProcessorCache = function (itemID) {
var self = this;
return Q.fcall(function () {
var cacheFile = self.getItemProcessorCacheFile(itemID);
if (!cacheFile.exists()) {
Zotero.debug("Full-text content processor cache file doesn't exist for item " + itemID);
return false;
}
let data;
return Zotero.File.getContentsAsync(cacheFile)
.then(function (json) {
data = JSON.parse(json);
// Write the text content to the regular cache file
cacheFile = self.getItemCacheFile(itemID);
Zotero.debug("Writing full-text content to " + cacheFile.path);
return Zotero.File.putContentsAsync(cacheFile, data.text, "UTF-8");
})
.then(function () {
Zotero.Fulltext.indexString(
data.text,
"UTF-8",
itemID,
{
indexedChars: data.indexedChars,
totalChars: data.totalChars,
indexedPages: data.indexedPages,
totalPages: data.totalPages
},
data.version,
1
);
});
})
.catch(function (e) {
Components.utils.reportError(e);
Zotero.debug(e, 1);
return false;
});
}
//
// End full-text content syncing
//
/*
* Scan a file for a text string
* Scan a string for another string
*
* _items_ -- one or more attachment items to search
* _searchText_ -- text pattern to search for
@ -548,21 +944,7 @@ Zotero.Fulltext = new function(){
*
* - Slashes in regex are optional
*/
function findTextInFile(file, charset, searchText, mode){
Zotero.debug("Searching for text '" + searchText + "' in " + file.path);
var maxLength = Zotero.Prefs.get('fulltext.textMaxLength');
var str = Zotero.File.getContents(file, charset, maxLength);
// If not binary mode, convert HTML to text
if (!mode || mode.indexOf('Binary')==-1){
// Split elements to avoid word concatentation
str = str.replace(/(>)/g, '$1 ');
// Parse to avoid searching on HTML
str = this.HTMLToText(str);
}
this.findTextInString = function (content, searchText, mode) {
switch (mode){
case 'regexp':
case 'regexpCS':
@ -583,7 +965,7 @@ Zotero.Fulltext = new function(){
try {
var re = new RegExp(searchText, flags);
var matches = re.exec(str);
var matches = re.exec(content);
}
catch (e) {
Zotero.debug(e, 1);
@ -591,7 +973,7 @@ Zotero.Fulltext = new function(){
}
if (matches){
Zotero.debug("Text found");
return str.substr(matches.index, 50);
return content.substr(matches.index, 50);
}
break;
@ -599,12 +981,12 @@ Zotero.Fulltext = new function(){
default:
// Case-insensitive
searchText = searchText.toLowerCase();
str = str.toLowerCase();
content = content.toLowerCase();
var pos = str.indexOf(searchText);
var pos = content.indexOf(searchText);
if (pos!=-1){
Zotero.debug('Text found');
return str.substr(pos, 50);
return content.substr(pos, 50);
}
}
@ -633,42 +1015,78 @@ Zotero.Fulltext = new function(){
var items = Zotero.Items.get(items);
var found = [];
for each(var i in items){
if (!i.isAttachment()){
for each (let item in items) {
if (!item.isAttachment()) {
continue;
}
var file = i.getFile();
if (!file){
continue;
}
let itemID = item.id;
let content;
let mimeType = item.attachmentMIMEType;
let maxLength = Zotero.Prefs.get('fulltext.textMaxLength');
let binaryMode = mode && mode.indexOf('Binary') != -1;
var mimeType = i.attachmentMIMEType;
if (isCachedMIMEType(mimeType)) {
var file = this.getItemCacheFile(i.id);
let file = this.getItemCacheFile(itemID);
if (!file.exists()) {
continue;
}
mimeType = 'text/plain';
var charset = 'utf-8';
Zotero.debug("Searching for text '" + searchText + "' in " + file.path);
content = Zotero.File.getContents(file, 'utf-8', maxLength);
}
else {
// If not binary mode, only scan plaintext files
if (!mode || mode.indexOf('Binary') == -1) {
if (!binaryMode) {
if (!Zotero.MIME.isTextType(mimeType)) {
Zotero.debug('Not scanning MIME type ' + mimeType, 4);
continue;
}
}
var charset = i.attachmentCharset;
// Check for a cache file
let cacheFile = this.getItemCacheFile(itemID);
if (cacheFile.exists()) {
Zotero.debug("Searching for text '" + searchText + "' in " + cacheFile.path);
content = Zotero.File.getContents(cacheFile, 'utf-8', maxLength);
}
else {
// If that doesn't exist, check for the actual file
let file = item.getFile();
if (!file) {
continue;
}
Zotero.debug("Searching for text '" + searchText + "' in " + file.path);
content = Zotero.File.getContents(file, item.attachmentCharset, maxLength);
// If HTML and not binary mode, convert to text
if (mimeType == 'text/html' && !binaryMode) {
// Split elements to avoid word concatentation
content = content.replace(/(>)/g, '$1 ');
content = this.HTMLToText(content);
// Write the converted text to a cache file for future searches
Zotero.debug("Writing converted full-text content to " + cacheFile.path);
if (!cacheFile.parent.exists()) {
Zotero.Attachments.createDirectoryForItem(itemID);
}
Zotero.File.putContentsAsync(cacheFile, content)
.catch(function (e) {
Zotero.debug(e, 1);
Components.utils.reportError(e);
})
}
}
}
var match = this.findTextInFile(file, charset, searchText, mode);
if (match != -1){
found.push({id:i.getID(), match:match});
let match = this.findTextInString(content, searchText, mode);
if (match != -1) {
found.push({
id: itemID,
match: match
});
}
}
@ -681,8 +1099,8 @@ Zotero.Fulltext = new function(){
var sql = "SELECT rowid FROM fulltextItems WHERE itemID=? LIMIT 1";
var indexed = Zotero.DB.valueQuery(sql, itemID);
if (indexed) {
Zotero.DB.query("DELETE FROM fulltextItems WHERE itemID=?", itemID);
Zotero.DB.query("DELETE FROM fulltextItemWords WHERE itemID=?", itemID);
Zotero.DB.query("DELETE FROM fulltextItems WHERE itemID=?", itemID);
}
Zotero.DB.commitTransaction();
@ -760,15 +1178,27 @@ Zotero.Fulltext = new function(){
function setPages(itemID, obj) {
var sql = "UPDATE fulltextItems SET indexedPages=?, totalPages=? WHERE itemID=?";
Zotero.DB.query(sql, [obj.indexed ? obj.indexed : null,
obj.total ? obj.total : null, itemID]);
Zotero.DB.query(
sql,
[
obj.indexed ? parseInt(obj.indexed) : null,
obj.total ? parseInt(obj.total) : null,
itemID
]
);
}
function setChars(itemID, obj) {
var sql = "UPDATE fulltextItems SET indexedChars=?, totalChars=? WHERE itemID=?";
Zotero.DB.query(sql, [obj.indexed ? obj.indexed : null,
obj.total ? obj.total : null, itemID]);
Zotero.DB.query(
sql,
[
obj.indexed ? parseInt(obj.indexed) : null,
obj.total ? parseInt(obj.total) : null,
itemID
]
);
}
@ -878,6 +1308,13 @@ Zotero.Fulltext = new function(){
}
this.getItemProcessorCacheFile = function (itemID) {
var cacheFile = Zotero.Attachments.getStorageDirectory(itemID);
cacheFile.append(_processorCacheFile);
return cacheFile;
}
/*
* Returns true if an item can be reindexed
*
@ -971,18 +1408,14 @@ Zotero.Fulltext = new function(){
}
Zotero.debug('Clearing full-text cache file for item ' + itemID);
switch (item.attachmentMIMEType) {
case 'application/pdf':
var cacheFile = this.getItemCacheFile(itemID);
if (cacheFile.exists()) {
try {
cacheFile.remove(false);
}
catch (e) {
Zotero.File.checkFileAccessError(e, cacheFile, 'delete');
}
}
break;
var cacheFile = this.getItemCacheFile(itemID);
if (cacheFile.exists()) {
try {
cacheFile.remove(false);
}
catch (e) {
Zotero.File.checkFileAccessError(e, cacheFile, 'delete');
}
}
}

69
chrome/content/zotero/xpcom/sync.js
View File

@ -55,6 +55,10 @@ Zotero.Sync = new function() {
setting: {
singular: 'Setting',
plural: 'Settings'
},
fulltext: {
singular: 'Fulltext',
plural: 'Fulltexts'
}
};
});
@ -132,7 +136,7 @@ Zotero.Sync = new function() {
}
for (var type in this.syncObjects) {
if (type == 'setting') {
if (type == 'setting' || type == 'fulltext') {
continue;
}
@ -451,6 +455,9 @@ Zotero.Sync.EventListener = new function () {
if (type == 'setting') {
[libraryID, key] = ids[i].split("/");
}
else if (type == 'fulltext') {
continue;
}
else {
var oldItem = extraData[ids[i]].old;
libraryID = oldItem.primary.libraryID;
@ -1462,6 +1469,8 @@ Zotero.Sync.Server = new function () {
body += '&upload=1';
}
body += Zotero.Fulltext.getUndownloadedPostData();
Zotero.Sync.Runner.setSyncStatus(Zotero.getString('sync.status.gettingUpdatedData'));
Zotero.HTTP.doPost(url, body, function (xmlhttp) {
@ -1694,6 +1703,14 @@ Zotero.Sync.Server = new function () {
var sql = "UPDATE syncedSettings SET synced=1";
Zotero.DB.query(sql);
if (syncSession.fulltextItems && syncSession.fulltextItems.length) {
let sql = "UPDATE fulltextItems SET synced=1 WHERE itemID=?";
for each (let lk in syncSession.fulltextItems) {
let item = Zotero.Items.getByLibraryAndKey(lk.libraryID, lk.key);
Zotero.DB.query(sql, item.id);
}
}
//throw('break2');
Zotero.DB.commitTransaction();
@ -2825,6 +2842,26 @@ Zotero.Sync.Server.Data = new function() {
Zotero.SyncedSettings.setSynchronous(libraryID, name, value, version, true);
continue;
}
else if (type == 'fulltext') {
if (!libraryID) {
libraryID = 0;
}
let key = objectNode.getAttribute('key');
Zotero.debug("Processing remote full-text content for item " + libraryID + "/" + key);
Zotero.Fulltext.setItemContent(
libraryID,
key,
objectNode.textContent,
{
indexedChars: parseInt(objectNode.getAttribute('indexedChars')),
totalChars: parseInt(objectNode.getAttribute('totalChars')),
indexedPages: parseInt(objectNode.getAttribute('indexedPages')),
totalPages: parseInt(objectNode.getAttribute('totalPages'))
},
parseInt(objectNode.getAttribute('version'))
);
continue;
}
var key = objectNode.getAttribute('key');
var objLibraryKeyHash = Zotero[Types].makeLibraryKeyHash(libraryID, key);
@ -3537,6 +3574,36 @@ Zotero.Sync.Server.Data = new function() {
docElem.appendChild(settingsNode);
}
// Add up to 500K characters of full-text content
try {
var rows = Zotero.Fulltext.getUnsyncedContent(500000);
}
catch (e) {
Zotero.debug(e, 1);
Components.utils.reportError(e);
var rows = [];
}
if (rows.length) {
let fulltextsNode = doc.createElement('fulltexts');
syncSession.fulltextItems = [];
for (let i=0; i<rows.length; i++) {
syncSession.fulltextItems.push({
libraryID: rows[i].libraryID,
key: rows[i].key
})
let node = doc.createElement('fulltext');
node.setAttribute('libraryID', rows[i].libraryID ? rows[i].libraryID : Zotero.libraryID);
node.setAttribute('key', rows[i].key);
node.setAttribute('indexedChars', rows[i].indexedChars);
node.setAttribute('totalChars', rows[i].totalChars);
node.setAttribute('indexedPages', rows[i].indexedPages);
node.setAttribute('totalPages', rows[i].totalPages);
node.appendChild(doc.createTextNode(_xmlize(rows[i].text)));
fulltextsNode.appendChild(node);
}
docElem.appendChild(fulltextsNode);
}
// Deletions
var deletedNode = doc.createElement('deleted');
var inserted = false;

1
resource/schema/userdata.sql
View File

@ -271,6 +271,7 @@ CREATE TABLE fulltextItems (
totalPages INT,
indexedChars INT,
totalChars INT,
synced INT DEFAULT 0,
FOREIGN KEY (itemID) REFERENCES items(itemID)
);
CREATE INDEX fulltextItems_version ON fulltextItems(version);