From 8931b3050dc21363b6762065c32276e47f82a49d Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Mon, 1 Feb 2010 08:28:52 +0000 Subject: [PATCH] - Fix error indexing documents containing words with invalid characters - Increase number of words inserted per statement up to max allowed --- chrome/content/zotero/xpcom/fulltext.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/chrome/content/zotero/xpcom/fulltext.js b/chrome/content/zotero/xpcom/fulltext.js index 7116fb8a5..cf742a1d9 100644 --- a/chrome/content/zotero/xpcom/fulltext.js +++ b/chrome/content/zotero/xpcom/fulltext.js @@ -199,7 +199,7 @@ Zotero.Fulltext = new function(){ var existing = []; var done = 0; - var maxWords = 500; + var maxWords = 999; // compiled limit var numWords = words.length; Zotero.DB.beginTransaction(); @@ -234,11 +234,17 @@ Zotero.Fulltext = new function(){ Zotero.DB.query("REPLACE INTO fulltextItems (itemID, version) VALUES (?,?)", [itemID, FULLTEXT_VERSION]); + // Handle bound parameters manually for optimal speed var statement1 = Zotero.DB.getStatement("INSERT INTO fulltextWords (word) VALUES (?)"); var statement2 = Zotero.DB.getStatement("INSERT OR IGNORE INTO fulltextItemWords VALUES (?,?)"); for each(var word in origWords) { + // Skip words containing invalid characters + if (word.match(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\ud800-\udfff\ufffe\uffff]/)) { + Zotero.debug("Skipping word '" + word + "' due to invalid characters"); + continue; + } if (existing['_' + word]){ var wordID = existing['_' + word]; }