From 5a7d65532c90db3bc22d599a94d5b089b01eca35 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Thu, 17 May 2007 06:30:13 +0000 Subject: [PATCH] - Sped up item importing dramatically by doing import in a single transaction -- this means that an error will cause already-imported items to be discarded, but that's probably better anyway - The (empty) created collection is now deleted after import failure - Sped up general DB performance dramatically (especially on XP and Linux) by using a dummy statement to keep the SQLite shared cache active between transactions -- this seems to have only a moderate effect on OS X, which was already much faster, perhaps because of its own disk cache (though running XP and Linux under Parallels with limited RAM probably didn't help) - Added some additional sanity checks to Date.strToDate() to prevent errors in the metadata pane and on import with dates like "1998-99" -- strToDate() was treating 99 as the month, which caused an error in the stricter DB trigger. "99" is now left as an unparsed part, though "2006-08" (where the second part is <= 31) will still get parsed as August 2006. Can't do much about that, though. - Fixed a bug when rolling back nested transactions that could temporarily leave things somewhat funky --- chrome/content/zotero/fileInterface.js | 8 ++-- chrome/content/zotero/xpcom/db.js | 65 +++++++++++++++++++++++++- chrome/content/zotero/xpcom/schema.js | 2 - chrome/content/zotero/xpcom/zotero.js | 26 ++++++----- 4 files changed, 83 insertions(+), 18 deletions(-) diff --git a/chrome/content/zotero/fileInterface.js b/chrome/content/zotero/fileInterface.js index a66725628..7c591a276 100644 --- a/chrome/content/zotero/fileInterface.js +++ b/chrome/content/zotero/fileInterface.js @@ -226,8 +226,8 @@ var Zotero_File_Interface = new function() { Zotero_File_Interface.Progress.show( Zotero.getString("fileInterface.itemsImported"), function() { - // disable notifier - _unlock = Zotero.Notifier.begin(true); + Zotero.DB.beginTransaction(); + // translate translation.translate(); }); @@ -255,13 +255,13 @@ var Zotero_File_Interface = new function() { _importCollection.addItem(itemID); } - // notify - Zotero.Notifier.commit(_unlock); + Zotero.DB.commitTransaction(); Zotero_File_Interface.Progress.close(); Zotero.UnresponsiveScriptIndicator.enable(); if(!worked) { + _importCollection.erase(); window.alert(Zotero.getString("fileInterface.importError")); } } diff --git a/chrome/content/zotero/xpcom/db.js b/chrome/content/zotero/xpcom/db.js index 04d45c63f..d0436d5bb 100644 --- a/chrome/content/zotero/xpcom/db.js +++ b/chrome/content/zotero/xpcom/db.js @@ -304,7 +304,7 @@ Zotero.DBConnection.prototype.commitTransaction = function () { } else if (this._transactionRollback) { this._debug('Rolling back previously flagged transaction', 5); - db.rollbackTransaction(); + this.rollbackTransaction(); } else { this._debug('Committing transaction',5); @@ -672,6 +672,69 @@ Zotero.DBConnection.prototype.backupDatabase = function (suffix) { } +/* + * Keep the SQLite shared cache live between transactions with a dummy statement, + * which speeds up DB access dramatically (at least on Windows and Linux--OS X + * seems to be much faster already, perhaps due to its own disk cache) + * + * This is the same technique used by Mozilla code. The one downside is that it + * prevents schema changes, so this is called after schema updating. If the + * schema really needs to be updated at another point, use stopDummyStatement(). + * + * See http://developer.mozilla.org/en/docs/Storage:Performance for more info. + */ +Zotero.DBConnection.prototype.startDummyStatement = function () { + try { + if (!this._dummyConnection) { + this._debug("Opening database '" + this._dbName + " for dummy statement"); + // Get the storage service + var store = Components.classes["@mozilla.org/storage/service;1"]. + getService(Components.interfaces.mozIStorageService); + var file = Zotero.getZoteroDatabase(this._dbName); + this._dummyConnection = store.openDatabase(file); + } + + if (this._dummyStatement) { + Zotero.debug("Dummy statement is already open"); + return; + } + + Zotero.debug("Initializing dummy statement for '" + this._dbName + "'"); + + var sql = "CREATE TABLE IF NOT EXISTS zoteroDummyTable (id INTEGER PRIMARY KEY)"; + this._dummyConnection.executeSimpleSQL(sql); + + sql = "INSERT OR IGNORE INTO zoteroDummyTable VALUES (1)"; + this._dummyConnection.executeSimpleSQL(sql); + + sql = "SELECT id FROM zoteroDummyTable LIMIT 1" + this._dummyStatement = this._dummyConnection.createStatement(sql) + this._dummyStatement.executeStep() + + } + catch (e) { + Components.utils.reportError(e); + Zotero.debug(e); + } +} + + +/* + * Stop the dummy statement temporarily to allow for schema changess + * + * The statement needs to be started again or performance will suffer. + */ +Zotero.DBConnection.prototype.stopDummyStatement = function () { + if (!this._dummyStatement) { + return; + } + + Zotero.debug("Stopping dummy statement for '" + this._dbName + "'"); + this._dummyStatement.reset(); + this._dummyStatement = null; +} + + ///////////////////////////////////////////////////////////////// // // Private methods diff --git a/chrome/content/zotero/xpcom/schema.js b/chrome/content/zotero/xpcom/schema.js index 7de318994..d08c27509 100644 --- a/chrome/content/zotero/xpcom/schema.js +++ b/chrome/content/zotero/xpcom/schema.js @@ -1073,8 +1073,6 @@ Zotero.Schema = new function(){ } } var parsedID = parseInt(rows[j].itemID); - Zotero.debug(parsedID); - Zotero.debug(rows[j].itemID); if ((parsedID + '').length != rows[j].itemID) { if (parseInt(rows[j].note) != rows[j].note || (parseInt(rows[j].note) + '').length != rows[j].note.length) { diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index a373a86e5..bffa8ef57 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -212,6 +212,7 @@ var Zotero = new function(){ Zotero.Schema.updateSchema(); } + Zotero.DB.startDummyStatement(); Zotero.Schema.updateScrapersRemote(); // Initialize integration web server @@ -1220,7 +1221,7 @@ Zotero.Date = new function(){ } } - if(!date.month || date.month <= 12) { + if((!date.month || date.month <= 12) && (!date.day || date.day <= 31)) { if(date.year && date.year < 100) { // for two digit years, determine proper // four digit year var today = new Date(); @@ -1288,18 +1289,21 @@ Zotero.Date = new function(){ var m = _dayRe.exec(date.part); if(m) { - date.day = parseInt(m[1], 10); - - if(m.index > 0) { - date.part = date.part.substr(0, m.index); - if(m[2]) { - date.part += " "+m[2];; + var day = parseInt(m[1], 10); + // Sanity check + if (day <= 31) { + date.day = day; + if(m.index > 0) { + date.part = date.part.substr(0, m.index); + if(m[2]) { + date.part += " "+m[2];; + } + } else { + date.part = m[2]; } - } else { - date.part = m[2]; + + Zotero.debug("DATE: got day ("+date.day+", "+date.part+")"); } - - Zotero.debug("DATE: got day ("+date.day+", "+date.part+")"); } }