From f7e411d56149b2aacf9f1f93351c523bac16613c Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Sat, 2 Jun 2018 04:07:05 -0400 Subject: [PATCH] Add support for databases in other directories Previously you could use Zotero.DBConnection to open another database in the data directory, but not one stored elsewhere in the filesystem. This allows an absolute path to be passed instead. Various operations (backups, corrupt DB recovery, pragma commands) are disabled for external databases. --- chrome/content/zotero/xpcom/db.js | 208 +++++++++++++------------- chrome/content/zotero/xpcom/zotero.js | 3 + 2 files changed, 111 insertions(+), 100 deletions(-) diff --git a/chrome/content/zotero/xpcom/db.js b/chrome/content/zotero/xpcom/db.js index 73c9ae182..b3886b622 100644 --- a/chrome/content/zotero/xpcom/db.js +++ b/chrome/content/zotero/xpcom/db.js @@ -31,8 +31,8 @@ // the same database is accessed simultaneously by multiple Zotero instances. const DB_LOCK_EXCLUSIVE = true; -Zotero.DBConnection = function(dbName) { - if (!dbName) { +Zotero.DBConnection = function(dbNameOrPath) { + if (!dbNameOrPath) { throw ('DB name not provided in Zotero.DBConnection()'); } @@ -70,8 +70,18 @@ Zotero.DBConnection = function(dbName) { return Zotero.Date.toUnixTimestamp(d); }); - // Private members - this._dbName = dbName; + // Absolute path to DB + if (dbNameOrPath.startsWith('/') || (Zotero.isWin && dbNameOrPath.includes('\\'))) { + this._dbName = OS.Path.basename(dbNameOrPath).replace(/\.sqlite$/, ''); + this._dbPath = dbNameOrPath; + this._externalDB = true; + } + // DB name in data directory + else { + this._dbName = dbNameOrPath; + this._dbPath = Zotero.DataDirectory.getDatabase(dbNameOrPath); + this._externalDB = false; + } this._shutdown = false; this._connection = null; this._transactionID = null; @@ -91,6 +101,14 @@ Zotero.DBConnection = function(dbName) { this._dbIsCorrupt = null this._transactionPromise = null; + + if (dbNameOrPath == 'zotero') { + this.IncompatibleVersionException = function (msg, dbClientVersion) { + this.message = msg; + this.dbClientVersion = dbClientVersion; + } + this.IncompatibleVersionException.prototype = Object.create(Error.prototype); + } } ///////////////////////////////////////////////////////////////// @@ -105,7 +123,7 @@ Zotero.DBConnection = function(dbName) { * @return void */ Zotero.DBConnection.prototype.test = function () { - return this._getConnectionAsync().return(); + return this._getConnectionAsync().then(() => {}); } Zotero.DBConnection.prototype.getAsyncStatement = Zotero.Promise.coroutine(function* (sql) { @@ -895,9 +913,13 @@ Zotero.DBConnection.prototype.integrityCheck = Zotero.Promise.coroutine(function Zotero.DBConnection.prototype.checkException = function (e) { + if (this._externalDB) { + return true; + } + if (e.message.includes(this.DB_CORRUPTION_STRING)) { // Write corrupt marker to data directory - var file = Zotero.File.pathToFile(Zotero.DataDirectory.getDatabase(this._dbName, 'is.corrupt')); + var file = Zotero.File.pathToFile(this._dbPath + '.is.corrupt'); Zotero.File.putContents(file, ''); this._dbIsCorrupt = true; @@ -948,6 +970,11 @@ Zotero.DBConnection.prototype.closeDatabase = Zotero.Promise.coroutine(function* Zotero.DBConnection.prototype.backupDatabase = Zotero.Promise.coroutine(function* (suffix, force) { + if (this.skipBackup || this._externalDB || Zotero.skipLoading) { + this._debug("Skipping backup of database '" + this._dbName + "'", 1); + return false; + } + var storageService = Components.classes["@mozilla.org/storage/service;1"] .getService(Components.interfaces.mozIStorageService); @@ -981,27 +1008,21 @@ Zotero.DBConnection.prototype.backupDatabase = Zotero.Promise.coroutine(function }); try { - var corruptMarker = Zotero.File.pathToFile( - Zotero.DataDirectory.getDatabase(this._dbName, 'is.corrupt') - ); + let corruptMarker = Zotero.File.pathToFile(this._dbPath + '.is.corrupt'); - if (this.skipBackup || Zotero.skipLoading) { - this._debug("Skipping backup of database '" + this._dbName + "'", 1); - return false; - } - else if (this._dbIsCorrupt || corruptMarker.exists()) { + if (this._dbIsCorrupt || corruptMarker.exists()) { this._debug("Database '" + this._dbName + "' is marked as corrupt -- skipping backup", 1); return false; } - var file = Zotero.File.pathToFile(Zotero.DataDirectory.getDatabase(this._dbName)); + let file = this._dbPath; // For standard backup, make sure last backup is old enough to replace if (!suffix && !force) { - var backupFile = Zotero.File.pathToFile(Zotero.DataDirectory.getDatabase(this._dbName, 'bak')); - if (yield OS.File.exists(backupFile.path)) { - var currentDBTime = (yield OS.File.stat(file.path)).lastModificationDate; - var lastBackupTime = (yield OS.File.stat(backupFile.path)).lastModificationDate; + let backupFile = this._dbPath + '.bak'; + if (yield OS.File.exists(backupFile)) { + let currentDBTime = (yield OS.File.stat(file.path)).lastModificationDate; + let lastBackupTime = (yield OS.File.stat(backupFile)).lastModificationDate; if (currentDBTime == lastBackupTime) { Zotero.debug("Database '" + this._dbName + "' hasn't changed -- skipping backup"); return; @@ -1022,7 +1043,7 @@ Zotero.DBConnection.prototype.backupDatabase = Zotero.Promise.coroutine(function // Copy via a temporary file so we don't run into disk space issues // after deleting the old backup file - var tmpFile = Zotero.DataDirectory.getDatabase(this._dbName, 'tmp'); + var tmpFile = this._dbPath + '.tmp'; if (yield OS.File.exists(tmpFile)) { try { yield OS.File.remove(tmpFile); @@ -1041,11 +1062,14 @@ Zotero.DBConnection.prototype.backupDatabase = Zotero.Promise.coroutine(function if (DB_LOCK_EXCLUSIVE) { yield this.queryAsync("PRAGMA main.locking_mode=NORMAL", false, { inBackup: true }); } - storageService.backupDatabaseFile(file, OS.Path.basename(tmpFile), file.parent); + storageService.backupDatabaseFile( + Zotero.File.pathToFile(file), + OS.Path.basename(tmpFile), + Zotero.File.pathToFile(file).parent + ); } catch (e) { - Zotero.debug(e); - Components.utils.reportError(e); + Zotero.logError(e); return false; } finally { @@ -1081,7 +1105,7 @@ Zotero.DBConnection.prototype.backupDatabase = Zotero.Promise.coroutine(function // Special backup if (!suffix && numBackups > 1) { // Remove oldest backup file - var targetFile = Zotero.DataDirectory.getDatabase(this._dbName, (numBackups - 1) + '.bak'); + let targetFile = this._dbPath + '.' + (numBackups - 1) + '.bak'; if (yield OS.File.exists(targetFile)) { yield OS.File.remove(targetFile); } @@ -1091,12 +1115,8 @@ Zotero.DBConnection.prototype.backupDatabase = Zotero.Promise.coroutine(function var targetNum = i; var sourceNum = targetNum - 1; - var targetFile = Zotero.DataDirectory.getDatabase( - this._dbName, targetNum + '.bak' - ); - var sourceFile = Zotero.DataDirectory.getDatabase( - this._dbName, sourceNum ? sourceNum + '.bak' : 'bak' - ); + let targetFile = this._dbPath + '.' + targetNum + '.bak'; + let sourceFile = this._dbPath + '.' + (sourceNum ? sourceNum + '.bak' : 'bak') if (!(yield OS.File.exists(sourceFile))) { continue; @@ -1108,9 +1128,7 @@ Zotero.DBConnection.prototype.backupDatabase = Zotero.Promise.coroutine(function } } - var backupFile = Zotero.DataDirectory.getDatabase( - this._dbName, (suffix ? suffix + '.' : '') + 'bak' - ); + let backupFile = this._dbPath + '.' + (suffix ? suffix + '.' : '') + 'bak'; // Remove old backup file if (yield OS.File.exists(backupFile)) { @@ -1147,11 +1165,11 @@ Zotero.DBConnection.prototype._getConnection = function (options) { /* * Retrieve a link to the data store asynchronously */ -Zotero.DBConnection.prototype._getConnectionAsync = Zotero.Promise.coroutine(function* (options) { +Zotero.DBConnection.prototype._getConnectionAsync = async function (options) { // If a backup is in progress, wait until it's done if (this._backupPromise && this._backupPromise.isPending() && (!options || !options.inBackup)) { Zotero.debug("Waiting for database backup to complete", 2); - yield this._backupPromise; + await this._backupPromise; } if (this._connection) { @@ -1162,48 +1180,50 @@ Zotero.DBConnection.prototype._getConnectionAsync = Zotero.Promise.coroutine(fun } this._debug("Asynchronously opening database '" + this._dbName + "'"); + Zotero.debug(this._dbPath); // Get the storage service var store = Components.classes["@mozilla.org/storage/service;1"]. getService(Components.interfaces.mozIStorageService); - var file = Zotero.File.pathToFile(Zotero.DataDirectory.getDatabase(this._dbName)); - var backupFile = Zotero.File.pathToFile(Zotero.DataDirectory.getDatabase(this._dbName, 'bak')); - - var fileName = this._dbName + '.sqlite'; + var file = this._dbPath; + var backupFile = this._dbPath + '.bak'; + var fileName = OS.Path.basename(file); + var corruptMarker = this._dbPath + '.is.corrupt'; catchBlock: try { - var corruptMarker = Zotero.File.pathToFile(Zotero.DataDirectory.getDatabase(this._dbName, 'is.corrupt')); - if (corruptMarker.exists()) { + if (await OS.File.exists(corruptMarker)) { throw new Error(this.DB_CORRUPTION_STRING); } - this._connection = yield Zotero.Promise.resolve(this.Sqlite.openConnection({ - path: file.path + this._connection = await Zotero.Promise.resolve(this.Sqlite.openConnection({ + path: file })); } catch (e) { + // Don't deal with corrupted external dbs + if (this._externalDB) { + throw e; + } + Zotero.logError(e); if (e.message.includes(this.DB_CORRUPTION_STRING)) { - this._debug("Database file '" + file.leafName + "' corrupted", 1); + this._debug(`Database file '${fileName}' corrupted`, 1); // No backup file! Eek! - if (!backupFile.exists()) { + if (!await OS.File.exists(backupFile)) { this._debug("No backup file for DB '" + this._dbName + "' exists", 1); // Save damaged filed this._debug('Saving damaged DB file with .damaged extension', 1); - var damagedFile = Zotero.File.pathToFile( - Zotero.DataDirectory.getDatabase(this._dbName, 'damaged') - ); + let damagedFile = this._dbPath + '.damaged'; Zotero.moveToUnique(file, damagedFile); // Create new main database - var file = Zotero.File.pathToFile(Zotero.DataDirectory.getDatabase(this._dbName)); this._connection = store.openDatabase(file); - if (corruptMarker.exists()) { - corruptMarker.remove(null); + if (await OS.File.exists(corruptMarker)) { + await OS.File.remove(corruptMarker); } Zotero.alert( @@ -1216,24 +1236,21 @@ Zotero.DBConnection.prototype._getConnectionAsync = Zotero.Promise.coroutine(fun // Save damaged file this._debug('Saving damaged DB file with .damaged extension', 1); - var damagedFile = Zotero.File.pathToFile( - Zotero.DataDirectory.getDatabase(this._dbName, 'damaged') - ); + let damagedFile = this._dbPath + '.damaged'; Zotero.moveToUnique(file, damagedFile); // Test the backup file try { Zotero.debug("Asynchronously opening DB connection"); - this._connection = yield Zotero.Promise.resolve(this.Sqlite.openConnection({ - path: backupFile.path + this._connection = await Zotero.Promise.resolve(this.Sqlite.openConnection({ + path: backupFile })); } // Can't open backup either catch (e) { // Create new main database - var file = Zotero.File.pathToFile(Zotero.DataDirectory.getDatabase(this._dbName)); - this._connection = yield Zotero.Promise.resolve(this.Sqlite.openConnection({ - path: file.path + this._connection = await Zotero.Promise.resolve(this.Sqlite.openConnection({ + path: file })); Zotero.alert( @@ -1242,8 +1259,8 @@ Zotero.DBConnection.prototype._getConnectionAsync = Zotero.Promise.coroutine(fun Zotero.getString('db.dbRestoreFailed', fileName) ); - if (corruptMarker.exists()) { - corruptMarker.remove(null); + if (await OS.File.exists(corruptMarker)) { + await OS.File.remove(corruptMarker); } break catchBlock; @@ -1254,7 +1271,7 @@ Zotero.DBConnection.prototype._getConnectionAsync = Zotero.Promise.coroutine(fun // Copy backup file to main DB file this._debug("Restoring database '" + this._dbName + "' from backup file", 1); try { - backupFile.copyTo(backupFile.parent, fileName); + await OS.File.copy(backupFile, file); } catch (e) { // TODO: deal with low disk space @@ -1262,8 +1279,7 @@ Zotero.DBConnection.prototype._getConnectionAsync = Zotero.Promise.coroutine(fun } // Open restored database - var file = OS.Path.join(Zotero.DataDirectory.dir, fileName); - this._connection = yield Zotero.Promise.resolve(this.Sqlite.openConnection({ + this._connection = await Zotero.Promise.resolve(this.Sqlite.openConnection({ path: file })); this._debug('Database restored', 1); @@ -1272,13 +1288,13 @@ Zotero.DBConnection.prototype._getConnectionAsync = Zotero.Promise.coroutine(fun Zotero.getString('general.warning'), Zotero.getString('db.dbRestored', [ fileName, - Zotero.Date.getFileDateString(backupFile), - Zotero.Date.getFileTimeString(backupFile) + Zotero.Date.getFileDateString(Zotero.File.pathToFile(backupFile)), + Zotero.Date.getFileTimeString(Zotero.File.pathToFile(backupFile)) ]) ); - if (corruptMarker.exists()) { - corruptMarker.remove(null); + if (await OS.File.exists(corruptMarker)) { + await OS.File.remove(corruptMarker); } break catchBlock; @@ -1288,44 +1304,36 @@ Zotero.DBConnection.prototype._getConnectionAsync = Zotero.Promise.coroutine(fun throw (e); } - if (DB_LOCK_EXCLUSIVE) { - yield this.queryAsync("PRAGMA main.locking_mode=EXCLUSIVE"); + if (!this._externalDB) { + if (DB_LOCK_EXCLUSIVE) { + await this.queryAsync("PRAGMA main.locking_mode=EXCLUSIVE"); + } + else { + await this.queryAsync("PRAGMA main.locking_mode=NORMAL"); + } + + // Set page cache size to 8MB + let pageSize = await this.valueQueryAsync("PRAGMA page_size"); + let cacheSize = 8192000 / pageSize; + await this.queryAsync("PRAGMA cache_size=" + cacheSize); + + // Enable foreign key checks + await this.queryAsync("PRAGMA foreign_keys=true"); + + // Register idle observer for DB backup + Zotero.Schema.schemaUpdatePromise.then(() => { + Zotero.debug("Initializing DB backup idle observer"); + var idleService = Components.classes["@mozilla.org/widget/idleservice;1"] + .getService(Components.interfaces.nsIIdleService); + idleService.addIdleObserver(this, 300); + }); } - else { - yield this.queryAsync("PRAGMA main.locking_mode=NORMAL"); - } - - // Set page cache size to 8MB - var pageSize = yield this.valueQueryAsync("PRAGMA page_size"); - var cacheSize = 8192000 / pageSize; - yield this.queryAsync("PRAGMA cache_size=" + cacheSize); - - // Enable foreign key checks - yield this.queryAsync("PRAGMA foreign_keys=true"); - - // Register idle observer for DB backup - Zotero.Schema.schemaUpdatePromise.then(() => { - Zotero.debug("Initializing DB backup idle observer"); - var idleService = Components.classes["@mozilla.org/widget/idleservice;1"] - .getService(Components.interfaces.nsIIdleService); - idleService.addIdleObserver(this, 300); - }); return this._connection; -}); +}; Zotero.DBConnection.prototype._debug = function (str, level) { var prefix = this._dbName == 'zotero' ? '' : '[' + this._dbName + '] '; Zotero.debug(prefix + str, level); } - - -// Initialize main database connection -Zotero.DB = new Zotero.DBConnection('zotero'); - -Zotero.DB.IncompatibleVersionException = function (msg, dbClientVersion) { - this.message = msg; - this.dbClientVersion = dbClientVersion; -} -Zotero.DB.IncompatibleVersionException.prototype = Object.create(Error.prototype); diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index e15ea10ca..a0947cf36 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -877,6 +877,9 @@ Services.scriptloader.loadSubScript("resource://zotero/polyfill.js"); * Initializes the DB connection */ var _initDB = Zotero.Promise.coroutine(function* (haveReleasedLock) { + // Initialize main database connection + Zotero.DB = new Zotero.DBConnection('zotero'); + try { // Test read access yield Zotero.DB.test();