From d8f3be4beec56efe1f58727242458e0b2894ac7e Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Sun, 22 Mar 2015 02:06:40 -0400 Subject: [PATCH] updateBundledStyles() asyncification and related changes - Use async DB and OS.File for bundled file updates - Remove support for translator/style ZIP files -- the two options are now non-unpacked XPIs with subfolders or unpacked source installations - Now that we have async file access, don't store translator code in database cache -- just store metadata so that it's available without reading each translator file - Change the (previously partially asyncified) Zotero.Styles/Translators APIs a bit -- while the getAll/getVisible methods are asynchronous and will wait for loading, the get() methods are synchronous and require styles/translators to be initialized before they're called. Most places that end up calling get() probably call getAll/getVisible first and should therefore be async, but if there's any way to trigger a get() first, that will need to be adjusted. - Asyncify various other style/translator-related code XPI support is untested, as is style/translator usage, so there are almost certainly bugs. The latter depends on updated export format support (#659), since toArray() no longer exists on this branch. Addresses #529 and #520 --- .../preferences/preferences_advanced.js | 9 +- .../zotero/preferences/preferences_cite.js | 30 +- .../zotero/preferences/preferences_export.js | 61 +- .../zotero/preferences/preferences_general.js | 41 +- chrome/content/zotero/xpcom/file.js | 46 +- chrome/content/zotero/xpcom/schema.js | 999 +++++++++--------- chrome/content/zotero/xpcom/style.js | 564 +++++----- .../zotero/xpcom/translation/translator.js | 18 +- .../zotero/xpcom/translation/translators.js | 196 ++-- chrome/content/zotero/xpcom/zotero.js | 5 +- resource/schema/userdata.sql | 7 +- 11 files changed, 1030 insertions(+), 946 deletions(-) diff --git a/chrome/content/zotero/preferences/preferences_advanced.js b/chrome/content/zotero/preferences/preferences_advanced.js index 0388bd8d3..255ec78ed 100644 --- a/chrome/content/zotero/preferences/preferences_advanced.js +++ b/chrome/content/zotero/preferences/preferences_advanced.js @@ -134,8 +134,7 @@ Zotero_Preferences.Advanced = { if (Zotero_Preferences.Export) { Zotero_Preferences.Export.populateQuickCopyList(); } - }) - .done(); + }); } }, @@ -160,8 +159,7 @@ Zotero_Preferences.Advanced = { if (Zotero_Preferences.Export) { Zotero_Preferences.Export.populateQuickCopyList(); } - }) - .done(); + }); } }, @@ -186,8 +184,7 @@ Zotero_Preferences.Advanced = { if (Zotero_Preferences.Export) { Zotero_Preferences.Export.populateQuickCopyList(); } - }) - .done(); + }); } }, diff --git a/chrome/content/zotero/preferences/preferences_cite.js b/chrome/content/zotero/preferences/preferences_cite.js index b5c3b2d7f..128dd8583 100644 --- a/chrome/content/zotero/preferences/preferences_cite.js +++ b/chrome/content/zotero/preferences/preferences_cite.js @@ -26,10 +26,10 @@ "use strict"; Zotero_Preferences.Cite = { - init: function () { + init: Zotero.Promise.coroutine(function* () { this.updateWordProcessorInstructions(); - this.refreshStylesList(); - }, + yield this.refreshStylesList(); + }), /** @@ -48,8 +48,9 @@ Zotero_Preferences.Cite = { /** * Refreshes the list of styles in the styles pane * @param {String} cslID Style to select + * @return {Promise} */ - refreshStylesList: function (cslID) { + refreshStylesList: Zotero.Promise.coroutine(function* (cslID) { Zotero.debug("Refreshing styles list"); var treechildren = document.getElementById('styleManager-rows'); @@ -57,11 +58,9 @@ Zotero_Preferences.Cite = { treechildren.removeChild(treechildren.firstChild); } - var styles = Zotero.Styles.getVisible(); - + var styles = yield Zotero.Styles.getVisible(); var selectIndex = false; - var i = 0; - for each(var style in styles) { + styles.forEach(function (style, i) { var treeitem = document.createElement('treeitem'); var treerow = document.createElement('treerow'); var titleCell = document.createElement('treecell'); @@ -86,9 +85,8 @@ Zotero_Preferences.Cite = { if (cslID == style.styleID) { document.getElementById('styleManager').view.selection.select(i); } - i++; - } - }, + }); + }), /** @@ -112,7 +110,7 @@ Zotero_Preferences.Cite = { /** * Deletes selected styles from the styles pane **/ - deleteStyle: function () { + deleteStyle: Zotero.Promise.coroutine(function* () { // get selected cslIDs var tree = document.getElementById('styleManager'); var treeItems = tree.lastChild.childNodes; @@ -141,17 +139,17 @@ Zotero_Preferences.Cite = { if(ps.confirm(null, '', text)) { // delete if requested if(cslIDs.length == 1) { - selectedStyle.remove(); + yield selectedStyle.remove(); } else { for(var i=0; i} A Q promise for TRUE if file was deleted, - * FALSE if missing + * @return {Promise} A promise for TRUE if file was deleted, FALSE if missing */ - this.deleteIfExists = function deleteIfExists(path) { + this.removeIfExists = function (path) { return Zotero.Promise.resolve(OS.File.remove(path)) - .thenResolve(true) + .return(true) .catch(function (e) { if (e instanceof OS.File.Error && e.becauseNoSuchFile) { return false; } + Zotero.debug(path, 1); throw e; }); } @@ -575,6 +584,19 @@ Zotero.File = new function(){ } + this.createDirectoryIfMissingAsync = function (path) { + return Zotero.Promise.resolve( + OS.File.makeDir( + path, + { + ignoreExisting: true, + unixMode: 0755 + } + ) + ); + } + + /** * Check whether a directory is an ancestor directory of another directory/file */ diff --git a/chrome/content/zotero/xpcom/schema.js b/chrome/content/zotero/xpcom/schema.js index 1af00e4c9..5eb034439 100644 --- a/chrome/content/zotero/xpcom/schema.js +++ b/chrome/content/zotero/xpcom/schema.js @@ -71,7 +71,7 @@ Zotero.Schema = new function(){ /* * Checks if the DB schema exists and is up-to-date, updating if necessary */ - this.updateSchema = function () { + this.updateSchema = Zotero.Promise.coroutine(function* () { // TODO: Check database integrity first with Zotero.DB.integrityCheck() // 'userdata' is the last upgrade step run in _migrateUserDataSchema() based on the @@ -79,107 +79,104 @@ Zotero.Schema = new function(){ // // 'compatibility' is incremented manually by upgrade steps in order to break DB // compatibility with older versions. - return Zotero.Promise.all([this.getDBVersion('userdata'), this.getDBVersion('compatibility')]) - .spread(function (userdata, compatibility) { - if (!userdata) { - Zotero.debug('Database does not exist -- creating\n'); - return _initializeSchema() - .then(function() { + var versions = yield Zotero.Promise.all([ + this.getDBVersion('userdata'), this.getDBVersion('compatibility') + ]); + var [userdata, compatibility] = versions; + if (!userdata) { + Zotero.debug('Database does not exist -- creating\n'); + return _initializeSchema() + .then(function() { + Zotero.initializationPromise + .then(1000) + .then(function () { + return Zotero.Schema.updateBundledFiles(); + }) + .then(function () { return _schemaUpdateDeferred.resolve(true); }); - } - - // We don't handle upgrades from pre-Zotero 2.1 databases - if (userdata < 76) { - let msg = Zotero.getString('upgrade.nonupgradeableDB1') - + "\n\n" + Zotero.getString('upgrade.nonupgradeableDB2', "4.0"); - throw new Error(msg); - } - - if (compatibility > _maxCompatibility) { - throw new Error("Database is incompatible this Zotero version " - + "(" + compatibility + " > " + _maxCompatibility + ")"); - } - - return _getSchemaSQLVersion('userdata') - // If upgrading userdata, make backup of database first - .then(function (schemaVersion) { - if (userdata < schemaVersion) { - return Zotero.DB.backupDatabase(userdata, true); - } - }) - .then(function () { - return Zotero.DB.executeTransaction(function* (conn) { - var updated = yield _updateSchema('system'); - - // Update custom tables if they exist so that changes are in - // place before user data migration - if (Zotero.DB.tableExists('customItemTypes')) { - yield Zotero.Schema.updateCustomTables(updated); - } - updated = yield _migrateUserDataSchema(userdata); - yield _updateSchema('triggers'); - - return updated; - }.bind(this)) - .then(function (updated) { - // Populate combined tables for custom types and fields - // -- this is likely temporary - // - // We do this even if updated in case custom fields were - // changed during user data migration - return Zotero.Schema.updateCustomTables() - .then(function () { - if (updated) { - // Upgrade seems to have been a success -- delete any previous backups - var maxPrevious = userdata - 1; - var file = Zotero.getZoteroDirectory(); - var toDelete = []; - try { - var files = file.directoryEntries; - while (files.hasMoreElements()) { - var file = files.getNext(); - file.QueryInterface(Components.interfaces.nsIFile); - if (file.isDirectory()) { - continue; - } - var matches = file.leafName.match(/zotero\.sqlite\.([0-9]{2,})\.bak/); - if (!matches) { - continue; - } - if (matches[1]>=28 && matches[1]<=maxPrevious) { - toDelete.push(file); - } - } - for each(var file in toDelete) { - Zotero.debug('Removing previous backup file ' + file.leafName); - file.remove(false); - } - } - catch (e) { - Zotero.debug(e); - } - } - - // After a delay, start update of bundled files and repo updates - // - // ************** - // TEMP TEMP TEMP - // ************** - // - /*Zotero.initializationPromise - .delay(5000) - .then(function () Zotero.Schema.updateBundledFiles(null, false, true)) - .finally(function () { - _schemaUpdateDeferred.resolve(true); - })*/ - - return updated; - }); - }); }); + } + + // We don't handle upgrades from pre-Zotero 2.1 databases + if (userdata < 76) { + let msg = Zotero.getString('upgrade.nonupgradeableDB1') + + "\n\n" + Zotero.getString('upgrade.nonupgradeableDB2', "4.0"); + throw new Error(msg); + } + + if (compatibility > _maxCompatibility) { + throw new Error("Database is incompatible this Zotero version " + + "(" + compatibility + " > " + _maxCompatibility + ")"); + } + + var schemaVersion = yield _getSchemaSQLVersion('userdata'); + + // If upgrading userdata, make backup of database first + if (userdata < schemaVersion) { + yield Zotero.DB.backupDatabase(userdata, true); + } + + var updated = yield Zotero.DB.executeTransaction(function* (conn) { + var updated = yield _updateSchema('system'); + + // Update custom tables if they exist so that changes are in + // place before user data migration + if (Zotero.DB.tableExists('customItemTypes')) { + yield Zotero.Schema.updateCustomTables(updated); + } + updated = yield _migrateUserDataSchema(userdata); + yield _updateSchema('triggers'); + + return updated; + }.bind(this)); + + // Populate combined tables for custom types and fields + // -- this is likely temporary + // + // We do this even if updated in case custom fields were + // changed during user data migration + yield Zotero.Schema.updateCustomTables() + + if (updated) { + // Upgrade seems to have been a success -- delete any previous backups + var maxPrevious = userdata - 1; + var file = Zotero.getZoteroDirectory(); + var toDelete = []; + try { + var files = file.directoryEntries; + while (files.hasMoreElements()) { + var file = files.getNext(); + file.QueryInterface(Components.interfaces.nsIFile); + if (file.isDirectory()) { + continue; + } + var matches = file.leafName.match(/zotero\.sqlite\.([0-9]{2,})\.bak/); + if (!matches) { + continue; + } + if (matches[1]>=28 && matches[1]<=maxPrevious) { + toDelete.push(file); + } + } + for each(var file in toDelete) { + Zotero.debug('Removing previous backup file ' + file.leafName); + file.remove(false); + } + } + catch (e) { + Zotero.debug(e); + } + } + + Zotero.initializationPromise + .then(1000) + .then(function () { + return Zotero.Schema.updateBundledFiles(); }); - } + + return updated; + }); // This is mostly temporary @@ -393,244 +390,267 @@ Zotero.Schema = new function(){ * Update styles and translators in data directory with versions from * ZIP file (XPI) or directory (source) in extension directory * - * @param {String} [mode] 'translators' or 'styles' - * @param {Boolean} [skipDeleteUpdated] Skip updating of the file deleting version -- - * since deleting uses a single version table key, - * it should only be updated the last time through + * @param {String} [mode] - 'translators' or 'styles' * @return {Promise} */ - this.updateBundledFiles = function(mode, skipDeleteUpdate, runRemoteUpdateWhenComplete) { - if (_localUpdateInProgress) return Zotero.Promise.resolve(); + this.updateBundledFiles = Zotero.Promise.coroutine(function* (mode) { + if (_localUpdateInProgress) { + Zotero.debug("Bundled file update already in progress", 2); + return; + } - return Zotero.Promise.try(function () { - _localUpdateInProgress = true; + _localUpdateInProgress = true; + + try { + yield Zotero.proxyAuthComplete.delay(1000); - // Get path to addon and then call updateBundledFilesCallback + // Get path to add-on // Synchronous in Standalone if (Zotero.isStandalone) { - var jar = Components.classes["@mozilla.org/file/directory_service;1"] + var installLocation = Components.classes["@mozilla.org/file/directory_service;1"] .getService(Components.interfaces.nsIProperties) .get("AChrom", Components.interfaces.nsIFile).parent; - jar.append("zotero.jar"); - return _updateBundledFilesCallback(jar, mode, skipDeleteUpdate); + installLocation.append("zotero.jar"); + } + // Asynchronous in Firefox + else { + let resolve, reject; + let promise = new Zotero.Promise(function () { + resolve = arguments[0]; + reject = arguments[1]; + }); + Components.utils.import("resource://gre/modules/AddonManager.jsm"); + AddonManager.getAddonByID( + ZOTERO_CONFIG.GUID, + function (addon) { + try { + installLocation = addon.getResourceURI() + .QueryInterface(Components.interfaces.nsIFileURL).file; + } + catch (e) { + reject(e); + return; + } + resolve(); + } + ); + yield promise; + } + installLocation = installLocation.path; + + // Update files + switch (mode) { + case 'styles': + yield Zotero.Styles.init(); + var updated = yield _updateBundledFilesAtLocation(installLocation, mode); + + case 'translators': + yield Zotero.Translators.init(); + var updated = yield _updateBundledFilesAtLocation(installLocation, mode); + + default: + yield Zotero.Translators.init(); + let up1 = yield _updateBundledFilesAtLocation(installLocation, 'translators', true); + yield Zotero.Styles.init(); + let up2 = yield _updateBundledFilesAtLocation(installLocation, 'styles'); + var updated = up1 || up2; } - // Asynchronous in Firefox - var deferred = Zotero.Promise.defer(); - Components.utils.import("resource://gre/modules/AddonManager.jsm"); - AddonManager.getAddonByID( - ZOTERO_CONFIG['GUID'], - function(addon) { - var up = _updateBundledFilesCallback( - addon.getResourceURI().QueryInterface(Components.interfaces.nsIFileURL).file, - mode, - skipDeleteUpdate - ); - deferred.resolve(up); - } - ); - return deferred.promise; - }) - .then(function (updated) { - if (runRemoteUpdateWhenComplete) { - if (updated) { - if (Zotero.Prefs.get('automaticScraperUpdates')) { - Zotero.unlockPromise - .then(Zotero.proxyAuthComplete) - .delay(1000) - .then(function () Zotero.Schema.updateFromRepository(2)) - .done(); - } - } - else { - Zotero.unlockPromise - .then(Zotero.proxyAuthComplete) - .delay(1000) - .then(function () Zotero.Schema.updateFromRepository(false)) - .done(); - } - } - }); - } - - /** - * Callback to update bundled files, after finding the path to the Zotero install location - */ - function _updateBundledFilesCallback(installLocation, mode, skipDeleteUpdate) { - _localUpdateInProgress = false; - - if (!mode) { - var up1 = _updateBundledFilesCallback(installLocation, 'translators', true); - var up2 = _updateBundledFilesCallback(installLocation, 'styles', false); - return up1 || up2; + _schemaUpdateDeferred.resolve(true); + } + finally { + _localUpdateInProgress = false; } - var xpiZipReader, isUnpacked = installLocation.isDirectory(); + if (updated) { + if (Zotero.Prefs.get('automaticScraperUpdates')) { + yield Zotero.Schema.updateFromRepository(2); + } + } + else { + yield Zotero.Schema.updateFromRepository(false); + } + }); + + /** + * Update bundled files in a given location + * + * @param {String} installLocation - Path to XPI or source dir + * @param {'translators','styles'} mode + * @param {Boolean} [skipVersionUpdates=false] + */ + var _updateBundledFilesAtLocation = Zotero.Promise.coroutine(function* (installLocation, mode, skipVersionUpdates) { + Components.utils.import("resource://gre/modules/FileUtils.jsm"); + + var isUnpacked = (yield OS.File.stat(installLocation)).isDir; if(!isUnpacked) { - xpiZipReader = Components.classes["@mozilla.org/libjar/zip-reader;1"] + var xpiZipReader = Components.classes["@mozilla.org/libjar/zip-reader;1"] .createInstance(Components.interfaces.nsIZipReader); - xpiZipReader.open(installLocation); - + xpiZipReader.open(new FileUtils.File(installLocation)); + if(Zotero.isStandalone && !xpiZipReader.hasEntry("translators.index")) { // Symlinked dev Standalone build - var installLocation2 = installLocation.parent, - translatorsDir = installLocation2.clone(); - translatorsDir.append("translators"); - if(translatorsDir.exists()) { - installLocation = installLocation2; + let parentDir = OS.Path.dirname(installLocation); + let translatorsDir = OS.Path.join(parentDir, 'translators'); + if (yield OS.File.exists(translatorsDir)) { + installLocation = parentDir; isUnpacked = true; xpiZipReader.close(); } } + else { + var zipFileName = OS.Path.basename(installLocation); + } } switch (mode) { case "translators": var titleField = 'label'; var fileExt = ".js"; + var destDir = Zotero.getTranslatorsDirectory().path; break; case "styles": var titleField = 'title'; var fileExt = ".csl"; - var hiddenDir = Zotero.getStylesDirectory(); - hiddenDir.append('hidden'); + var destDir = Zotero.getStylesDirectory().path; + var hiddenDir = OS.Path.join(destDir, 'hidden'); break; default: - throw ("Invalid mode '" + mode + "' in Zotero.Schema.updateBundledFiles()"); + throw new Error("Invalid mode '" + mode + "'"); } - var modes = mode; - mode = mode.substr(0, mode.length - 1); - var Mode = mode[0].toUpperCase() + mode.substr(1); - var Modes = Mode + "s"; + var modeType = mode.substr(0, mode.length - 1); + var ModeType = Zotero.Utilities.capitalize(modeType); + var Mode = Zotero.Utilities.capitalize(mode); - var repotime = Zotero.File.getContentsFromURL("resource://zotero/schema/repotime.txt"); + var repotime = yield Zotero.File.getContentsFromURLAsync("resource://zotero/schema/repotime.txt"); var date = Zotero.Date.sqlToDate(repotime, true); repotime = Zotero.Date.toUnixTimestamp(date); var fileNameRE = new RegExp("^[^\.].+\\" + fileExt + "$"); - var destDir = Zotero["get" + Modes + "Directory"](); - // If directory is empty, force reinstall var forceReinstall = true; - var entries = destDir.directoryEntries; - while (entries.hasMoreElements()) { - var file = entries.getNext(); - file.QueryInterface(Components.interfaces.nsIFile); - if (!file.leafName.match(fileNameRE) || file.isDirectory()) { - continue; + let iterator = new OS.File.DirectoryIterator(destDir); + try { + outer: + while (true) { + let entries = yield iterator.nextBatch(10); + if (!entries.length) break; + for (let i = 0; i < entries.length; i++) { + let entry = entries[i]; + if (!entry.name.match(fileNameRE) || entry.isDir) { + continue; + } + // Not empty + forceReinstall = false; + break outer; + } } - // Not empty - forceReinstall = false; - break; + } + finally { + iterator.close(); } // // Delete obsolete files // var sql = "SELECT version FROM version WHERE schema='delete'"; - var lastVersion = Zotero.DB.valueQuery(sql); + var lastVersion = yield Zotero.DB.valueQueryAsync(sql); if(isUnpacked) { - var deleted = installLocation.clone(); - deleted.append('deleted.txt'); + var deleted = OS.Path.join(installLocation, 'deleted.txt'); // In source builds, deleted.txt is in the translators directory - if (!deleted.exists()) { - deleted = installLocation.clone(); - deleted.append('translators'); - deleted.append('deleted.txt'); - } - if (!deleted.exists()) { - deleted = false; + if (!(yield OS.File.exists(deleted))) { + deleted = OS.Path.join(installLocation, 'translators', 'deleted.txt'); + if (!(yield OS.File.exists(deleted))) { + deleted = false; + } } } else { var deleted = xpiZipReader.getInputStream("deleted.txt"); } - deleted = Zotero.File.getContents(deleted); + deleted = yield Zotero.File.getContentsAsync(deleted); deleted = deleted.match(/^([^\s]+)/gm); var version = deleted.shift(); if (!lastVersion || lastVersion < version) { var toDelete = []; - var entries = destDir.directoryEntries; - while (entries.hasMoreElements()) { - var file = entries.getNext(); - file.QueryInterface(Components.interfaces.nsIFile); - - if (!file.exists() // symlink to non-existent file - || file.isDirectory()) { - continue; - } - - switch (file.leafName) { - // Delete incorrectly named files saved via repo pre-1.5b3 - case 'ama': - case 'apa': - case 'apsa': - case 'asa': - case 'chicago-author-date': - case 'chicago-fullnote-bibliography': - case 'chicago-note': - case 'chicago-note-bibliography': - case 'harvard1': - case 'ieee': - case 'mhra': - case 'mhra_note_without_bibliography': - case 'mla': - case 'nature': - case 'nlm': - case 'vancouver': - - // Remove update script (included with 3.0 accidentally) - case 'update': - - // Delete renamed/obsolete files - case 'chicago-note.csl': - case 'mhra_note_without_bibliography.csl': - case 'mhra.csl': - case 'mla.csl': - toDelete.push(file); - continue; - - // Be a little more careful with this one, in case someone - // created a custom 'aaa' style - case 'aaa.csl': - var str = Zotero.File.getContents(file, false, 300); - if (str.indexOf("American Anthropological Association") != -1) { - toDelete.push(file); + let iterator = new OS.File.DirectoryIterator(destDir); + try { + while (true) { + let entries = yield iterator.nextBatch(10); + if (!entries.length) break; + for (let i = 0; i < entries.length; i++) { + let entry = entries[i]; + + if ((entry.isSymLink && !(yield OS.File.exists(entry.path))) // symlink to non-existent file + || entry.isDir) { + continue; } - continue; + + if (mode == 'styles') { + switch (entry.name) { + // Remove update script (included with 3.0 accidentally) + case 'update': + + // Delete renamed/obsolete files + case 'chicago-note.csl': + case 'mhra_note_without_bibliography.csl': + case 'mhra.csl': + case 'mla.csl': + toDelete.push(entry.path); + continue; + + // Be a little more careful with this one, in case someone + // created a custom 'aaa' style + case 'aaa.csl': + let str = yield Zotero.File.getContentsAsync(entry.path, false, 300); + if (str.indexOf("American Anthropological Association") != -1) { + toDelete.push(entry.path); + } + continue; + } + } + + if (forceReinstall || !entry.name.match(fileNameRE)) { + continue; + } + + if (mode == 'translators') { + // TODO: Change if the APIs change + let newObj = new Zotero[Mode].loadFromFile(entry.path); + if (deleted.indexOf(newObj[modeType + "ID"]) == -1) { + continue; + } + toDelete.push(entry.path); + } + } } - - if (forceReinstall || !file.leafName.match(fileNameRE)) { - continue; - } - - var newObj = new Zotero[Mode](file); - if (deleted.indexOf(newObj[mode + "ID"]) == -1) { - continue; - } - toDelete.push(file); + } + finally { + iterator.close(); } - for each(var file in toDelete) { - Zotero.debug("Deleting " + file.path); + for (let i = 0; i < toDelete.length; i++) { + let path = toDelete[i]; + Zotero.debug("Deleting " + path); try { - file.remove(false); + yield OS.File.remove(path); } catch (e) { - Zotero.debug(e); + Components.utils.reportError(e); + Zotero.debug(e, 1); } } - if (!skipDeleteUpdate) { - sql = "REPLACE INTO version (schema, version) VALUES ('delete', ?)"; - Zotero.DB.query(sql, version); + if (!skipVersionUpdates) { + let sql = "REPLACE INTO version (schema, version) VALUES ('delete', ?)"; + yield Zotero.DB.queryAsync(sql, version); } } @@ -638,107 +658,76 @@ Zotero.Schema = new function(){ // Update files // var sql = "SELECT version FROM version WHERE schema=?"; - var lastModTime = Zotero.DB.valueQuery(sql, modes); - - var zipFileName = modes + ".zip", zipFile; - if(isUnpacked) { - zipFile = installLocation.clone(); - zipFile.append(zipFileName); - if(!zipFile.exists()) zipFile = undefined; - } else { - if(xpiZipReader.hasEntry(zipFileName)) { - zipFile = xpiZipReader.getEntry(zipFileName); - } - } + var lastModTime = yield Zotero.DB.valueQueryAsync(sql, mode); // XPI installation - if (zipFile) { - var modTime = Math.round(zipFile.lastModifiedTime / 1000); + if (!isUnpacked) { + var modTime = Math.round( + (yield OS.File.stat(installLocation)).lastModificationDate.getTime() / 1000 + ); if (!forceReinstall && lastModTime && modTime <= lastModTime) { - Zotero.debug("Installed " + modes + " are up-to-date with " + zipFileName); + Zotero.debug("Installed " + mode + " are up-to-date with " + zipFileName); return false; } - Zotero.debug("Updating installed " + modes + " from " + zipFileName); + Zotero.debug("Updating installed " + mode + " from " + zipFileName); - if (mode == 'translator') { + let tmpDir = Zotero.getTempDirectory().path; + + if (mode == 'translators') { // Parse translators.index - var indexFile; - if(isUnpacked) { - indexFile = installLocation.clone(); - indexFile.append('translators.index'); - if (!indexFile.exists()) { - Components.utils.reportError("translators.index not found in Zotero.Schema.updateBundledFiles()"); - return false; - } - } else { - if(!xpiZipReader.hasEntry("translators.index")) { - Components.utils.reportError("translators.index not found in Zotero.Schema.updateBundledFiles()"); - return false; - } - var indexFile = xpiZipReader.getInputStream("translators.index"); + if (!xpiZipReader.hasEntry("translators.index")) { + Components.utils.reportError("translators.index not found"); + return false; } + let indexFile = xpiZipReader.getInputStream("translators.index"); - indexFile = Zotero.File.getContents(indexFile); + indexFile = yield Zotero.File.getContentsAsync(indexFile); indexFile = indexFile.split("\n"); - var index = {}; - for each(var line in indexFile) { + let index = {}; + for (let i = 0; i < indexFile.length; i++) { + let line = indexFile[i]; if (!line) { continue; } - var [fileName, translatorID, label, lastUpdated] = line.split(','); + let [translatorID, label, lastUpdated] = line.split(','); if (!translatorID) { - Components.utils.reportError("Invalid translatorID '" + translatorID + "' in Zotero.Schema.updateBundledFiles()"); + Components.utils.reportError("Invalid translatorID '" + translatorID + "'"); return false; } index[translatorID] = { label: label, lastUpdated: lastUpdated, - fileName: fileName, // Numbered JS file within ZIP extract: true }; } - var sql = "SELECT translatorJSON FROM translatorCache"; - var dbCache = Zotero.DB.columnQuery(sql); + let sql = "SELECT metadataJSON FROM translatorCache"; + let dbCache = yield Zotero.DB.columnQueryAsync(sql); // If there's anything in the cache, see what we actually need to extract if (dbCache) { - for each(var json in dbCache) { - var metadata = JSON.parse(json); - var id = metadata.translatorID; + for (let i = 0; i < dbCache.length; i++) { + let metadata = JSON.parse(dbCache[i]); + let id = metadata.translatorID; if (index[id] && index[id].lastUpdated == metadata.lastUpdated) { index[id].extract = false; } } } - } - - var zipReader = Components.classes["@mozilla.org/libjar/zip-reader;1"] - .createInstance(Components.interfaces.nsIZipReader); - if(isUnpacked) { - zipReader.open(zipFile); - } else { - zipReader.openInner(xpiZipReader, zipFileName); - } - var tmpDir = Zotero.getTempDirectory(); - - if (mode == 'translator') { - for (var translatorID in index) { + + for (let translatorID in index) { // Use index file and DB cache for translator entries, // extracting only what's necessary - var entry = index[translatorID]; + let entry = index[translatorID]; if (!entry.extract) { //Zotero.debug("Not extracting '" + entry.label + "' -- same version already in cache"); continue; } - var tmpFile = tmpDir.clone(); - tmpFile.append(entry.fileName); - if (tmpFile.exists()) { - tmpFile.remove(false); - } - zipReader.extract(entry.fileName, tmpFile); + let tmpFile = OS.Path.join(tmpDir, entry.fileName) + yield Zotero.File.removeIfExists(tmpFile); + xpiZipReader.extract("translators/" + entry.fileName, new FileUtils.File(tmpFile)); var existingObj = Zotero.Translators.get(translatorID); if (!existingObj) { @@ -746,185 +735,227 @@ Zotero.Schema = new function(){ } else { Zotero.debug("Updating translator '" + existingObj.label + "'"); - if (existingObj.file.exists()) { - existingObj.file.remove(false); - } + yield Zotero.File.removeIfExists(existingObj.path); } - var fileName = Zotero.Translators.getFileNameFromLabel( + let fileName = Zotero.Translators.getFileNameFromLabel( entry.label, translatorID ); - var destFile = destDir.clone(); - destFile.append(fileName); - if (destFile.exists()) { - var msg = "Overwriting translator with same filename '" - + fileName + "'"; - Zotero.debug(msg, 1); - Components.utils.reportError(msg + " in Zotero.Schema.updateBundledFiles()"); - destFile.remove(false); + let destFile = OS.Path.join(destDir, fileName); + try { + yield OS.File.move(tmpFile, destFile, { + noOverwrite: true + }); + } + catch (e) { + if (e instanceof OS.File.Error && e.becauseExists) { + // Could overwrite automatically, but we want to log this + let msg = "Overwriting translator with same filename '" + fileName + "'"; + Zotero.debug(msg, 1); + Components.utils.reportError(msg); + yield OS.File.move(tmpFile, destFile); + } + else { + throw e; + } } - - tmpFile.moveTo(destDir, fileName); - - Zotero.wait(); } } // Styles else { - var entries = zipReader.findEntries(null); + let entries = zipReader.findEntries('styles/*.csl'); while (entries.hasMore()) { - var entry = entries.getNext(); + let entry = entries.getNext(); + let fileName = entry.substr(7); // strip 'styles/' - var tmpFile = tmpDir.clone(); - tmpFile.append(entry); - if (tmpFile.exists()) { - tmpFile.remove(false); - } - zipReader.extract(entry, tmpFile); - var newObj = new Zotero[Mode](tmpFile); + let tmpFile = OS.Path.join(tmpDir, fileName); + yield Zotero.File.removeIfExists(tmpFile); + zipReader.extract(entry, new FileUtils.File(tmpFile)); + let code = yield Zotero.File.getContentsAsync(tmpFile); + let newObj = new Zotero.Style(code); - var existingObj = Zotero[Modes].get(newObj[mode + "ID"]); + let existingObj = Zotero.Styles.get(newObj[modeType + "ID"]); if (!existingObj) { - Zotero.debug("Installing " + mode + " '" + newObj[titleField] + "'"); + Zotero.debug("Installing style '" + newObj[titleField] + "'"); } else { Zotero.debug("Updating " + (existingObj.hidden ? "hidden " : "") - + mode + " '" + existingObj[titleField] + "'"); - if (existingObj.file.exists()) { - existingObj.file.remove(false); - } + + "style '" + existingObj[titleField] + "'"); + yield Zotero.File.removeIfExists(existingObj.path); } - var fileName = tmpFile.leafName; - if (!existingObj || !existingObj.hidden) { - tmpFile.moveTo(destDir, fileName); + yield OS.File.move(tmpFile, OS.Path.join(destDir, fileName)); } else { - tmpFile.moveTo(hiddenDir, fileName); + yield OS.File.move(tmpFile, OS.Path.join(hiddenDir, fileName)); } - - Zotero.wait(); } } - zipReader.close(); if(xpiZipReader) xpiZipReader.close(); } // Source installation else { - var sourceDir = installLocation.clone(); - sourceDir.append(modes); - if (!sourceDir.exists()) { - Components.utils.reportError("No " + modes + " ZIP file or directory " - + " in Zotero.Schema.updateBundledFiles()"); - return false; - } + let sourceDir = OS.Path.join(installLocation, mode); - var entries = sourceDir.directoryEntries; var modTime = 0; - var sourceFilesExist = false; - while (entries.hasMoreElements()) { - var file = entries.getNext(); - file.QueryInterface(Components.interfaces.nsIFile); - // File might not exist in an source build with style symlinks - if (!file.exists() - || !file.leafName.match(fileNameRE) - || file.isDirectory()) { - continue; + let sourceFilesExist = false; + let iterator; + try { + iterator = new OS.File.DirectoryIterator(sourceDir); + } + catch (e) { + if (e instanceof OS.File.Error && e.becauseNoSuchFile) { + let msg = "No " + mode + " directory"; + Zotero.debug(msg, 1); + Components.utils.reportError(msg); + return false; } - sourceFilesExist = true; - var fileModTime = Math.round(file.lastModifiedTime / 1000); - if (fileModTime > modTime) { - modTime = fileModTime; + throw e; + } + try { + while (true) { + let entries = yield iterator.nextBatch(10); // TODO: adjust as necessary + if (!entries.length) break; + for (let i = 0; i < entries.length; i++) { + let entry = entries[i]; + if (!entry.name.match(fileNameRE) || entry.isDir) { + continue; + } + sourceFilesExist = true; + let d; + if ('winLastWriteDate' in entry) { + d = entry.winLastWriteDate; + } + else { + d = (yield OS.File.stat(entry.path)).lastModificationDate; + } + let fileModTime = Math.round(d.getTime() / 1000); + if (fileModTime > modTime) { + modTime = fileModTime; + } + } } } + finally { + iterator.close(); + } // Don't attempt installation for source build with missing styles if (!sourceFilesExist) { - Zotero.debug("No source " + mode + " files exist -- skipping update"); + Zotero.debug("No source " + modeType + " files exist -- skipping update"); return false; } if (!forceReinstall && lastModTime && modTime <= lastModTime) { - Zotero.debug("Installed " + modes + " are up-to-date with " + modes + " directory"); + Zotero.debug("Installed " + mode + " are up-to-date with " + mode + " directory"); return false; } - Zotero.debug("Updating installed " + modes + " from " + modes + " directory"); + Zotero.debug("Updating installed " + mode + " from " + mode + " directory"); - var entries = sourceDir.directoryEntries; - while (entries.hasMoreElements()) { - var file = entries.getNext(); - file.QueryInterface(Components.interfaces.nsIFile); - if (!file.exists() || !file.leafName.match(fileNameRE) || file.isDirectory()) { - continue; - } - var newObj = new Zotero[Mode](file); - var existingObj = Zotero[Modes].get(newObj[mode + "ID"]); - if (!existingObj) { - Zotero.debug("Installing " + mode + " '" + newObj[titleField] + "'"); - } - else { - Zotero.debug("Updating " - + (existingObj.hidden ? "hidden " : "") - + mode + " '" + existingObj[titleField] + "'"); - if (existingObj.file.exists()) { - existingObj.file.remove(false); - } - } - - if (mode == 'translator') { - var fileName = Zotero.Translators.getFileNameFromLabel( - newObj[titleField], newObj.translatorID - ); - } - else if (mode == 'style') { - var fileName = file.leafName; - } - - try { - var destFile = destDir.clone(); - destFile.append(fileName); - if (destFile.exists()) { - var msg = "Overwriting " + mode + " with same filename '" - + fileName + "'"; - Zotero.debug(msg, 1); - Components.utils.reportError(msg + " in Zotero.Schema.updateBundledFiles()"); - destFile.remove(false); - } + iterator = new OS.File.DirectoryIterator(sourceDir); + try { + while (true) { + let entries = yield iterator.nextBatch(10); // TODO: adjust as necessary + if (!entries.length) break; - if (!existingObj || !existingObj.hidden) { - file.copyTo(destDir, fileName); - } - else { - file.copyTo(hiddenDir, fileName); + for (let i = 0; i < entries.length; i++) { + let entry = entries[i]; + if (!entry.name.match(fileNameRE) || entry.isDir) { + continue; + } + let newObj; + if (mode == 'styles') { + let code = yield Zotero.File.getContentsAsync(entry.path); + newObj = new Zotero.Style(code); + } + else if (mode == 'translators') { + newObj = yield Zotero.Translators.loadFromFile(entry.path); + } + let existingObj = Zotero[Mode].get(newObj[modeType + "ID"]); + if (!existingObj) { + Zotero.debug("Installing " + modeType + " '" + newObj[titleField] + "'"); + } + else { + Zotero.debug("Updating " + + (existingObj.hidden ? "hidden " : "") + + modeType + " '" + existingObj[titleField] + "'"); + yield Zotero.File.removeIfExists(existingObj.path); + } + + let fileName; + if (mode == 'translators') { + fileName = Zotero.Translators.getFileNameFromLabel( + newObj[titleField], newObj.translatorID + ); + } + else if (mode == 'styles') { + fileName = entry.name; + } + + try { + let destFile = OS.Path.join(destDir, fileName); + + try { + if (!existingObj || !existingObj.hidden) { + yield OS.File.copy(entry.path, destFile, { + noOverwrite: true + }); + } + else { + yield OS.File.copy(entry.path, OS.Path.join(hiddenDir, fileName), { + noOverwrite: true + }); + } + } + catch (e) { + if (e instanceof OS.File.Error && e.becauseExists) { + // Could overwrite automatically, but we want to log this + let msg = "Overwriting " + modeType + " with same filename " + + "'" + fileName + "'"; + Zotero.debug(msg, 1); + Components.utils.reportError(msg); + if (!existingObj || !existingObj.hidden) { + yield OS.File.copy(entry.path, destFile); + } + else { + yield OS.File.copy(entry.path, OS.Path.join(hiddenDir, fileName)); + } + } + else { + throw e; + } + } + } + catch (e) { + Components.utils.reportError("Error copying file " + fileName + ": " + e); + } } } - catch (e) { - Components.utils.reportError("Error copying file " + fileName + ": " + e); - } - - Zotero.wait(); + } + finally { + iterator.close(); } } - Zotero.DB.beginTransaction(); + yield Zotero.DB.executeTransaction(function* () { + var sql = "REPLACE INTO version VALUES (?, ?)"; + yield Zotero.DB.queryAsync(sql, [mode, modTime]); + + if (!skipVersionUpdates) { + sql = "REPLACE INTO version VALUES ('repository', ?)"; + yield Zotero.DB.queryAsync(sql, repotime); + } + }); - var sql = "REPLACE INTO version VALUES (?, ?)"; - Zotero.DB.query(sql, [modes, modTime]); - - var sql = "REPLACE INTO version VALUES ('repository', ?)"; - Zotero.DB.query(sql, repotime); - - Zotero.DB.commitTransaction(); - - Zotero[Modes].init(); + yield Zotero[Mode].reinit(); return true; - } + }); /** @@ -998,7 +1029,7 @@ Zotero.Schema = new function(){ } // Send list of installed styles - var styles = Zotero.Styles.getAll(); + var styles = yield Zotero.Styles.getAll(); var styleTimestamps = []; for (var id in styles) { var updated = Zotero.Date.sqlToDate(styles[id].updated); @@ -1016,7 +1047,7 @@ Zotero.Schema = new function(){ var body = 'styles=' + encodeURIComponent(JSON.stringify(styleTimestamps)); try { - let xmlhttp = Zotero.HTTP.promise("POST", ZOTERO_CONFIG.REPOSITORY_URL, { body: body }); + var xmlhttp = yield Zotero.HTTP.request("POST", url, { body: body }); return _updateFromRepositoryCallback(xmlhttp, !!force); } catch (e) { @@ -1027,12 +1058,18 @@ Zotero.Schema = new function(){ } else { Components.utils.reportError(e); + Zotero.debug(xmlhttp.status, 1); + Zotero.debug(xmlhttp.responseText, 1); Zotero.debug("Error updating from repository " + msg, 1); } // TODO: instead, add an observer to start and stop timer on online state change _setRepositoryTimer(ZOTERO_CONFIG.REPOSITORY_RETRY_INTERVAL); return; } + if (xmlhttp) { + Zotero.debug(xmlhttp.status, 1); + Zotero.debug(xmlhttp.responseText, 1); + } throw e; }; } @@ -1050,65 +1087,62 @@ Zotero.Schema = new function(){ } - this.resetTranslatorsAndStyles = function (callback) { + this.resetTranslatorsAndStyles = Zotero.Promise.coroutine(function* () { Zotero.debug("Resetting translators and styles"); var sql = "DELETE FROM version WHERE schema IN " + "('translators', 'styles', 'repository', 'lastcheck')"; - Zotero.DB.query(sql); + yield Zotero.DB.queryAsync(sql); _dbVersions.repository = null; _dbVersions.lastcheck = null; var translatorsDir = Zotero.getTranslatorsDirectory(); + var stylesDir = Zotero.getStylesDirectory(); + translatorsDir.remove(true); - Zotero.getTranslatorsDirectory(); // recreate directory - return Zotero.Translators.reinit() - .then(function () self.updateBundledFiles('translators', null, false)) - .then(function () { - var stylesDir = Zotero.getStylesDirectory(); - stylesDir.remove(true); - Zotero.getStylesDirectory(); // recreate directory - Zotero.Styles.init(); - return self.updateBundledFiles('styles', null, true); - }) - .then(callback); - } + stylesDir.remove(true); + + // Recreate directories + Zotero.getTranslatorsDirectory(); + Zotero.getStylesDirectory(); + + yield Zotero.Promise.all(Zotero.Translators.reinit(), Zotero.Styles.reinit()); + yield this.updateBundledFiles(); + }); - this.resetTranslators = function (callback, skipUpdate) { + this.resetTranslators = Zotero.Promise.coroutine(function* () { Zotero.debug("Resetting translators"); var sql = "DELETE FROM version WHERE schema IN " + "('translators', 'repository', 'lastcheck')"; - Zotero.DB.query(sql); + yield Zotero.DB.queryAsync(sql); _dbVersions.repository = null; _dbVersions.lastcheck = null; var translatorsDir = Zotero.getTranslatorsDirectory(); translatorsDir.remove(true); Zotero.getTranslatorsDirectory(); // recreate directory - return Zotero.Translators.reinit() - .then(function () self.updateBundledFiles('translators', null, true)) - .then(callback); - } + yield Zotero.Translators.reinit(); + return this.updateBundledFiles('translators'); + }); - this.resetStyles = function (callback) { + this.resetStyles = Zotero.Promise.coroutine(function* () { Zotero.debug("Resetting translators and styles"); var sql = "DELETE FROM version WHERE schema IN " + "('styles', 'repository', 'lastcheck')"; - Zotero.DB.query(sql); + yield Zotero.DB.queryAsync(sql); _dbVersions.repository = null; _dbVersions.lastcheck = null; var stylesDir = Zotero.getStylesDirectory(); stylesDir.remove(true); Zotero.getStylesDirectory(); // recreate directory - return Zotero.Styles.init() - .then(function () self.updateBundledFiles('styles', null, true)) - .then(callback); - } + yield Zotero.Styles.reinit() + return this.updateBundledFiles('styles'); + }); this.integrityCheck = Zotero.Promise.coroutine(function* (fix) { @@ -1465,16 +1499,10 @@ Zotero.Schema = new function(){ .catch(function (e) { Zotero.debug(e, 1); Components.utils.reportError(e); - alert('Error initializing Zotero database'); + let ps = Components.classes["@mozilla.org/embedcomp/prompt-service;1"] + .getService(Components.interfaces.nsIPromptService); + ps.alert(null, Zotero.getString('general.error'), Zotero.getString('startupError')); throw e; - }) - .then(function () { - return Zotero.Schema.updateBundledFiles(null, null, true) - .catch(function (e) { - Zotero.debug(e); - Zotero.logError(e); - alert('Error updating Zotero translators and styles'); - }); }); } @@ -1523,6 +1551,8 @@ Zotero.Schema = new function(){ Zotero.debug('No network connection', 2); } else { + Zotero.debug(xmlhttp.status); + Zotero.debug(xmlhttp.responseText); Zotero.debug('Invalid response from repository', 2); } } @@ -1795,6 +1825,9 @@ Zotero.Schema = new function(){ yield Zotero.DB.queryAsync("ALTER TABLE libraries ADD COLUMN lastsync INT NOT NULL DEFAULT 0"); yield Zotero.DB.queryAsync("CREATE TABLE syncCache (\n libraryID INT NOT NULL,\n key TEXT NOT NULL,\n syncObjectTypeID INT NOT NULL,\n version INT NOT NULL,\n data TEXT,\n PRIMARY KEY (libraryID, key, syncObjectTypeID),\n FOREIGN KEY (libraryID) REFERENCES libraries(libraryID) ON DELETE CASCADE,\n FOREIGN KEY (syncObjectTypeID) REFERENCES syncObjectTypes(syncObjectTypeID)\n)"); + yield Zotero.DB.queryAsync("DROP TABLE translatorCache"); + yield Zotero.DB.queryAsync("CREATE TABLE translatorCache (\n fileName TEXT PRIMARY KEY,\n metadataJSON TEXT,\n lastModifiedTime INT\n);"); + yield Zotero.DB.queryAsync("DROP TRIGGER IF EXISTS fki_annotations_itemID_itemAttachments_itemID"); yield Zotero.DB.queryAsync("DROP TRIGGER IF EXISTS fku_annotations_itemID_itemAttachments_itemID"); yield Zotero.DB.queryAsync("DROP TRIGGER IF EXISTS fkd_annotations_itemID_itemAttachments_itemID"); diff --git a/chrome/content/zotero/xpcom/style.js b/chrome/content/zotero/xpcom/style.js index 633f3372f..095711acf 100644 --- a/chrome/content/zotero/xpcom/style.js +++ b/chrome/content/zotero/xpcom/style.js @@ -30,28 +30,50 @@ */ Zotero.Styles = new function() { var _initialized = false; - var _styles, _visibleStyles, _cacheTranslatorData; + var _styles, _visibleStyles; var _renamedStyles = null; - //Components.utils.import("resource://zotero/bluebird.js"); Components.utils.import("resource://gre/modules/Services.jsm"); + Components.utils.import("resource://gre/modules/FileUtils.jsm"); this.xsltProcessor = null; - this.ios = Components.classes["@mozilla.org/network/io-service;1"]. - getService(Components.interfaces.nsIIOService); - this.ns = { "csl":"http://purl.org/net/xbiblio/csl" }; - - // TEMP - // Until we get asynchronous style loading, load renamed styles at startup, since the - // synchronous call we were using breaks the first drag of the session (on OS X, at least) - this.preinit = function () { + + + /** + * Initializes styles cache, loading metadata for styles into memory + */ + this.reinit = Zotero.Promise.coroutine(function* () { + Zotero.debug("Initializing styles"); + var start = new Date; + _initialized = true; + + _styles = {}; + _visibleStyles = []; + this.lastCSL = null; + + // main dir + var dir = Zotero.getStylesDirectory().path; + var num = yield _readStylesFromDirectory(dir, false); + + // hidden dir + var hiddenDir = OS.Path.join(dir, 'hidden'); + if (yield OS.File.exists(hiddenDir)) { + num += yield _readStylesFromDirectory(hiddenDir, true); + } + + Zotero.debug("Cached " + num + " styles in " + (new Date - start) + " ms"); + _renamedStyles = {}; - Zotero.HTTP.promise( - "GET", "resource://zotero/schema/renamed-styles.json", { responseType: 'json' } + yield Zotero.HTTP.request( + "GET", + "resource://zotero/schema/renamed-styles.json", + { + responseType: 'json' + } ) .then(function (xmlhttp) { // Map some obsolete styles to current ones @@ -59,87 +81,70 @@ Zotero.Styles = new function() { _renamedStyles = xmlhttp.response; } }) - .done(); - } - - /** - * Initializes styles cache, loading metadata for styles into memory - */ - this.init = function() { - _initialized = true; - - var start = (new Date()).getTime() - - _styles = {}; - _visibleStyles = []; - _cacheTranslatorData = Zotero.Prefs.get("cacheTranslatorData"); - this.lastCSL = null; - - // main dir - var dir = Zotero.getStylesDirectory(); - var i = _readStylesFromDirectory(dir, false); - - // hidden dir - dir.append("hidden"); - if(dir.exists()) i += _readStylesFromDirectory(dir, true); - - Zotero.debug("Cached "+i+" styles in "+((new Date()).getTime() - start)+" ms"); - } + }); + this.init = Zotero.lazy(this.reinit); /** * Reads all styles from a given directory and caches their metadata * @private */ - function _readStylesFromDirectory(dir, hidden) { - var i = 0; - var contents = dir.directoryEntries; - while(contents.hasMoreElements()) { - var file = contents.getNext().QueryInterface(Components.interfaces.nsIFile), - filename = file.leafName; - if(!filename || filename[0] === "." - || filename.substr(-4).toLowerCase() !== ".csl" - || file.isDirectory()) continue; - - try { - var style = new Zotero.Style(file); - } - catch (e) { - Zotero.log( - "Error loading style '" + file.leafName + "': " + e.message, - "error", - file.path, - null, - e.lineNumber - ); - continue; - } - if(style.styleID) { - if(_styles[style.styleID]) { - // same style is already cached - Zotero.log('Style with ID '+style.styleID+' already loaded from "'+ - _styles[style.styleID].file.leafName+'"', "error", - Zotero.Styles.ios.newFileURI(style.file).spec); - } else { - // add to cache - _styles[style.styleID] = style; - _styles[style.styleID].hidden = hidden; - if(!hidden) _visibleStyles.push(style); + var _readStylesFromDirectory = Zotero.Promise.coroutine(function* (dir, hidden) { + var numCached = 0; + + var iterator = new OS.File.DirectoryIterator(dir); + try { + while (true) { + let entries = yield iterator.nextBatch(10); // TODO: adjust as necessary + if (!entries.length) break; + + for (let i = 0; i < entries.length; i++) { + let entry = entries[i]; + let path = entry.path; + let fileName = entry.name; + if (!fileName || fileName[0] === "." + || fileName.substr(-4).toLowerCase() !== ".csl" + || entry.isDir) continue; + + try { + let code = yield Zotero.File.getContentsAsync(path); + var style = new Zotero.Style(code, path); + } + catch (e) { + Components.utils.reportError(e); + Zotero.debug(e, 1); + continue; + } + if(style.styleID) { + // same style is already cached + if (_styles[style.styleID]) { + Components.utils.reportError('Style with ID ' + style.styleID + + ' already loaded from ' + _styles[style.styleID].fileName); + } else { + // add to cache + _styles[style.styleID] = style; + _styles[style.styleID].hidden = hidden; + if(!hidden) _visibleStyles.push(style); + } + } + numCached++; } } - i++; } - return i; - } + finally { + iterator.close(); + } + return numCached; + }); /** * Gets a style with a given ID * @param {String} id * @param {Boolean} skipMappings Don't automatically return renamed style */ - this.get = function(id, skipMappings) { - if(!_initialized) this.init(); - - // TODO: With asynchronous style loading, move renamedStyles call back here + this.get = function (id, skipMappings) { + if (!_initialized) { + throw new Zotero.Exception.UnloadedDataException("Styles not yet loaded", 'styles'); + } if(!skipMappings) { var prefix = "http://www.zotero.org/styles/"; @@ -156,20 +161,24 @@ Zotero.Styles = new function() { /** * Gets all visible styles - * @return {Zotero.Style[]} An array of Zotero.Style objects + * @return {Promise} A promise for an array of Zotero.Style objects */ - this.getVisible = function() { - if(!_initialized || !_cacheTranslatorData) this.init(); - return _visibleStyles.slice(0); + this.getVisible = function () { + return this.init().then(function () { + return _visibleStyles.slice(0); + }); } /** * Gets all styles - * @return {Object} An object whose keys are style IDs, and whose values are Zotero.Style objects + * + * @return {Promise} A promise for an object with style IDs for keys and + * Zotero.Style objects for values */ - this.getAll = function() { - if(!_initialized || !_cacheTranslatorData) this.init(); - return _styles; + this.getAll = function () { + return this.init().then(function () { + return _styles; + }); } /** @@ -200,8 +209,9 @@ Zotero.Styles = new function() { * @param {String} origin The origin of the style, either a filename or URL, to be * displayed in dialogs referencing the style */ - this.install = function(style, origin) { + this.install = Zotero.Promise.coroutine(function* (style, origin) { var styleInstalled; + if(style instanceof Components.interfaces.nsIFile) { // handle nsIFiles var url = Services.io.newFileURI(style); @@ -224,7 +234,7 @@ Zotero.Styles = new function() { origin, "styles.install.title", error)).present(); } }).done(); - } + }); /** * Installs a style @@ -234,176 +244,183 @@ Zotero.Styles = new function() { * @param {Boolean} [hidden] Whether style is to be hidden. * @return {Promise} */ - function _install(style, origin, hidden) { - if(!_initialized || !_cacheTranslatorData) Zotero.Styles.init(); + var _install = Zotero.Promise.coroutine(function* (style, origin, hidden) { + if (!_initialized) yield Zotero.Styles.init(); var existingFile, destFile, source, styleID - return Zotero.Promise.try(function() { - // First, parse style and make sure it's valid XML - var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"] - .createInstance(Components.interfaces.nsIDOMParser), - doc = parser.parseFromString(style, "application/xml"); - - styleID = Zotero.Utilities.xpathText(doc, '/csl:style/csl:info[1]/csl:id[1]', - Zotero.Styles.ns), - // Get file name from URL - m = /[^\/]+$/.exec(styleID), - fileName = Zotero.File.getValidFileName(m ? m[0] : styleID), - title = Zotero.Utilities.xpathText(doc, '/csl:style/csl:info[1]/csl:title[1]', - Zotero.Styles.ns); - - if(!styleID || !title) { - // If it's not valid XML, we'll return a promise that immediately resolves - // to an error - throw new Zotero.Exception.Alert("styles.installError", origin, - "styles.install.title", "Style is not valid XML, or the styleID or title is missing"); - } - - // look for a parent - source = Zotero.Utilities.xpathText(doc, - '/csl:style/csl:info[1]/csl:link[@rel="source" or @rel="independent-parent"][1]/@href', + + // First, parse style and make sure it's valid XML + var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"] + .createInstance(Components.interfaces.nsIDOMParser), + doc = parser.parseFromString(style, "application/xml"); + + styleID = Zotero.Utilities.xpathText(doc, '/csl:style/csl:info[1]/csl:id[1]', + Zotero.Styles.ns), + // Get file name from URL + m = /[^\/]+$/.exec(styleID), + fileName = Zotero.File.getValidFileName(m ? m[0] : styleID), + title = Zotero.Utilities.xpathText(doc, '/csl:style/csl:info[1]/csl:title[1]', Zotero.Styles.ns); - if(source == styleID) { - throw new Zotero.Exception.Alert("styles.installError", origin, - "styles.install.title", "Style references itself as source"); + + if(!styleID || !title) { + // If it's not valid XML, we'll return a promise that immediately resolves + // to an error + throw new Zotero.Exception.Alert("styles.installError", origin, + "styles.install.title", "Style is not valid XML, or the styleID or title is missing"); + } + + // look for a parent + source = Zotero.Utilities.xpathText(doc, + '/csl:style/csl:info[1]/csl:link[@rel="source" or @rel="independent-parent"][1]/@href', + Zotero.Styles.ns); + if(source == styleID) { + throw new Zotero.Exception.Alert("styles.installError", origin, + "styles.install.title", "Style references itself as source"); + } + + // ensure csl extension + if(fileName.substr(-4).toLowerCase() != ".csl") fileName += ".csl"; + + destFile = Zotero.getStylesDirectory(); + var destFileHidden = destFile.clone(); + destFile.append(fileName); + destFileHidden.append("hidden"); + if(hidden) Zotero.File.createDirectoryIfMissing(destFileHidden); + destFileHidden.append(fileName); + + // look for an existing style with the same styleID or filename + var existingTitle; + if(_styles[styleID]) { + existingFile = _styles[styleID].file; + existingTitle = _styles[styleID].title; + } else { + if(destFile.exists()) { + existingFile = destFile; + } else if(destFileHidden.exists()) { + existingFile = destFileHidden; } - // ensure csl extension - if(fileName.substr(-4).toLowerCase() != ".csl") fileName += ".csl"; - - destFile = Zotero.getStylesDirectory(); - var destFileHidden = destFile.clone(); - destFile.append(fileName); - destFileHidden.append("hidden"); - if(hidden) Zotero.File.createDirectoryIfMissing(destFileHidden); - destFileHidden.append(fileName); - - // look for an existing style with the same styleID or filename - var existingTitle; - if(_styles[styleID]) { - existingFile = _styles[styleID].file; - existingTitle = _styles[styleID].title; - } else { - if(destFile.exists()) { - existingFile = destFile; - } else if(destFileHidden.exists()) { - existingFile = destFileHidden; - } - - if(existingFile) { - // find associated style - for each(var existingStyle in _styles) { - if(destFile.equals(existingStyle.file)) { - existingTitle = existingStyle.title; - break; - } - } - } - } - - // also look for an existing style with the same title - if(!existingFile) { - for each(var existingStyle in Zotero.Styles.getAll()) { - if(title === existingStyle.title) { - existingFile = existingStyle.file; + if(existingFile) { + // find associated style + for each(var existingStyle in _styles) { + if(destFile.equals(existingStyle.file)) { existingTitle = existingStyle.title; break; } } } - - // display a dialog to tell the user we're about to install the style - if(hidden) { - destFile = destFileHidden; - } else { - if(existingTitle) { - var text = Zotero.getString('styles.updateStyle', [existingTitle, title, origin]); - } else { - var text = Zotero.getString('styles.installStyle', [title, origin]); - } - - var index = Services.prompt.confirmEx(null, Zotero.getString('styles.install.title'), - text, - ((Services.prompt.BUTTON_POS_0) * (Services.prompt.BUTTON_TITLE_IS_STRING) - + (Services.prompt.BUTTON_POS_1) * (Services.prompt.BUTTON_TITLE_CANCEL)), - Zotero.getString('general.install'), null, null, null, {} - ); - - if(index !== 0) { - throw new Zotero.Exception.UserCancelled("style installation"); - } - } + } - return Zotero.Styles.validate(style).catch(function(validationErrors) { - Zotero.logError("Style from "+origin+" failed to validate:\n\n"+validationErrors); - - // If validation fails on the parent of a dependent style, ignore it (for now) - if(hidden) return; - - // If validation fails on a different style, we ask the user if s/he really - // wants to install it - Components.utils.import("resource://gre/modules/Services.jsm"); - var shouldInstall = Services.prompt.confirmEx(null, - Zotero.getString('styles.install.title'), - Zotero.getString('styles.validationWarning', origin), - (Services.prompt.BUTTON_POS_0) * (Services.prompt.BUTTON_TITLE_OK) - + (Services.prompt.BUTTON_POS_1) * (Services.prompt.BUTTON_TITLE_CANCEL) - + Services.prompt.BUTTON_POS_1_DEFAULT + Services.prompt.BUTTON_DELAY_ENABLE, - null, null, null, null, {} - ); - if(shouldInstall !== 0) { - throw new Zotero.Exception.UserCancelled("style installation"); - } - }); - }).then(function() { - // User wants to install/update - if(source && !_styles[source]) { - // Need to fetch source - if(source.substr(0, 7) === "http://" || source.substr(0, 8) === "https://") { - return Zotero.HTTP.promise("GET", source).then(function(xmlhttp) { - return _install(xmlhttp.responseText, origin, true); - }).catch(function(error) { - if(typeof error === "object" && error instanceof Zotero.Exception.Alert) { - throw new Zotero.Exception.Alert("styles.installSourceError", [origin, source], - "styles.install.title", error); - } else { - throw error; - } - }); - } else { - throw new Zotero.Exception.Alert("styles.installSourceError", [origin, source], - "styles.install.title", "Source CSL URI is invalid"); + // also look for an existing style with the same title + if(!existingFile) { + let styles = yield Zotero.Styles.getAll(); + for (let i in styles) { + let existingStyle = styles[i]; + if(title === existingStyle.title) { + existingFile = existingStyle.file; + existingTitle = existingStyle.title; + break; } } - }).then(function() { - // Dependent style has been retrieved if there was one, so we're ready to - // continue + } + + // display a dialog to tell the user we're about to install the style + if(hidden) { + destFile = destFileHidden; + } else { + if(existingTitle) { + var text = Zotero.getString('styles.updateStyle', [existingTitle, title, origin]); + } else { + var text = Zotero.getString('styles.installStyle', [title, origin]); + } - // Remove any existing file with a different name - if(existingFile) existingFile.remove(false); + var index = Services.prompt.confirmEx(null, Zotero.getString('styles.install.title'), + text, + ((Services.prompt.BUTTON_POS_0) * (Services.prompt.BUTTON_TITLE_IS_STRING) + + (Services.prompt.BUTTON_POS_1) * (Services.prompt.BUTTON_TITLE_CANCEL)), + Zotero.getString('general.install'), null, null, null, {} + ); - return Zotero.File.putContentsAsync(destFile, style); - }).then(function() { - // Cache - Zotero.Styles.init(); + if(index !== 0) { + throw new Zotero.Exception.UserCancelled("style installation"); + } + } + + yield Zotero.Styles.validate(style) + .catch(function(validationErrors) { + Zotero.logError("Style from " + origin + " failed to validate:\n\n" + validationErrors); - // Refresh preferences windows - var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]. - getService(Components.interfaces.nsIWindowMediator); - var enumerator = wm.getEnumerator("zotero:pref"); - while(enumerator.hasMoreElements()) { - var win = enumerator.getNext(); - if(win.Zotero_Preferences.Cite) { - win.Zotero_Preferences.Cite.refreshStylesList(styleID); - } + // If validation fails on the parent of a dependent style, ignore it (for now) + if(hidden) return; + + // If validation fails on a different style, we ask the user if s/he really + // wants to install it + Components.utils.import("resource://gre/modules/Services.jsm"); + var shouldInstall = Services.prompt.confirmEx(null, + Zotero.getString('styles.install.title'), + Zotero.getString('styles.validationWarning', origin), + (Services.prompt.BUTTON_POS_0) * (Services.prompt.BUTTON_TITLE_OK) + + (Services.prompt.BUTTON_POS_1) * (Services.prompt.BUTTON_TITLE_CANCEL) + + Services.prompt.BUTTON_POS_1_DEFAULT + Services.prompt.BUTTON_DELAY_ENABLE, + null, null, null, null, {} + ); + if(shouldInstall !== 0) { + throw new Zotero.Exception.UserCancelled("style installation"); } }); - } + + // User wants to install/update + if(source && !_styles[source]) { + // Need to fetch source + if(source.substr(0, 7) === "http://" || source.substr(0, 8) === "https://") { + try { + let xmlhttp = yield Zotero.HTTP.request("GET", source); + yield _install(xmlhttp.responseText, origin, true); + } + catch (e) { + if (typeof e === "object" && e instanceof Zotero.Exception.Alert) { + throw new Zotero.Exception.Alert( + "styles.installSourceError", + [origin, source], + "styles.install.title", + e + ); + } + throw e; + } + } else { + throw new Zotero.Exception.Alert("styles.installSourceError", [origin, source], + "styles.install.title", "Source CSL URI is invalid"); + } + } + + // Dependent style has been retrieved if there was one, so we're ready to + // continue + + // Remove any existing file with a different name + if(existingFile) existingFile.remove(false); + + yield Zotero.File.putContentsAsync(destFile, style); + + yield Zotero.Styles.reinit(); + + // Refresh preferences windows + var wm = Components.classes["@mozilla.org/appshell/window-mediator;1"]. + getService(Components.interfaces.nsIWindowMediator); + var enumerator = wm.getEnumerator("zotero:pref"); + while(enumerator.hasMoreElements()) { + var win = enumerator.getNext(); + if(win.Zotero_Preferences.Cite) { + yield win.Zotero_Preferences.Cite.refreshStylesList(styleID); + } + } + }); } /** * @class Represents a style file and its metadata - * @property {nsIFile} file The path to the style file + * @property {String} path The path to the style file + * @property {String} fileName The name of the style file * @property {String} styleID * @property {String} url The URL where the style can be found (rel="self") * @property {String} type "csl" for CSL styles @@ -416,25 +433,25 @@ Zotero.Styles = new function() { * @property {Boolean} hidden True if this style is hidden in style selection dialogs, false if it * is not */ -Zotero.Style = function(arg) { - if(typeof arg === "string") { - this.string = arg; - } else if(typeof arg === "object") { - this.file = arg; - } else { - throw "Invalid argument passed to Zotero.Style"; +Zotero.Style = function (style, path) { + if (typeof style != "string") { + throw new Error("Style code must be a string"); } this.type = "csl"; - var style = typeof arg === "string" ? arg : Zotero.File.getContents(arg), - parser = Components.classes["@mozilla.org/xmlextras/domparser;1"] + var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"] .createInstance(Components.interfaces.nsIDOMParser), doc = parser.parseFromString(style, "application/xml"); if(doc.documentElement.localName === "parsererror") { throw new Error("File is not valid XML"); } - + + if (path) { + this.path = path; + this.fileName = OS.Path.basename(path); + } + this.styleID = Zotero.Utilities.xpathText(doc, '/csl:style/csl:info[1]/csl:id[1]', Zotero.Styles.ns); this.url = Zotero.Utilities.xpathText(doc, @@ -494,8 +511,10 @@ Zotero.Style.prototype.getCiteProc = function(automaticJournalAbbreviations) { if(this.source) { var parentStyle = Zotero.Styles.get(this.source); if(!parentStyle) { - throw(new Error('Style references '+this.source+', but this style is not installed', - Zotero.Styles.ios.newFileURI(this.file).spec, null)); + throw new Error( + 'Style references ' + this.source + ', but this style is not installed', + Zotero.Utilities.pathToFileURI(this.path) + ); } var version = parentStyle._version; @@ -552,11 +571,6 @@ Zotero.Style.prototype.getCiteProc = function(automaticJournalAbbreviations) { } }; -Zotero.Style.prototype.__defineGetter__("csl", function() { - Zotero.logError("Zotero.Style.csl is deprecated. Use Zotero.Style.getCiteProc()"); - return this.getCiteProc(); -}); - Zotero.Style.prototype.__defineGetter__("class", /** * Retrieves the style class, either from the metadata that's already loaded or by loading the file @@ -578,8 +592,7 @@ function() { // use hasBibliography from source style var parentStyle = Zotero.Styles.get(this.source); if(!parentStyle) { - throw(new Error('Style references '+this.source+', but this style is not installed', - Zotero.Styles.ios.newFileURI(this.file).spec, null)); + throw new Error('Style references missing parent ' + this.source); } return parentStyle.hasBibliography; } @@ -610,12 +623,11 @@ function() { // parent/child var formatCSL = Zotero.Styles.get(this.source); if(!formatCSL) { - throw(new Error('Style references '+this.source+', but this style is not installed', - Zotero.Styles.ios.newFileURI(this.file).spec, null)); + throw new Error('Style references missing parent ' + this.source); } - return formatCSL.file; - } else if(this.file) { - return this.file; + return formatCSL.path; + } else if (this.path) { + return this.path; } return null; }); @@ -633,15 +645,16 @@ Zotero.Style.prototype.getXML = function() { /** * Deletes a style */ -Zotero.Style.prototype.remove = function() { - if(!this.file) { - throw "Cannot delete a style with no associated file." +Zotero.Style.prototype.remove = Zotero.Promise.coroutine(function* () { + if (!this.path) { + throw new Error("Cannot delete a style with no associated file") } // make sure no styles depend on this one var dependentStyles = false; - var styles = Zotero.Styles.getAll(); - for each(var style in styles) { + var styles = yield Zotero.Styles.getAll(); + for (let i in styles) { + let style = styles[i]; if(style.source == this.styleID) { dependentStyles = true; break; @@ -650,13 +663,12 @@ Zotero.Style.prototype.remove = function() { if(dependentStyles) { // copy dependent styles to hidden directory - var hiddenDir = Zotero.getStylesDirectory(); - hiddenDir.append("hidden"); - Zotero.File.createDirectoryIfMissing(hiddenDir); - this.file.moveTo(hiddenDir, null); + let hiddenDir = OS.Path.join(Zotero.getStylesDirectory().path, 'hidden'); + yield Zotero.File.createDirectoryIfMissingAsync(hiddenDir); + yield OS.File.move(this.path, OS.Path.join(hiddenDir, OS.Path.basename(this.path))); } else { // remove defunct files - this.file.remove(false); + yield OS.File.remove(this.path); } // check to see if this style depended on a hidden one @@ -666,7 +678,9 @@ Zotero.Style.prototype.remove = function() { var deleteSource = true; // check to see if any other styles depend on the hidden one - for each(var style in Zotero.Styles.getAll()) { + let styles = yield Zotero.Styles.getAll(); + for (let i in styles) { + let style = styles[i]; if(style.source == this.source && style.styleID != this.styleID) { deleteSource = false; break; @@ -675,10 +689,10 @@ Zotero.Style.prototype.remove = function() { // if it was only this style with the dependency, delete the source if(deleteSource) { - source.remove(); + yield source.remove(); } } } - Zotero.Styles.init(); -} + return Zotero.Styles.reinit(); +}); diff --git a/chrome/content/zotero/xpcom/translation/translator.js b/chrome/content/zotero/xpcom/translation/translator.js index 25abd0ac3..88ba12d3b 100644 --- a/chrome/content/zotero/xpcom/translation/translator.js +++ b/chrome/content/zotero/xpcom/translation/translator.js @@ -61,7 +61,8 @@ var TRANSLATOR_SAVE_PROPERTIES = TRANSLATOR_REQUIRED_PROPERTIES.concat(["browser * @property {String} lastUpdated SQL-style date and time of translator's last update * @property {String} code The executable JavaScript for the translator * @property {Boolean} cacheCode Whether to cache code for this session (non-connector only) - * @property {nsIFile} [file] File corresponding to this translator (non-connector only) + * @property {String} [path] File path corresponding to this translator (non-connector only) + * @property {String} [fileName] File name corresponding to this translator (non-connector only) */ Zotero.Translator = function(info) { this.init(info); @@ -119,7 +120,10 @@ Zotero.Translator.prototype.init = function(info) { delete this.webRegexp; } - if(info.file) this.file = info.file; + if (info.path) { + this.path = info.path; + this.fileName = OS.Path.basename(info.path); + } if(info.code && this.cacheCode) { this.code = info.code; } else if(this.hasOwnProperty("code")) { @@ -148,7 +152,7 @@ Zotero.Translator.prototype.getCode = function() { return code; }); } else { - var promise = Zotero.File.getContentsAsync(this.file); + var promise = Zotero.File.getContentsAsync(this.path); if(this.cacheCode) { // Cache target-less web translators for session, since we // will use them a lot @@ -168,7 +172,7 @@ Zotero.Translator.prototype.serialize = function(properties) { var info = {}; for(var i in properties) { var property = properties[i]; - info[property] = translator[property]; + info[property] = this[property]; } return info; } @@ -182,10 +186,12 @@ Zotero.Translator.prototype.serialize = function(properties) { * @param {Integer} colNumber */ Zotero.Translator.prototype.logError = function(message, type, line, lineNumber, colNumber) { - if(Zotero.isFx && this.file) { + if (Zotero.isFx && this.path) { + Components.utils.import("resource://gre/modules/FileUtils.jsm"); + var file = new FileUtils.File(this.path); var ios = Components.classes["@mozilla.org/network/io-service;1"]. getService(Components.interfaces.nsIIOService); - Zotero.log(message, type ? type : "error", ios.newFileURI(this.file).spec); + Zotero.log(message, type ? type : "error", ios.newFileURI(file).spec); } else { Zotero.logError(message); } diff --git a/chrome/content/zotero/xpcom/translation/translators.js b/chrome/content/zotero/xpcom/translation/translators.js index 0a6625a77..f22fb627d 100644 --- a/chrome/content/zotero/xpcom/translation/translators.js +++ b/chrome/content/zotero/xpcom/translation/translators.js @@ -35,85 +35,113 @@ Zotero.Translators = new function() { var _initialized = false; /** - * Initializes translator cache, loading all relevant translators into memory + * Initializes translator cache, loading all translator metadata into memory */ this.reinit = Zotero.Promise.coroutine(function* () { - var start = (new Date()).getTime(); - var transactionStarted = false; + if (_initialized) { + Zotero.debug("Translators already initialized", 2); + return; + } + + Zotero.debug("Initializing translators"); + var start = new Date; + _initialized = true; _cache = {"import":[], "export":[], "web":[], "search":[]}; _translators = {}; - var dbCacheResults = yield Zotero.DB.queryAsync("SELECT leafName, translatorJSON, "+ - "code, lastModifiedTime FROM translatorCache"); + var sql = "SELECT fileName, metadataJSON, lastModifiedTime FROM translatorCache"; + var dbCacheResults = yield Zotero.DB.queryAsync(sql); var dbCache = {}; - for each(var cacheEntry in dbCacheResults) { - dbCache[cacheEntry.leafName] = cacheEntry; + for (let i = 0; i < dbCacheResults.length; i++) { + let entry = dbCacheResults[i]; + dbCache[entry.fileName] = entry; } - var i = 0; + var numCached = 0; var filesInCache = {}; - var contents = Zotero.getTranslatorsDirectory().directoryEntries; - while(contents.hasMoreElements()) { - var file = contents.getNext().QueryInterface(Components.interfaces.nsIFile); - var leafName = file.leafName; - if(!(/^[^.].*\.js$/.test(leafName))) continue; - var lastModifiedTime = file.lastModifiedTime; - - var dbCacheEntry = false; - if(dbCache[leafName]) { - filesInCache[leafName] = true; - if(dbCache[leafName].lastModifiedTime == lastModifiedTime) { - dbCacheEntry = dbCache[file.leafName]; - } - } - - if(dbCacheEntry) { - // get JSON from cache if possible - var translator = Zotero.Translators.load(file, dbCacheEntry.translatorJSON, dbCacheEntry.code); - filesInCache[leafName] = true; - } else { - // otherwise, load from file - var translator = yield Zotero.Translators.loadFromDisk(file); - } - - if(translator.translatorID) { - if(_translators[translator.translatorID]) { - // same translator is already cached - translator.logError('Translator with ID '+ - translator.translatorID+' already loaded from "'+ - _translators[translator.translatorID].file.leafName+'"'); - } else { - // add to cache - _translators[translator.translatorID] = translator; - for(var type in TRANSLATOR_TYPES) { - if(translator.translatorType & TRANSLATOR_TYPES[type]) { - _cache[type].push(translator); + var translatorsDir = Zotero.getTranslatorsDirectory().path; + var iterator = new OS.File.DirectoryIterator(translatorsDir); + try { + while (true) { + let entries = yield iterator.nextBatch(5); // TODO: adjust as necessary + if (!entries.length) break; + for (let i = 0; i < entries.length; i++) { + let entry = entries[i]; + let path = entry.path; + let fileName = entry.name; + + if (!(/^[^.].*\.js$/.test(fileName))) continue; + + let lastModifiedTime; + if ('winLastWriteDate' in entry) { + lastModifiedTime = entry.winLastWriteDate.getTime(); + } + else { + lastModifiedTime = (yield OS.File.stat(path)).lastModificationDate.getTime(); + } + let lastModified + + var dbCacheEntry = false; + if (dbCache[fileName]) { + filesInCache[fileName] = true; + if (dbCache[fileName].lastModifiedTime == lastModifiedTime) { + dbCacheEntry = dbCache[fileName]; } } - if(!dbCacheEntry) { - var code = yield translator.getCode(); - yield Zotero.Translators.cacheInDB( - leafName, - translator.serialize(TRANSLATOR_REQUIRED_PROPERTIES. - concat(TRANSLATOR_OPTIONAL_PROPERTIES)), - translator.cacheCode ? translator.code : null, - lastModifiedTime - ); - delete translator.metadataString; + if(dbCacheEntry) { + // get JSON from cache if possible + var translator = Zotero.Translators.load(dbCacheEntry.metadataJSON, path); + filesInCache[fileName] = true; + } else { + // otherwise, load from file + var translator = yield Zotero.Translators.loadFromFile(path); } + + // When can this happen? + if (!translator.translatorID) { + Zotero.debug("Translator ID for " + path + " not found"); + continue; + } + + if (_translators[translator.translatorID]) { + // same translator is already cached + translator.logError('Translator with ID '+ + translator.translatorID+' already loaded from "'+ + _translators[translator.translatorID].fileName + '"'); + } else { + // add to cache + _translators[translator.translatorID] = translator; + for(var type in TRANSLATOR_TYPES) { + if(translator.translatorType & TRANSLATOR_TYPES[type]) { + _cache[type].push(translator); + } + } + + if (!dbCacheEntry) { + yield Zotero.Translators.cacheInDB( + fileName, + translator.serialize(TRANSLATOR_REQUIRED_PROPERTIES. + concat(TRANSLATOR_OPTIONAL_PROPERTIES)), + lastModifiedTime + ); + } + } + + numCached++; } } - - i++; + } + finally { + iterator.close(); } // Remove translators from DB as necessary - for(var leafName in dbCache) { - if(!filesInCache[leafName]) { + for (let fileName in dbCache) { + if (!filesInCache[fileName]) { yield Zotero.DB.queryAsync( - "DELETE FROM translatorCache WHERE leafName = ?", [leafName] + "DELETE FROM translatorCache WHERE fileName = ?", fileName ); } } @@ -133,56 +161,55 @@ Zotero.Translators = new function() { _cache[type].sort(cmp); } - Zotero.debug("Cached "+i+" translators in "+((new Date()).getTime() - start)+" ms"); + Zotero.debug("Cached " + numCached + " translators in " + ((new Date) - start) + " ms"); }); this.init = Zotero.lazy(this.reinit); /** * Loads a translator from JSON, with optional code */ - this.load = function(file, json, code) { + this.load = function (json, path, code) { var info = JSON.parse(json); - info.file = file; + info.path = path; info.code = code; return new Zotero.Translator(info); } /** * Loads a translator from the disk + * + * @param {String} file - Path to translator file */ - this.loadFromDisk = function(file) { + this.loadFromFile = function(path) { const infoRe = /^\s*{[\S\s]*?}\s*?[\r\n]/; - return Zotero.File.getContentsAsync(file) + return Zotero.File.getContentsAsync(path) .then(function(source) { - return Zotero.Translators.load(file, infoRe.exec(source)[0], source); + return Zotero.Translators.load(infoRe.exec(source)[0], path, source); }) .catch(function() { - throw "Invalid or missing translator metadata JSON object in " + file.leafName; + throw "Invalid or missing translator metadata JSON object in " + OS.Path.basename(path); }); } /** * Gets the translator that corresponds to a given ID + * * @param {String} id The ID of the translator - * @param {Function} [callback] An optional callback to be executed when translators have been - * retrieved. If no callback is specified, translators are - * returned. */ this.get = function(id) { - return this.init().then(function() { - return _translators[id] ? _translators[id] : false - }); + if (!_initialized) { + throw new Zotero.Exception.UnloadedDataException("Translators not yet loaded", 'translators'); + } + return _translators[id] ? _translators[id] : false } /** * Gets all translators for a specific type of translation + * * @param {String} type The type of translators to get (import, export, web, or search) - * @param {Function} [callback] An optional callback to be executed when translators have been - * retrieved. If no callback is specified, translators are - * returned. */ this.getAllForType = function(type) { - return this.init().then(function() { + return this.init().then(function () { return _cache[type].slice(); }); } @@ -191,21 +218,16 @@ Zotero.Translators = new function() { * Gets all translators for a specific type of translation */ this.getAll = function() { - return this.init().then(function() { - return [translator for each(translator in _translators)]; + return this.init().then(function () { + return Object.keys(_translators); }); } /** * Gets web translators for a specific location * @param {String} uri The URI for which to look for translators - * @param {Function} [callback] An optional callback to be executed when translators have been - * retrieved. If no callback is specified, translators are - * returned. The callback is passed a set of functions for - * converting URLs from proper to proxied forms as the second - * argument. */ - this.getWebTranslatorsForLocation = function(uri, callback) { + this.getWebTranslatorsForLocation = function(uri) { return this.getAllForType("web").then(function(allTranslators) { var potentialTranslators = []; @@ -415,10 +437,10 @@ Zotero.Translators = new function() { }); } - this.cacheInDB = function(fileName, metadataJSON, code, lastModifiedTime) { + this.cacheInDB = function(fileName, metadataJSON, lastModifiedTime) { return Zotero.DB.queryAsync( - "REPLACE INTO translatorCache VALUES (?, ?, ?, ?)", - [fileName, metadataJSON, code, lastModifiedTime] + "REPLACE INTO translatorCache VALUES (?, ?, ?)", + [fileName, JSON.stringify(metadataJSON), lastModifiedTime] ); } } diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index 155fc2387..3c846a7dc 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -485,7 +485,7 @@ Components.utils.import("resource://gre/modules/osfile.jsm"); // Recreate database with no quick start guide Zotero.Schema.skipDefaultData = true; - Zotero.Schema.updateSchema(); + yield Zotero.Schema.updateSchema(); Zotero.restoreFromServer = true; } @@ -577,7 +577,6 @@ Components.utils.import("resource://gre/modules/osfile.jsm"); Zotero.locked = false; // Initialize various services - Zotero.Styles.preinit(); Zotero.Integration.init(); if(Zotero.Prefs.get("httpServer.enabled")) { @@ -605,8 +604,8 @@ Components.utils.import("resource://gre/modules/osfile.jsm"); Zotero.Searches.init(); Zotero.Groups.init(); - // TODO: Delay until after UI is shown yield Zotero.QuickCopy.init(); + Zotero.Items.startEmptyTrashTimer(); } catch (e) { diff --git a/resource/schema/userdata.sql b/resource/schema/userdata.sql index 106015ff9..1cd70833a 100644 --- a/resource/schema/userdata.sql +++ b/resource/schema/userdata.sql @@ -411,8 +411,7 @@ CREATE INDEX customBaseFieldMappings_baseFieldID ON customBaseFieldMappings(base CREATE INDEX customBaseFieldMappings_customFieldID ON customBaseFieldMappings(customFieldID); CREATE TABLE translatorCache ( - leafName TEXT PRIMARY KEY, - translatorJSON TEXT, - code TEXT, - lastModifiedTime INT + fileName TEXT PRIMARY KEY, + metadataJSON TEXT, + lastModifiedTime INT ); \ No newline at end of file