84bb61ab7b
When an export translator is selected for Quick Copy, Quick Copy initialization triggers translator initialization a few seconds after startup, because the translator code needs to be available synchronously for drag/drop. A Quick Copy test was changing the setting to BibTeX, which was resulting in random timeouts after subsequent resetDB() calls due to slow translator loading. This change skips initialization in test mode. This might actually fix a lot of timeouts on Travis in the second half of the tests... This also resets the Quick Copy pref in those tests so that it's left at the default, though really we should automatically reset all prefs after all test groups and in resetDB().
501 lines
16 KiB
JavaScript
501 lines
16 KiB
JavaScript
/*
|
|
***** BEGIN LICENSE BLOCK *****
|
|
|
|
Copyright © 2009 Center for History and New Media
|
|
George Mason University, Fairfax, Virginia, USA
|
|
http://zotero.org
|
|
|
|
This file is part of Zotero.
|
|
|
|
Zotero is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU Affero General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
Zotero is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU Affero General Public License for more details.
|
|
|
|
You should have received a copy of the GNU Affero General Public License
|
|
along with Zotero. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
***** END LICENSE BLOCK *****
|
|
*/
|
|
|
|
"use strict";
|
|
|
|
// Enumeration of types of translators
|
|
var TRANSLATOR_TYPES = {"import":1, "export":2, "web":4, "search":8};
|
|
|
|
/**
|
|
* Singleton to handle loading and caching of translators
|
|
* @namespace
|
|
*/
|
|
Zotero.Translators = new function() {
|
|
var _cache, _translators;
|
|
var _initialized = false;
|
|
|
|
/**
|
|
* Initializes translator cache, loading all translator metadata into memory
|
|
*
|
|
* @param {Object} [options.metadataCache] - Translator metadata keyed by filename, if already
|
|
* available (e.g., in updateBundledFiles()), to avoid unnecesary file reads
|
|
*/
|
|
this.reinit = Zotero.Promise.coroutine(function* (options = {}) {
|
|
// Wait until bundled files have been updated, except when this is called by the schema update
|
|
// code itself
|
|
if (!options.fromSchemaUpdate) {
|
|
yield Zotero.Schema.schemaUpdatePromise;
|
|
}
|
|
// Before bundled files can be updated, any existing translators need to be loaded, but other
|
|
// init() calls from elsewhere should still wait on schemaUpdatePromise, so init()/lazy()
|
|
// can't be used. Instead, the schema update code calls reinit() with noReinit.
|
|
else if (options.noReinit && _initialized) {
|
|
return;
|
|
}
|
|
|
|
Zotero.debug("Initializing translators");
|
|
var start = new Date;
|
|
|
|
_cache = {"import":[], "export":[], "web":[], "webWithTargetAll":[], "search":[]};
|
|
_translators = {};
|
|
|
|
var sql = "SELECT fileName, metadataJSON, lastModifiedTime FROM translatorCache";
|
|
var dbCacheResults = yield Zotero.DB.queryAsync(sql);
|
|
var dbCache = {};
|
|
for (let i = 0; i < dbCacheResults.length; i++) {
|
|
let entry = dbCacheResults[i];
|
|
dbCache[entry.fileName] = entry;
|
|
}
|
|
|
|
var numCached = 0;
|
|
var filesInCache = {};
|
|
var translatorsDir = Zotero.getTranslatorsDirectory().path;
|
|
var iterator = new OS.File.DirectoryIterator(translatorsDir);
|
|
try {
|
|
while (true) {
|
|
let entries = yield iterator.nextBatch(5); // TODO: adjust as necessary
|
|
if (!entries.length) break;
|
|
for (let i = 0; i < entries.length; i++) {
|
|
let entry = entries[i];
|
|
let path = entry.path;
|
|
let fileName = entry.name;
|
|
|
|
if (!(/^[^.].*\.js$/.test(fileName))) continue;
|
|
|
|
let lastModifiedTime;
|
|
if ('winLastWriteDate' in entry) {
|
|
lastModifiedTime = entry.winLastWriteDate.getTime();
|
|
}
|
|
else {
|
|
lastModifiedTime = (yield OS.File.stat(path)).lastModificationDate.getTime();
|
|
}
|
|
|
|
// Check passed cache for metadata
|
|
let memCacheJSON = false;
|
|
if (options.metadataCache && options.metadataCache[fileName]) {
|
|
memCacheJSON = options.metadataCache[fileName];
|
|
}
|
|
|
|
// Check DB cache
|
|
let dbCacheEntry = false;
|
|
if (dbCache[fileName]) {
|
|
filesInCache[fileName] = true;
|
|
if (dbCache[fileName].lastModifiedTime == lastModifiedTime) {
|
|
dbCacheEntry = dbCache[fileName];
|
|
}
|
|
}
|
|
|
|
// Get JSON from cache if possible
|
|
if (memCacheJSON || dbCacheEntry) {
|
|
var translator = Zotero.Translators.load(
|
|
memCacheJSON || dbCacheEntry.metadataJSON, path
|
|
);
|
|
}
|
|
// Otherwise, load from file
|
|
else {
|
|
try {
|
|
var translator = yield Zotero.Translators.loadFromFile(path);
|
|
}
|
|
catch (e) {
|
|
Zotero.logError(e);
|
|
|
|
// If translator file is invalid, delete it and clear the cache entry
|
|
// so that the translator is reinstalled the next time it's updated.
|
|
//
|
|
// TODO: Reinstall the correct translator immediately
|
|
yield OS.File.remove(path);
|
|
let sql = "DELETE FROM translatorCache WHERE fileName=?";
|
|
yield Zotero.DB.queryAsync(sql, fileName);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// When can this happen?
|
|
if (!translator.translatorID) {
|
|
Zotero.debug("Translator ID for " + path + " not found");
|
|
continue;
|
|
}
|
|
|
|
// Check if there's already a cached translator with the same id
|
|
if (_translators[translator.translatorID]) {
|
|
let existingTranslator = _translators[translator.translatorID];
|
|
// If cached translator is older, delete it
|
|
if (existingTranslator.lastUpdated < translator.lastUpdated) {
|
|
translator.logError("Deleting older translator "
|
|
+ existingTranslator.fileName + " with same ID as "
|
|
+ translator.fileName);
|
|
yield OS.File.remove(existingTranslator.path);
|
|
delete _translators[translator.translatorID];
|
|
}
|
|
// If cached translator is newer or the same, delete the current one
|
|
else {
|
|
translator.logError("Translator " + existingTranslator.fileName
|
|
+ " with same ID is already loaded -- deleting "
|
|
+ translator.fileName);
|
|
yield OS.File.remove(translator.path);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// add to cache
|
|
_translators[translator.translatorID] = translator;
|
|
for (let type in TRANSLATOR_TYPES) {
|
|
if (translator.translatorType & TRANSLATOR_TYPES[type]) {
|
|
_cache[type].push(translator);
|
|
if ((translator.translatorType & TRANSLATOR_TYPES.web) && translator.targetAll) {
|
|
_cache.webWithTargetAll.push(translator);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!dbCacheEntry) {
|
|
yield Zotero.Translators.cacheInDB(
|
|
fileName,
|
|
translator.serialize(TRANSLATOR_REQUIRED_PROPERTIES.
|
|
concat(TRANSLATOR_OPTIONAL_PROPERTIES)),
|
|
lastModifiedTime
|
|
);
|
|
}
|
|
|
|
numCached++;
|
|
}
|
|
}
|
|
}
|
|
finally {
|
|
iterator.close();
|
|
}
|
|
|
|
// Remove translators from DB cache if no file
|
|
for (let fileName in dbCache) {
|
|
if (!filesInCache[fileName]) {
|
|
yield Zotero.DB.queryAsync(
|
|
"DELETE FROM translatorCache WHERE fileName = ?", fileName
|
|
);
|
|
}
|
|
}
|
|
|
|
// Sort by priority
|
|
var collation = Zotero.getLocaleCollation();
|
|
var cmp = function (a, b) {
|
|
if (a.priority > b.priority) {
|
|
return 1;
|
|
}
|
|
else if (a.priority < b.priority) {
|
|
return -1;
|
|
}
|
|
return collation.compareString(1, a.label, b.label);
|
|
}
|
|
for(var type in _cache) {
|
|
_cache[type].sort(cmp);
|
|
}
|
|
|
|
_initialized = true;
|
|
|
|
Zotero.debug("Cached " + numCached + " translators in " + ((new Date) - start) + " ms");
|
|
});
|
|
this.init = Zotero.lazy(this.reinit);
|
|
|
|
/**
|
|
* Loads a translator from JSON, with optional code
|
|
*
|
|
* @param {String|Object} json - Metadata JSON
|
|
* @param {String} path
|
|
* @param {String} [code]
|
|
*/
|
|
this.load = function (json, path, code) {
|
|
var info = typeof json == 'string' ? JSON.parse(json) : json;
|
|
info.path = path;
|
|
info.code = code;
|
|
return new Zotero.Translator(info);
|
|
}
|
|
|
|
/**
|
|
* Loads a translator from the disk
|
|
*
|
|
* @param {String} file - Path to translator file
|
|
*/
|
|
this.loadFromFile = function(path) {
|
|
const infoRe = /^\s*{[\S\s]*?}\s*?[\r\n]/;
|
|
return Zotero.File.getContentsAsync(path)
|
|
.then(function(source) {
|
|
return Zotero.Translators.load(infoRe.exec(source)[0], path, source);
|
|
})
|
|
.catch(function() {
|
|
throw "Invalid or missing translator metadata JSON object in " + OS.Path.basename(path);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Gets the translator that corresponds to a given ID
|
|
*
|
|
* @param {String} id The ID of the translator
|
|
*/
|
|
this.get = function(id) {
|
|
if (!_initialized) {
|
|
throw new Zotero.Exception.UnloadedDataException("Translators not yet loaded", 'translators');
|
|
}
|
|
return _translators[id] ? _translators[id] : false
|
|
}
|
|
|
|
/**
|
|
* Gets all translators for a specific type of translation
|
|
*
|
|
* @param {String} type The type of translators to get (import, export, web, or search)
|
|
*/
|
|
this.getAllForType = function(type) {
|
|
return this.init().then(function () {
|
|
return _cache[type].slice();
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Gets all translators for a specific type of translation
|
|
*/
|
|
this.getAll = function() {
|
|
return this.init().then(function () {
|
|
return Object.keys(_translators).map(id => _translators[id]);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Gets web translators for a specific location
|
|
* @param {String} uri The URI for which to look for translators
|
|
* @param {String} rootUri The root URI of the page, different from `uri` if running in an iframe
|
|
*/
|
|
this.getWebTranslatorsForLocation = function(URI, rootURI) {
|
|
var isFrame = URI !== rootURI;
|
|
var type = isFrame ? "webWithTargetAll" : "web";
|
|
|
|
return this.getAllForType(type).then(function(allTranslators) {
|
|
var potentialTranslators = [];
|
|
var converterFunctions = [];
|
|
|
|
var rootSearchURIs = this.getSearchURIs(rootURI);
|
|
var frameSearchURIs = isFrame ? this.getSearchURIs(URI) : rootSearchURIs;
|
|
|
|
Zotero.debug("Translators: Looking for translators for "+Object.keys(frameSearchURIs).join(', '));
|
|
|
|
for (let translator of allTranslators) {
|
|
rootURIsLoop:
|
|
for (let rootSearchURI in rootSearchURIs) {
|
|
let isGeneric = !translator.webRegexp.root;
|
|
|
|
let rootURIMatches = isGeneric || rootSearchURI.length < 8192 && translator.webRegexp.root.test(rootSearchURI);
|
|
if (translator.webRegexp.all && rootURIMatches) {
|
|
for (let frameSearchURI in frameSearchURIs) {
|
|
let frameURIMatches = frameSearchURI.length < 8192 && translator.webRegexp.all.test(frameSearchURI);
|
|
|
|
if (frameURIMatches) {
|
|
potentialTranslators.push(translator);
|
|
converterFunctions.push(frameSearchURIs[frameSearchURI]);
|
|
// prevent adding the translator multiple times
|
|
break rootURIsLoop;
|
|
}
|
|
}
|
|
}
|
|
else if(!isFrame && (isGeneric || rootURIMatches)) {
|
|
potentialTranslators.push(translator);
|
|
converterFunctions.push(rootSearchURIs[rootSearchURI]);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return [potentialTranslators, converterFunctions];
|
|
}.bind(this));
|
|
},
|
|
|
|
/**
|
|
* Get the array of searchURIs and related proxy converter functions
|
|
*
|
|
* @param {String} URI to get searchURIs and converterFunctions for
|
|
*/
|
|
this.getSearchURIs = function(URI) {
|
|
var properURI = Zotero.Proxies.proxyToProper(URI);
|
|
if (properURI !== URI) {
|
|
// if we know this proxy, just use the proper URI for detection
|
|
let obj = {};
|
|
obj[properURI] = Zotero.Proxies.properToProxy;
|
|
return obj;
|
|
}
|
|
|
|
var searchURIs = {};
|
|
searchURIs[URI] = null;
|
|
|
|
// if there is a subdomain that is also a TLD, also test against URI with the domain
|
|
// dropped after the TLD
|
|
// (i.e., www.nature.com.mutex.gmu.edu => www.nature.com)
|
|
var m = /^(https?:\/\/)([^\/]+)/i.exec(URI);
|
|
if (m) {
|
|
// First, drop the 0- if it exists (this is an III invention)
|
|
var host = m[2];
|
|
if(host.substr(0, 2) === "0-") host = host.substr(2);
|
|
var hostnames = host.split(".");
|
|
for (var i=1; i<hostnames.length-2; i++) {
|
|
if (TLDS[hostnames[i].toLowerCase()]) {
|
|
var properHost = hostnames.slice(0, i+1).join(".");
|
|
searchURIs[m[1]+properHost+URI.substr(m[0].length)] = new function() {
|
|
var re = new RegExp('^https?://(?:[^/]+\\.)?'+Zotero.Utilities.quotemeta(properHost)+'(?=/)', "gi");
|
|
var proxyHost = hostnames.slice(i+1).join(".").replace(/\$/g, "$$$$");
|
|
return function(uri) { return uri.replace(re, "$&."+proxyHost) };
|
|
};
|
|
}
|
|
}
|
|
}
|
|
return searchURIs;
|
|
},
|
|
|
|
/**
|
|
* Gets import translators for a specific location
|
|
* @param {String} location The location for which to look for translators
|
|
* @param {Function} [callback] An optional callback to be executed when translators have been
|
|
* retrieved
|
|
* @return {Promise<Zotero.Translator[]|true>} - An array of translators if no callback is specified;
|
|
* otherwise true
|
|
*/
|
|
this.getImportTranslatorsForLocation = function(location, callback) {
|
|
return Zotero.Translators.getAllForType("import").then(function(allTranslators) {
|
|
var tier1Translators = [];
|
|
var tier2Translators = [];
|
|
|
|
for(var i=0; i<allTranslators.length; i++) {
|
|
if(allTranslators[i].importRegexp && allTranslators[i].importRegexp.test(location)) {
|
|
tier1Translators.push(allTranslators[i]);
|
|
} else {
|
|
tier2Translators.push(allTranslators[i]);
|
|
}
|
|
}
|
|
|
|
var translators = tier1Translators.concat(tier2Translators);
|
|
if(callback) {
|
|
callback(translators);
|
|
return true;
|
|
}
|
|
return translators;
|
|
});
|
|
}
|
|
|
|
/**
|
|
* @param {String} label
|
|
* @return {String}
|
|
*/
|
|
this.getFileNameFromLabel = function(label, alternative) {
|
|
var fileName = Zotero.Utilities.removeDiacritics(
|
|
Zotero.File.getValidFileName(label)) + ".js";
|
|
// Use translatorID if name still isn't ASCII (e.g., Cyrillic)
|
|
if (alternative && !fileName.match(/^[\x00-\x7f]+$/)) {
|
|
fileName = alternative + ".js";
|
|
}
|
|
return fileName;
|
|
}
|
|
|
|
/**
|
|
* @param {String} metadata
|
|
* @param {String} metadata.translatorID Translator GUID
|
|
* @param {Integer} metadata.translatorType See TRANSLATOR_TYPES in translate.js
|
|
* @param {String} metadata.label Translator title
|
|
* @param {String} metadata.creator Translator author
|
|
* @param {String|Null} metadata.target Target regexp
|
|
* @param {String|Null} metadata.minVersion
|
|
* @param {String} metadata.maxVersion
|
|
* @param {String|undefined} metadata.configOptions
|
|
* @param {String|undefined} metadata.displayOptions
|
|
* @param {Integer} metadata.priority
|
|
* @param {String} metadata.browserSupport
|
|
* @param {Boolean} metadata.inRepository
|
|
* @param {String} metadata.lastUpdated SQL date
|
|
* @param {String} code
|
|
* @return {Promise<nsIFile>}
|
|
*/
|
|
this.save = Zotero.Promise.coroutine(function* (metadata, code) {
|
|
if (!metadata.translatorID) {
|
|
throw ("metadata.translatorID not provided in Zotero.Translators.save()");
|
|
}
|
|
|
|
if (!metadata.translatorType) {
|
|
var found = false;
|
|
for (let type in TRANSLATOR_TYPES) {
|
|
if (metadata.translatorType & TRANSLATOR_TYPES[type]) {
|
|
found = true;
|
|
break;
|
|
}
|
|
}
|
|
if (!found) {
|
|
throw ("Invalid translatorType '" + metadata.translatorType + "' in Zotero.Translators.save()");
|
|
}
|
|
}
|
|
|
|
if (!metadata.label) {
|
|
throw new Error("metadata.label not provided");
|
|
}
|
|
|
|
if (!metadata.priority) {
|
|
throw new Error("metadata.priority not provided");
|
|
}
|
|
|
|
if (!metadata.lastUpdated) {
|
|
throw new Error("metadata.lastUpdated not provided");
|
|
}
|
|
|
|
if (!code) {
|
|
throw new Error("code not provided");
|
|
}
|
|
|
|
var fileName = Zotero.Translators.getFileNameFromLabel(
|
|
metadata.label, metadata.translatorID
|
|
);
|
|
var destFile = OS.Path.join(Zotero.getTranslatorsDirectory().path, fileName);
|
|
|
|
// JSON.stringify has the benefit of indenting JSON
|
|
var metadataJSON = JSON.stringify(metadata, null, "\t");
|
|
|
|
var str = metadataJSON + "\n\n" + code,
|
|
translator;
|
|
|
|
var translator = Zotero.Translators.get(metadata.translatorID);
|
|
var sameFile = translator && destFile == translator.path;
|
|
|
|
var exists = yield OS.File.exists(destFile);
|
|
if (!sameFile && exists) {
|
|
var msg = "Overwriting translator with same filename '"
|
|
+ fileName + "'";
|
|
Zotero.debug(msg, 1);
|
|
Zotero.debug(metadata, 1);
|
|
Components.utils.reportError(msg);
|
|
}
|
|
|
|
Zotero.debug("Saving translator '" + metadata.label + "'");
|
|
Zotero.debug(metadata);
|
|
return Zotero.File.putContentsAsync(destFile, str).return(destFile);
|
|
});
|
|
|
|
this.cacheInDB = function(fileName, metadataJSON, lastModifiedTime) {
|
|
return Zotero.DB.queryAsync(
|
|
"REPLACE INTO translatorCache VALUES (?, ?, ?)",
|
|
[fileName, JSON.stringify(metadataJSON), lastModifiedTime]
|
|
);
|
|
}
|
|
}
|