diff --git a/chrome/content/zotero/preferences/preferences.js b/chrome/content/zotero/preferences/preferences.js index bd689811c..fcdd5755b 100644 --- a/chrome/content/zotero/preferences/preferences.js +++ b/chrome/content/zotero/preferences/preferences.js @@ -1004,7 +1004,7 @@ function runIntegrityCheck() { function updateTranslators() { - Zotero.Schema.updateScrapersRemote(true, function (xmlhttp, updated) { + Zotero.Schema.updateFromRepository(true, function (xmlhttp, updated) { var button = document.getElementById('updateButton'); if (button) { if (updated===-1) { @@ -1037,7 +1037,7 @@ function resetTranslatorsAndStyles() { null, null, null, {}); if (index == 0) { - Zotero.Schema.rebuildTranslatorsAndStylesTables(function (xmlhttp, updated) { + Zotero.Schema.resetTranslatorsAndStyles(function (xmlhttp, updated) { populateQuickCopyList(); }); } @@ -1059,7 +1059,7 @@ function resetTranslators() { null, null, null, {}); if (index == 0) { - Zotero.Schema.rebuildTranslatorsTable(); + Zotero.Schema.resetTranslators(); } } @@ -1079,7 +1079,7 @@ function resetStyles() { null, null, null, {}); if (index == 0) { - Zotero.Schema.rebuildStylesTable(function (xmlhttp, updated) { + Zotero.Schema.resetStyles(function (xmlhttp, updated) { populateQuickCopyList(); }); } @@ -1117,26 +1117,29 @@ function refreshStylesList(cslID) { treechildren.removeChild(treechildren.firstChild); } - var sql = "SELECT cslID, title, updated FROM csl ORDER BY title"; - var styleData = Zotero.DB.query(sql); - if (!styleData) return; + var styles = Zotero.Styles.getAll(); var selectIndex = false; - for (var i=0; i modTime) { + modTime = fileModTime; + } + } + + if (forceReinstall && lastModTime && modTime <= lastModTime) { + Zotero.debug("Installed " + modes + " are up-to-date with " + modes + " directory"); + return 0; + } + + Zotero.debug("Updating installed " + modes + " from " + modes + " directory"); + + var entries = sourceDir.directoryEntries; + while (entries.hasMoreElements()) { + var file = entries.getNext(); + file.QueryInterface(Components.interfaces.nsIFile); + if (!file.leafName.match(fileNameRE) || file.isDirectory()) { + continue; + } + var newObj = new Zotero[Mode](file); + var existingObj = Zotero[Modes].get(newObj[mode + "ID"]); + if (!existingObj) { + Zotero.debug("Installing " + mode + " '" + newObj[titleField] + "'"); + } + else { + Zotero.debug("Updating " + + (existingObj.hidden ? "hidden " : "") + + mode + " '" + existingObj[titleField] + "'"); + if (existingObj.file.exists()) { + existingObj.file.remove(false); + } + } + + if (mode == 'translator') { + var fileName = Zotero.File.getValidFileName(newObj[titleField]) + fileExt + + var destFile = destDir.clone(); + destFile.append(fileName); + if (destFile.exists()) { + var msg = "Overwriting translator with same filename '" + + fileName + "'"; + Zotero.debug(msg, 1); + Components.utils.reportError(msg + " in Zotero.Schema.updateBundledFiles()"); + destFile.remove(false); + } + } + else if (mode == 'style') { + var fileName = file.leafName; + } + + if (!existingObj || !existingObj.hidden) { + file.copyTo(destDir, fileName); + } + else { + file.copyTo(hiddenDir, fileName); + } + } + } + + Zotero.DB.beginTransaction(); + + var sql = "REPLACE INTO version VALUES (?, ?)"; + Zotero.DB.query(sql, [modes, modTime]); + + var sql = "REPLACE INTO version VALUES ('repository', ?)"; + Zotero.DB.query(sql, repotime); + + Zotero.DB.commitTransaction(); + + Zotero[Modes].init(); + return 1; + } + + + /** + * Send XMLHTTP request for updated translators and styles to the central repository + * + * @param {Boolean} force Force a repository query regardless of how + * long it's been since the last check + * @param {Function} callback + */ + this.updateFromRepository = function (force, callback) { // Little hack to manually update CSLs from repo on upgrades if (!force && Zotero.Prefs.get('automaticScraperUpdates')) { var syncTargetVersion = 3; // increment this when releasing new version that requires it @@ -188,7 +407,7 @@ Zotero.Schema = new function(){ // Check user preference for automatic updates if (!Zotero.Prefs.get('automaticScraperUpdates')){ - Zotero.debug('Automatic scraper updating disabled -- not checking repository', 4); + Zotero.debug('Automatic repository updating disabled -- not checking repository', 4); return false; } @@ -240,7 +459,7 @@ Zotero.Schema = new function(){ } var get = Zotero.Utilities.HTTP.doGet(url, function (xmlhttp) { - var updated = _updateScrapersRemoteCallback(xmlhttp, !!force); + var updated = _updateFromRepositoryCallback(xmlhttp, !!force); if (callback) { callback(xmlhttp, updated) } @@ -262,61 +481,30 @@ Zotero.Schema = new function(){ } - function rebuildTranslatorsAndStylesTables(callback) { - Zotero.debug("Rebuilding translators and styles tables"); - Zotero.DB.beginTransaction(); + this.resetTranslatorsAndStyles = function (callback) { + Zotero.debug("Resetting translators and styles"); - Zotero.DB.query("DELETE FROM translators"); - Zotero.DB.query("DELETE FROM csl"); var sql = "DELETE FROM version WHERE schema IN " - + "('scrapers', 'repository', 'lastcheck')"; + + "('translators', 'styles', 'repository', 'lastcheck')"; Zotero.DB.query(sql); - _dbVersions['scrapers'] = null; - _dbVersions['repository'] = null; - _dbVersions['lastcheck'] = null; + _dbVersions.repository = null; + _dbVersions.lastcheck = null; - // Rebuild from scrapers.sql - _updateSchema('scrapers'); + var translatorsDir = Zotero.getTranslatorsDirectory(); + translatorsDir.remove(true); + Zotero.getTranslatorsDirectory(); // recreate directory + Zotero.Translators.init(); + this.updateBundledFiles('translators'); - // Rebuild the translator cache - Zotero.debug("Clearing translator cache"); - Zotero.Translate.cache = null; - Zotero.Translate.init(); - - Zotero.DB.commitTransaction(); + var stylesDir = Zotero.getStylesDirectory(); + stylesDir.remove(true); + Zotero.getStylesDirectory(); // recreate directory + Zotero.Styles.init(); + this.updateBundledFiles('styles'); // Run a manual update from repository if pref set if (Zotero.Prefs.get('automaticScraperUpdates')) { - this.updateScrapersRemote(2, callback); - } - } - - - function rebuildTranslatorsTable(callback) { - Zotero.debug("Rebuilding translators table"); - Zotero.DB.beginTransaction(); - - Zotero.DB.query("DELETE FROM translators"); - var sql = "DELETE FROM version WHERE schema IN " - + "('scrapers', 'repository', 'lastcheck')"; - Zotero.DB.query(sql); - _dbVersions['scrapers'] = null; - _dbVersions['repository'] = null; - _dbVersions['lastcheck'] = null; - - // Rebuild from scrapers.sql - _updateSchema('scrapers'); - - // Rebuild the translator cache - Zotero.debug("Clearing translator cache"); - Zotero.Translate.cache = null; - Zotero.Translate.init(); - - Zotero.DB.commitTransaction(); - - // Run a manual update from repository if pref set - if (Zotero.Prefs.get('automaticScraperUpdates')) { - this.updateScrapersRemote(2, callback); + this.updateFromRepository(2, callback); } } @@ -489,12 +677,10 @@ Zotero.Schema = new function(){ Zotero.DB.query(_getSchemaSQL('system')); Zotero.DB.query(_getSchemaSQL('userdata')); Zotero.DB.query(_getSchemaSQL('triggers')); - Zotero.DB.query(_getSchemaSQL('scrapers')); _updateDBVersion('system', _getSchemaSQLVersion('system')); _updateDBVersion('userdata', _getSchemaSQLVersion('userdata')); _updateDBVersion('triggers', _getSchemaSQLVersion('triggers')); - _updateDBVersion('scrapers', _getSchemaSQLVersion('scrapers')); /* TODO: uncomment for release @@ -574,7 +760,7 @@ Zotero.Schema = new function(){ /** * Process the response from the repository **/ - function _updateScrapersRemoteCallback(xmlhttp, manual){ + function _updateFromRepositoryCallback(xmlhttp, manual){ if (!xmlhttp.responseXML){ try { if (xmlhttp.status>1000){ @@ -635,17 +821,16 @@ Zotero.Schema = new function(){ try { for (var i=0, len=translatorUpdates.length; i4K chunks into multiple nodes // https://bugzilla.mozilla.org/show_bug.cgi?id=194231 xmlnode.normalize(); + var translatorID = xmlnode.getAttribute('id'); + var translator = Zotero.Translators.get(translatorID); // Delete local version of remote translators with priority 0 if (xmlnode.getElementsByTagName('priority')[0].firstChild.nodeValue === "0") { - var sql = "DELETE FROM translators WHERE translatorID=?"; - return Zotero.DB.query(sql, {string: xmlnode.getAttribute('id')}); + if (translator && translator.file.exists()) { + Zotero.debug("Deleting translator '" + translator.label + "'"); + translator.file.remove(false); + } + return false; } - var sqlValues = [ - {string: xmlnode.getAttribute('id')}, - {string: xmlnode.getAttribute('minVersion')}, - {string: xmlnode.getAttribute('maxVersion')}, - {string: xmlnode.getAttribute('lastUpdated')}, - 1, // inRepository - {int: xmlnode.getElementsByTagName('priority')[0].firstChild.nodeValue}, - {int: xmlnode.getAttribute('type')}, - {string: xmlnode.getElementsByTagName('label')[0].firstChild.nodeValue}, - {string: xmlnode.getElementsByTagName('creator')[0].firstChild.nodeValue}, - // target - (xmlnode.getElementsByTagName('target').item(0) && - xmlnode.getElementsByTagName('target')[0].firstChild) - ? {string: xmlnode.getElementsByTagName('target')[0].firstChild.nodeValue} - : {null: true}, - // detectCode can not exist or be empty - (xmlnode.getElementsByTagName('detectCode').item(0) && - xmlnode.getElementsByTagName('detectCode')[0].firstChild) - ? {string: xmlnode.getElementsByTagName('detectCode')[0].firstChild.nodeValue} - : {null: true}, - {string: xmlnode.getElementsByTagName('code')[0].firstChild.nodeValue} - ]; + var metadata = { + translatorID: translatorID, + translatorType: parseInt(xmlnode.getAttribute('type')), + label: xmlnode.getElementsByTagName('label')[0].firstChild.nodeValue, + creator: xmlnode.getElementsByTagName('creator')[0].firstChild.nodeValue, + target: (xmlnode.getElementsByTagName('target').item(0) && + xmlnode.getElementsByTagName('target')[0].firstChild) + ? xmlnode.getElementsByTagName('target')[0].firstChild.nodeValue + : null, + minVersion: xmlnode.getAttribute('minVersion'), + maxVersion: xmlnode.getAttribute('maxVersion'), + priority: parseInt( + xmlnode.getElementsByTagName('priority')[0].firstChild.nodeValue + ), + inRepository: true, + lastUpdated: xmlnode.getAttribute('lastUpdated') + }; - var sql = "REPLACE INTO translators VALUES (?,?,?,?,?,?,?,?,?,?,?,?)"; - return Zotero.DB.query(sql, sqlValues); + // detectCode can not exist or be empty + var detectCode = (xmlnode.getElementsByTagName('detectCode').item(0) && + xmlnode.getElementsByTagName('detectCode')[0].firstChild) + ? xmlnode.getElementsByTagName('detectCode')[0].firstChild.nodeValue + : null; + var code = xmlnode.getElementsByTagName('code')[0].firstChild.nodeValue; + + var fileName = Zotero.Translators.getFileNameFromLabel(metadata.label); + var destFile = Zotero.getTranslatorsDirectory(); + destFile.append(fileName); + + var nsIJSON = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON); + var metadataJSON = nsIJSON.encode(metadata); + + var str = metadataJSON + "\n\n" + (detectCode ? detectCode + "\n\n" : "") + code; + + if (translator && destFile.equals(translator.file)) { + var sameFile = true; + } + + if (!sameFile && destFile.exists()) { + var msg = "Overwriting translator with same filename '" + + fileName + "'"; + Zotero.debug(msg, 1); + Zotero.debug(metadata, 1); + Components.utils.reportError(msg + " in Zotero.Schema._translatorXMLToFile()"); + } + + if (translator && translator.file.exists()) { + translator.file.remove(false); + } + + Zotero.debug("Saving translator '" + metadata.label + "'"); + Zotero.File.putContents(destFile, str); + return destFile; } /** - * Traverse an XML style node from the repository and - * update the local csl table with the style data - **/ - function _styleXMLToDB(xmlnode){ + * Traverse an XML style node from the repository and + * update the local styles folder with the style data + */ + function _styleXMLToFile(xmlnode) { // Don't split >4K chunks into multiple nodes // https://bugzilla.mozilla.org/show_bug.cgi?id=194231 xmlnode.normalize(); var uri = xmlnode.getAttribute('id'); - // - // Workaround for URI change -- delete existing versions with old URIs of updated styles - // - var re = new RegExp("http://www.zotero.org/styles/(.+)"); - var matches = uri.match(re); - - if (matches) { - var zoteroReplacements = ['chicago-author-date', 'chicago-note-bibliography']; - var purlReplacements = [ - 'apa', 'asa', 'chicago-note', 'ieee', 'mhra_note_without_bibliography', - 'mla', 'nature', 'nlm' - ]; - - if (zoteroReplacements.indexOf(matches[1]) != -1) { - var sql = "DELETE FROM csl WHERE cslID=?"; - Zotero.DB.query(sql, 'http://www.zotero.org/namespaces/CSL/' + matches[1] + '.csl'); - } - else if (purlReplacements.indexOf(matches[1]) != -1) { - var sql = "DELETE FROM csl WHERE cslID=?"; - Zotero.DB.query(sql, 'http://purl.org/net/xbiblio/csl/styles/' + matches[1] + '.csl'); - } - } - - var uri = xmlnode.getAttribute('id'); - // Delete local style if CSL code is empty if (!xmlnode.getElementsByTagName('csl')[0].firstChild) { - var sql = "DELETE FROM csl WHERE cslID=?"; - Zotero.DB.query(sql, uri); - return true; + var style = Zotero.Styles.get(uri); + if (style) { + style.file.remove(null); + } + return; } - var sqlValues = [ - {string: uri}, - {string: xmlnode.getAttribute('updated')}, - {string: xmlnode.getElementsByTagName('title')[0].firstChild.nodeValue}, - {string: xmlnode.getElementsByTagName('csl')[0].firstChild.nodeValue} - ]; + var str = xmlnode.getElementsByTagName('csl')[0].firstChild.nodeValue; - var sql = "REPLACE INTO csl VALUES (?,?,?,?)"; - return Zotero.DB.query(sql, sqlValues); + var style = Zotero.Styles.get(uri); + if (style) { + if (style.file.exists()) { + style.file.remove(false); + } + var destFile = style.file; + } + else { + // Get last part of URI for filename + var matches = uri.match(/([^\/]+)$/); + if (!matches) { + throw ("Invalid style URI '" + uri + "' from repository"); + } + var destFile = Zotero.getStylesDirectory(); + destFile.append(matches[1]); + if (destFile.exists()) { + throw ("Different style with filename '" + matches[1] + + "' already exists in Zotero.Schema._styleXMLToFile()"); + } + } + + Zotero.debug("Saving style '" + uri + "'"); + Zotero.File.putContents(destFile, str); + return; } @@ -1679,6 +1891,50 @@ Zotero.Schema = new function(){ Zotero.DB.query("CREATE TABLE storageDeleteLog (\n key TEXT PRIMARY KEY,\n timestamp INT NOT NULL\n)"); Zotero.DB.query("CREATE INDEX storageDeleteLog_timestamp ON storageDeleteLog(timestamp)"); } + + if (i==41) { + var translators = Zotero.DB.query("SELECT * FROM translators WHERE inRepository!=1"); + if (translators) { + var dir = Zotero.getTranslatorsDirectory(); + if (dir.exists()) { + dir.remove(true); + } + Zotero.getTranslatorsDirectory() + for each(var row in translators) { + var file = dir.clone(); + var fileName = Zotero.Translators.getFileNameFromLabel(row.label); + file.append(fileName); + var metadata = { translatorID: row.translatorID, translatorType: parseInt(row.translatorType), label: row.label, creator: row.creator, target: row.target ? row.target : null, minVersion: row.minVersion, maxVersion: row.maxVersion, priority: parseInt(row.priority), inRepository: row.inRepository == 1 ? true : false, lastUpdated: row.lastUpdated }; + var nsIJSON = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON); + var metadataJSON = nsIJSON.encode(metadata); + var str = metadataJSON + "\n\n" + (row.detectCode ? row.detectCode + "\n\n" : "") + row.code; + Zotero.debug("Extracting translator '" + row.label + "' from database"); + Zotero.File.putContents(file, str); + } + Zotero.Translators.init(); + } + var styles = Zotero.DB.query("SELECT * FROM csl"); + if (styles) { + var dir = Zotero.getStylesDirectory(); + if (dir.exists()) { + dir.remove(true); + } + Zotero.getStylesDirectory() + for each(var row in styles) { + var file = dir.clone(); + var matches = row.cslID.match(/([^\/]+)$/); + if (!matches) { + continue; + } + file.append(matches[1]); + Zotero.debug("Extracting styles '" + matches[1] + "' from database"); + Zotero.File.putContents(file, row.csl); + } + Zotero.Styles.init(); + } + Zotero.DB.query("DROP TABLE translators"); + Zotero.DB.query("DROP TABLE csl"); + } } _updateDBVersion('userdata', toVersion); diff --git a/chrome/content/zotero/xpcom/translate.js b/chrome/content/zotero/xpcom/translate.js index 783623b03..a61169182 100644 --- a/chrome/content/zotero/xpcom/translate.js +++ b/chrome/content/zotero/xpcom/translate.js @@ -96,6 +96,15 @@ Zotero.Translators = new function() { if(!_initialized) this.init(); return _cache[type].slice(0); } + + + /** + * @param {String} label + * @return {String} + */ + this.getFileNameFromLabel = function(label) { + return Zotero.File.getValidFileName(label) + ".js"; + } } /** diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index c572cdc69..b312bcd6b 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -262,7 +262,7 @@ var Zotero = new function(){ } Zotero.DB.startDummyStatement(); - Zotero.Schema.updateScrapersRemote(); + Zotero.Schema.updateFromRepository(); // Initialize integration web server Zotero.Integration.init(); @@ -1028,7 +1028,7 @@ Zotero.Prefs = new function(){ switch (data){ case "automaticScraperUpdates": if (this.get('automaticScraperUpdates')){ - Zotero.Schema.updateScrapersRemote(); + Zotero.Schema.updateFromRepository(); } else { Zotero.Schema.stopRepositoryTimer(); diff --git a/repotime.txt b/repotime.txt new file mode 100644 index 000000000..4b2a1cacc --- /dev/null +++ b/repotime.txt @@ -0,0 +1 @@ +2008-09-03 23:35:00 diff --git a/translators/build_zip b/translators/build_zip new file mode 100755 index 000000000..387cc1f85 --- /dev/null +++ b/translators/build_zip @@ -0,0 +1,20 @@ +#!/bin/bash +if [ -f translators.zip ]; then + rm translators.zip +fi +if [ ! -d output ]; then + mkdir output; +fi + +counter=0; +for file in *.js; do + newfile=$counter.js; + cp "$file" output/$newfile; + counter=`echo $counter + 1 | bc`; +done; + +cd output +zip ../translators.zip * +cd .. +rm -rf output +mv translators.zip .. \ No newline at end of file diff --git a/userdata.sql b/userdata.sql index 8a3a9448f..f74c07cc4 100644 --- a/userdata.sql +++ b/userdata.sql @@ -1,4 +1,4 @@ --- 40 +-- 41 -- This file creates tables containing user-specific data -- any changes made -- here must be mirrored in transition steps in schema.js::_migrateSchema() @@ -211,29 +211,6 @@ CREATE TABLE storageDeleteLog ( ); CREATE INDEX storageDeleteLog_timestamp ON storageDeleteLog(timestamp); -CREATE TABLE translators ( - translatorID TEXT PRIMARY KEY, - minVersion TEXT, - maxVersion TEXT, - lastUpdated DATETIME, - inRepository INT, - priority INT, - translatorType INT, - label TEXT, - creator TEXT, - target TEXT, - detectCode TEXT, - code TEXT -); -CREATE INDEX translators_type ON translators(translatorType); - -CREATE TABLE csl ( - cslID TEXT PRIMARY KEY, - updated DATETIME, - title TEXT, - csl TEXT -); - CREATE TABLE annotations ( annotationID INTEGER PRIMARY KEY, itemID INT,