From 9dd11a7d1b4262664d7160fa9928f00ebc63c301 Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Fri, 12 Sep 2008 22:09:54 +0000 Subject: [PATCH] Install/upgrade support for flat-file translators and styles Files are copied from translators.zip and styles.zip (or, for SVN installs, 'translators' and (for now) 'csl' directories) in the installation directory to 'translators' and 'styles' directories in the data directory. A build_zip file is provided for testing translators.zip (which will take precedence over a 'translators' directory) but isn't required. The timestamp stored in repotime.txt is stored in the database and is sent to the server for updates since that time. Updating a file in [install-dir]/translators or [install-dir]/styles automatically copies all files in that directory to the data directory. --- .../content/zotero/preferences/preferences.js | 31 +- chrome/content/zotero/xpcom/cite.js | 5 +- chrome/content/zotero/xpcom/schema.js | 536 +++++++++++++----- chrome/content/zotero/xpcom/translate.js | 9 + chrome/content/zotero/xpcom/zotero.js | 4 +- repotime.txt | 1 + translators/build_zip | 20 + userdata.sql | 25 +- 8 files changed, 448 insertions(+), 183 deletions(-) create mode 100644 repotime.txt create mode 100755 translators/build_zip diff --git a/chrome/content/zotero/preferences/preferences.js b/chrome/content/zotero/preferences/preferences.js index bd689811c..fcdd5755b 100644 --- a/chrome/content/zotero/preferences/preferences.js +++ b/chrome/content/zotero/preferences/preferences.js @@ -1004,7 +1004,7 @@ function runIntegrityCheck() { function updateTranslators() { - Zotero.Schema.updateScrapersRemote(true, function (xmlhttp, updated) { + Zotero.Schema.updateFromRepository(true, function (xmlhttp, updated) { var button = document.getElementById('updateButton'); if (button) { if (updated===-1) { @@ -1037,7 +1037,7 @@ function resetTranslatorsAndStyles() { null, null, null, {}); if (index == 0) { - Zotero.Schema.rebuildTranslatorsAndStylesTables(function (xmlhttp, updated) { + Zotero.Schema.resetTranslatorsAndStyles(function (xmlhttp, updated) { populateQuickCopyList(); }); } @@ -1059,7 +1059,7 @@ function resetTranslators() { null, null, null, {}); if (index == 0) { - Zotero.Schema.rebuildTranslatorsTable(); + Zotero.Schema.resetTranslators(); } } @@ -1079,7 +1079,7 @@ function resetStyles() { null, null, null, {}); if (index == 0) { - Zotero.Schema.rebuildStylesTable(function (xmlhttp, updated) { + Zotero.Schema.resetStyles(function (xmlhttp, updated) { populateQuickCopyList(); }); } @@ -1117,26 +1117,29 @@ function refreshStylesList(cslID) { treechildren.removeChild(treechildren.firstChild); } - var sql = "SELECT cslID, title, updated FROM csl ORDER BY title"; - var styleData = Zotero.DB.query(sql); - if (!styleData) return; + var styles = Zotero.Styles.getAll(); var selectIndex = false; - for (var i=0; i modTime) { + modTime = fileModTime; + } + } + + if (forceReinstall && lastModTime && modTime <= lastModTime) { + Zotero.debug("Installed " + modes + " are up-to-date with " + modes + " directory"); + return 0; + } + + Zotero.debug("Updating installed " + modes + " from " + modes + " directory"); + + var entries = sourceDir.directoryEntries; + while (entries.hasMoreElements()) { + var file = entries.getNext(); + file.QueryInterface(Components.interfaces.nsIFile); + if (!file.leafName.match(fileNameRE) || file.isDirectory()) { + continue; + } + var newObj = new Zotero[Mode](file); + var existingObj = Zotero[Modes].get(newObj[mode + "ID"]); + if (!existingObj) { + Zotero.debug("Installing " + mode + " '" + newObj[titleField] + "'"); + } + else { + Zotero.debug("Updating " + + (existingObj.hidden ? "hidden " : "") + + mode + " '" + existingObj[titleField] + "'"); + if (existingObj.file.exists()) { + existingObj.file.remove(false); + } + } + + if (mode == 'translator') { + var fileName = Zotero.File.getValidFileName(newObj[titleField]) + fileExt + + var destFile = destDir.clone(); + destFile.append(fileName); + if (destFile.exists()) { + var msg = "Overwriting translator with same filename '" + + fileName + "'"; + Zotero.debug(msg, 1); + Components.utils.reportError(msg + " in Zotero.Schema.updateBundledFiles()"); + destFile.remove(false); + } + } + else if (mode == 'style') { + var fileName = file.leafName; + } + + if (!existingObj || !existingObj.hidden) { + file.copyTo(destDir, fileName); + } + else { + file.copyTo(hiddenDir, fileName); + } + } + } + + Zotero.DB.beginTransaction(); + + var sql = "REPLACE INTO version VALUES (?, ?)"; + Zotero.DB.query(sql, [modes, modTime]); + + var sql = "REPLACE INTO version VALUES ('repository', ?)"; + Zotero.DB.query(sql, repotime); + + Zotero.DB.commitTransaction(); + + Zotero[Modes].init(); + return 1; + } + + + /** + * Send XMLHTTP request for updated translators and styles to the central repository + * + * @param {Boolean} force Force a repository query regardless of how + * long it's been since the last check + * @param {Function} callback + */ + this.updateFromRepository = function (force, callback) { // Little hack to manually update CSLs from repo on upgrades if (!force && Zotero.Prefs.get('automaticScraperUpdates')) { var syncTargetVersion = 3; // increment this when releasing new version that requires it @@ -188,7 +407,7 @@ Zotero.Schema = new function(){ // Check user preference for automatic updates if (!Zotero.Prefs.get('automaticScraperUpdates')){ - Zotero.debug('Automatic scraper updating disabled -- not checking repository', 4); + Zotero.debug('Automatic repository updating disabled -- not checking repository', 4); return false; } @@ -240,7 +459,7 @@ Zotero.Schema = new function(){ } var get = Zotero.Utilities.HTTP.doGet(url, function (xmlhttp) { - var updated = _updateScrapersRemoteCallback(xmlhttp, !!force); + var updated = _updateFromRepositoryCallback(xmlhttp, !!force); if (callback) { callback(xmlhttp, updated) } @@ -262,61 +481,30 @@ Zotero.Schema = new function(){ } - function rebuildTranslatorsAndStylesTables(callback) { - Zotero.debug("Rebuilding translators and styles tables"); - Zotero.DB.beginTransaction(); + this.resetTranslatorsAndStyles = function (callback) { + Zotero.debug("Resetting translators and styles"); - Zotero.DB.query("DELETE FROM translators"); - Zotero.DB.query("DELETE FROM csl"); var sql = "DELETE FROM version WHERE schema IN " - + "('scrapers', 'repository', 'lastcheck')"; + + "('translators', 'styles', 'repository', 'lastcheck')"; Zotero.DB.query(sql); - _dbVersions['scrapers'] = null; - _dbVersions['repository'] = null; - _dbVersions['lastcheck'] = null; + _dbVersions.repository = null; + _dbVersions.lastcheck = null; - // Rebuild from scrapers.sql - _updateSchema('scrapers'); + var translatorsDir = Zotero.getTranslatorsDirectory(); + translatorsDir.remove(true); + Zotero.getTranslatorsDirectory(); // recreate directory + Zotero.Translators.init(); + this.updateBundledFiles('translators'); - // Rebuild the translator cache - Zotero.debug("Clearing translator cache"); - Zotero.Translate.cache = null; - Zotero.Translate.init(); - - Zotero.DB.commitTransaction(); + var stylesDir = Zotero.getStylesDirectory(); + stylesDir.remove(true); + Zotero.getStylesDirectory(); // recreate directory + Zotero.Styles.init(); + this.updateBundledFiles('styles'); // Run a manual update from repository if pref set if (Zotero.Prefs.get('automaticScraperUpdates')) { - this.updateScrapersRemote(2, callback); - } - } - - - function rebuildTranslatorsTable(callback) { - Zotero.debug("Rebuilding translators table"); - Zotero.DB.beginTransaction(); - - Zotero.DB.query("DELETE FROM translators"); - var sql = "DELETE FROM version WHERE schema IN " - + "('scrapers', 'repository', 'lastcheck')"; - Zotero.DB.query(sql); - _dbVersions['scrapers'] = null; - _dbVersions['repository'] = null; - _dbVersions['lastcheck'] = null; - - // Rebuild from scrapers.sql - _updateSchema('scrapers'); - - // Rebuild the translator cache - Zotero.debug("Clearing translator cache"); - Zotero.Translate.cache = null; - Zotero.Translate.init(); - - Zotero.DB.commitTransaction(); - - // Run a manual update from repository if pref set - if (Zotero.Prefs.get('automaticScraperUpdates')) { - this.updateScrapersRemote(2, callback); + this.updateFromRepository(2, callback); } } @@ -489,12 +677,10 @@ Zotero.Schema = new function(){ Zotero.DB.query(_getSchemaSQL('system')); Zotero.DB.query(_getSchemaSQL('userdata')); Zotero.DB.query(_getSchemaSQL('triggers')); - Zotero.DB.query(_getSchemaSQL('scrapers')); _updateDBVersion('system', _getSchemaSQLVersion('system')); _updateDBVersion('userdata', _getSchemaSQLVersion('userdata')); _updateDBVersion('triggers', _getSchemaSQLVersion('triggers')); - _updateDBVersion('scrapers', _getSchemaSQLVersion('scrapers')); /* TODO: uncomment for release @@ -574,7 +760,7 @@ Zotero.Schema = new function(){ /** * Process the response from the repository **/ - function _updateScrapersRemoteCallback(xmlhttp, manual){ + function _updateFromRepositoryCallback(xmlhttp, manual){ if (!xmlhttp.responseXML){ try { if (xmlhttp.status>1000){ @@ -635,17 +821,16 @@ Zotero.Schema = new function(){ try { for (var i=0, len=translatorUpdates.length; i4K chunks into multiple nodes // https://bugzilla.mozilla.org/show_bug.cgi?id=194231 xmlnode.normalize(); + var translatorID = xmlnode.getAttribute('id'); + var translator = Zotero.Translators.get(translatorID); // Delete local version of remote translators with priority 0 if (xmlnode.getElementsByTagName('priority')[0].firstChild.nodeValue === "0") { - var sql = "DELETE FROM translators WHERE translatorID=?"; - return Zotero.DB.query(sql, {string: xmlnode.getAttribute('id')}); + if (translator && translator.file.exists()) { + Zotero.debug("Deleting translator '" + translator.label + "'"); + translator.file.remove(false); + } + return false; } - var sqlValues = [ - {string: xmlnode.getAttribute('id')}, - {string: xmlnode.getAttribute('minVersion')}, - {string: xmlnode.getAttribute('maxVersion')}, - {string: xmlnode.getAttribute('lastUpdated')}, - 1, // inRepository - {int: xmlnode.getElementsByTagName('priority')[0].firstChild.nodeValue}, - {int: xmlnode.getAttribute('type')}, - {string: xmlnode.getElementsByTagName('label')[0].firstChild.nodeValue}, - {string: xmlnode.getElementsByTagName('creator')[0].firstChild.nodeValue}, - // target - (xmlnode.getElementsByTagName('target').item(0) && - xmlnode.getElementsByTagName('target')[0].firstChild) - ? {string: xmlnode.getElementsByTagName('target')[0].firstChild.nodeValue} - : {null: true}, - // detectCode can not exist or be empty - (xmlnode.getElementsByTagName('detectCode').item(0) && - xmlnode.getElementsByTagName('detectCode')[0].firstChild) - ? {string: xmlnode.getElementsByTagName('detectCode')[0].firstChild.nodeValue} - : {null: true}, - {string: xmlnode.getElementsByTagName('code')[0].firstChild.nodeValue} - ]; + var metadata = { + translatorID: translatorID, + translatorType: parseInt(xmlnode.getAttribute('type')), + label: xmlnode.getElementsByTagName('label')[0].firstChild.nodeValue, + creator: xmlnode.getElementsByTagName('creator')[0].firstChild.nodeValue, + target: (xmlnode.getElementsByTagName('target').item(0) && + xmlnode.getElementsByTagName('target')[0].firstChild) + ? xmlnode.getElementsByTagName('target')[0].firstChild.nodeValue + : null, + minVersion: xmlnode.getAttribute('minVersion'), + maxVersion: xmlnode.getAttribute('maxVersion'), + priority: parseInt( + xmlnode.getElementsByTagName('priority')[0].firstChild.nodeValue + ), + inRepository: true, + lastUpdated: xmlnode.getAttribute('lastUpdated') + }; - var sql = "REPLACE INTO translators VALUES (?,?,?,?,?,?,?,?,?,?,?,?)"; - return Zotero.DB.query(sql, sqlValues); + // detectCode can not exist or be empty + var detectCode = (xmlnode.getElementsByTagName('detectCode').item(0) && + xmlnode.getElementsByTagName('detectCode')[0].firstChild) + ? xmlnode.getElementsByTagName('detectCode')[0].firstChild.nodeValue + : null; + var code = xmlnode.getElementsByTagName('code')[0].firstChild.nodeValue; + + var fileName = Zotero.Translators.getFileNameFromLabel(metadata.label); + var destFile = Zotero.getTranslatorsDirectory(); + destFile.append(fileName); + + var nsIJSON = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON); + var metadataJSON = nsIJSON.encode(metadata); + + var str = metadataJSON + "\n\n" + (detectCode ? detectCode + "\n\n" : "") + code; + + if (translator && destFile.equals(translator.file)) { + var sameFile = true; + } + + if (!sameFile && destFile.exists()) { + var msg = "Overwriting translator with same filename '" + + fileName + "'"; + Zotero.debug(msg, 1); + Zotero.debug(metadata, 1); + Components.utils.reportError(msg + " in Zotero.Schema._translatorXMLToFile()"); + } + + if (translator && translator.file.exists()) { + translator.file.remove(false); + } + + Zotero.debug("Saving translator '" + metadata.label + "'"); + Zotero.File.putContents(destFile, str); + return destFile; } /** - * Traverse an XML style node from the repository and - * update the local csl table with the style data - **/ - function _styleXMLToDB(xmlnode){ + * Traverse an XML style node from the repository and + * update the local styles folder with the style data + */ + function _styleXMLToFile(xmlnode) { // Don't split >4K chunks into multiple nodes // https://bugzilla.mozilla.org/show_bug.cgi?id=194231 xmlnode.normalize(); var uri = xmlnode.getAttribute('id'); - // - // Workaround for URI change -- delete existing versions with old URIs of updated styles - // - var re = new RegExp("http://www.zotero.org/styles/(.+)"); - var matches = uri.match(re); - - if (matches) { - var zoteroReplacements = ['chicago-author-date', 'chicago-note-bibliography']; - var purlReplacements = [ - 'apa', 'asa', 'chicago-note', 'ieee', 'mhra_note_without_bibliography', - 'mla', 'nature', 'nlm' - ]; - - if (zoteroReplacements.indexOf(matches[1]) != -1) { - var sql = "DELETE FROM csl WHERE cslID=?"; - Zotero.DB.query(sql, 'http://www.zotero.org/namespaces/CSL/' + matches[1] + '.csl'); - } - else if (purlReplacements.indexOf(matches[1]) != -1) { - var sql = "DELETE FROM csl WHERE cslID=?"; - Zotero.DB.query(sql, 'http://purl.org/net/xbiblio/csl/styles/' + matches[1] + '.csl'); - } - } - - var uri = xmlnode.getAttribute('id'); - // Delete local style if CSL code is empty if (!xmlnode.getElementsByTagName('csl')[0].firstChild) { - var sql = "DELETE FROM csl WHERE cslID=?"; - Zotero.DB.query(sql, uri); - return true; + var style = Zotero.Styles.get(uri); + if (style) { + style.file.remove(null); + } + return; } - var sqlValues = [ - {string: uri}, - {string: xmlnode.getAttribute('updated')}, - {string: xmlnode.getElementsByTagName('title')[0].firstChild.nodeValue}, - {string: xmlnode.getElementsByTagName('csl')[0].firstChild.nodeValue} - ]; + var str = xmlnode.getElementsByTagName('csl')[0].firstChild.nodeValue; - var sql = "REPLACE INTO csl VALUES (?,?,?,?)"; - return Zotero.DB.query(sql, sqlValues); + var style = Zotero.Styles.get(uri); + if (style) { + if (style.file.exists()) { + style.file.remove(false); + } + var destFile = style.file; + } + else { + // Get last part of URI for filename + var matches = uri.match(/([^\/]+)$/); + if (!matches) { + throw ("Invalid style URI '" + uri + "' from repository"); + } + var destFile = Zotero.getStylesDirectory(); + destFile.append(matches[1]); + if (destFile.exists()) { + throw ("Different style with filename '" + matches[1] + + "' already exists in Zotero.Schema._styleXMLToFile()"); + } + } + + Zotero.debug("Saving style '" + uri + "'"); + Zotero.File.putContents(destFile, str); + return; } @@ -1679,6 +1891,50 @@ Zotero.Schema = new function(){ Zotero.DB.query("CREATE TABLE storageDeleteLog (\n key TEXT PRIMARY KEY,\n timestamp INT NOT NULL\n)"); Zotero.DB.query("CREATE INDEX storageDeleteLog_timestamp ON storageDeleteLog(timestamp)"); } + + if (i==41) { + var translators = Zotero.DB.query("SELECT * FROM translators WHERE inRepository!=1"); + if (translators) { + var dir = Zotero.getTranslatorsDirectory(); + if (dir.exists()) { + dir.remove(true); + } + Zotero.getTranslatorsDirectory() + for each(var row in translators) { + var file = dir.clone(); + var fileName = Zotero.Translators.getFileNameFromLabel(row.label); + file.append(fileName); + var metadata = { translatorID: row.translatorID, translatorType: parseInt(row.translatorType), label: row.label, creator: row.creator, target: row.target ? row.target : null, minVersion: row.minVersion, maxVersion: row.maxVersion, priority: parseInt(row.priority), inRepository: row.inRepository == 1 ? true : false, lastUpdated: row.lastUpdated }; + var nsIJSON = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON); + var metadataJSON = nsIJSON.encode(metadata); + var str = metadataJSON + "\n\n" + (row.detectCode ? row.detectCode + "\n\n" : "") + row.code; + Zotero.debug("Extracting translator '" + row.label + "' from database"); + Zotero.File.putContents(file, str); + } + Zotero.Translators.init(); + } + var styles = Zotero.DB.query("SELECT * FROM csl"); + if (styles) { + var dir = Zotero.getStylesDirectory(); + if (dir.exists()) { + dir.remove(true); + } + Zotero.getStylesDirectory() + for each(var row in styles) { + var file = dir.clone(); + var matches = row.cslID.match(/([^\/]+)$/); + if (!matches) { + continue; + } + file.append(matches[1]); + Zotero.debug("Extracting styles '" + matches[1] + "' from database"); + Zotero.File.putContents(file, row.csl); + } + Zotero.Styles.init(); + } + Zotero.DB.query("DROP TABLE translators"); + Zotero.DB.query("DROP TABLE csl"); + } } _updateDBVersion('userdata', toVersion); diff --git a/chrome/content/zotero/xpcom/translate.js b/chrome/content/zotero/xpcom/translate.js index 783623b03..a61169182 100644 --- a/chrome/content/zotero/xpcom/translate.js +++ b/chrome/content/zotero/xpcom/translate.js @@ -96,6 +96,15 @@ Zotero.Translators = new function() { if(!_initialized) this.init(); return _cache[type].slice(0); } + + + /** + * @param {String} label + * @return {String} + */ + this.getFileNameFromLabel = function(label) { + return Zotero.File.getValidFileName(label) + ".js"; + } } /** diff --git a/chrome/content/zotero/xpcom/zotero.js b/chrome/content/zotero/xpcom/zotero.js index c572cdc69..b312bcd6b 100644 --- a/chrome/content/zotero/xpcom/zotero.js +++ b/chrome/content/zotero/xpcom/zotero.js @@ -262,7 +262,7 @@ var Zotero = new function(){ } Zotero.DB.startDummyStatement(); - Zotero.Schema.updateScrapersRemote(); + Zotero.Schema.updateFromRepository(); // Initialize integration web server Zotero.Integration.init(); @@ -1028,7 +1028,7 @@ Zotero.Prefs = new function(){ switch (data){ case "automaticScraperUpdates": if (this.get('automaticScraperUpdates')){ - Zotero.Schema.updateScrapersRemote(); + Zotero.Schema.updateFromRepository(); } else { Zotero.Schema.stopRepositoryTimer(); diff --git a/repotime.txt b/repotime.txt new file mode 100644 index 000000000..4b2a1cacc --- /dev/null +++ b/repotime.txt @@ -0,0 +1 @@ +2008-09-03 23:35:00 diff --git a/translators/build_zip b/translators/build_zip new file mode 100755 index 000000000..387cc1f85 --- /dev/null +++ b/translators/build_zip @@ -0,0 +1,20 @@ +#!/bin/bash +if [ -f translators.zip ]; then + rm translators.zip +fi +if [ ! -d output ]; then + mkdir output; +fi + +counter=0; +for file in *.js; do + newfile=$counter.js; + cp "$file" output/$newfile; + counter=`echo $counter + 1 | bc`; +done; + +cd output +zip ../translators.zip * +cd .. +rm -rf output +mv translators.zip .. \ No newline at end of file diff --git a/userdata.sql b/userdata.sql index 8a3a9448f..f74c07cc4 100644 --- a/userdata.sql +++ b/userdata.sql @@ -1,4 +1,4 @@ --- 40 +-- 41 -- This file creates tables containing user-specific data -- any changes made -- here must be mirrored in transition steps in schema.js::_migrateSchema() @@ -211,29 +211,6 @@ CREATE TABLE storageDeleteLog ( ); CREATE INDEX storageDeleteLog_timestamp ON storageDeleteLog(timestamp); -CREATE TABLE translators ( - translatorID TEXT PRIMARY KEY, - minVersion TEXT, - maxVersion TEXT, - lastUpdated DATETIME, - inRepository INT, - priority INT, - translatorType INT, - label TEXT, - creator TEXT, - target TEXT, - detectCode TEXT, - code TEXT -); -CREATE INDEX translators_type ON translators(translatorType); - -CREATE TABLE csl ( - cslID TEXT PRIMARY KEY, - updated DATETIME, - title TEXT, - csl TEXT -); - CREATE TABLE annotations ( annotationID INTEGER PRIMARY KEY, itemID INT,