From 448fde9ff11ef8b6dcbc50549d64435dd539a1ab Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Wed, 7 Jun 2006 01:02:59 +0000 Subject: [PATCH] Made the schema update system moderately less convoluted - Broke schema functions into separate object and got rid of DB_VERSION config constant in favor of a toVersion variable in the _migrateSchema command (which isn't technically necessary either, since the version number at the top of schema.sql is now always compared to the DB version at startup) but will help reduce the chance that someone will update the schema file without adding migration steps) - Removed Amazon scraper from schema.sql, as it will be loaded with the rest of the scrapers --- .../chromeFiles/content/scholar/xpcom/db.js | 181 +--------------- .../content/scholar/xpcom/schema.js | 195 ++++++++++++++++++ .../content/scholar/xpcom/scholar.js | 3 +- components/chnmIScholarService.js | 5 +- schema.sql | 72 ------- 5 files changed, 206 insertions(+), 250 deletions(-) create mode 100644 chrome/chromeFiles/content/scholar/xpcom/schema.js diff --git a/chrome/chromeFiles/content/scholar/xpcom/db.js b/chrome/chromeFiles/content/scholar/xpcom/db.js index 27487d66f..e2b5cd9f3 100644 --- a/chrome/chromeFiles/content/scholar/xpcom/db.js +++ b/chrome/chromeFiles/content/scholar/xpcom/db.js @@ -15,11 +15,11 @@ Scholar.DB = new function(){ this.statementQuery = statementQuery; this.getColumns = getColumns; this.getColumnHash = getColumnHash; - this.updateSchema = updateSchema; this.beginTransaction = beginTransaction; this.commitTransaction = commitTransaction; this.rollbackTransaction = rollbackTransaction; this.transactionInProgress = transactionInProgress; + this.tableExists = tableExists; ///////////////////////////////////////////////////////////////// // @@ -246,6 +246,11 @@ Scholar.DB = new function(){ } + function tableExists(table){ + return _getDBConnection().tableExists(table); + } + + function getColumns(table){ var db = _getDBConnection(); @@ -277,30 +282,6 @@ Scholar.DB = new function(){ return hash; } - /* - * Checks if the DB schema exists and is up-to-date, updating if necessary - */ - function updateSchema(){ - var DBVersion = _getDBVersion(); - - if (DBVersion > SCHOLAR_CONFIG['DB_VERSION']){ - throw("Scholar DB version is newer than config version"); - } - else if (DBVersion < SCHOLAR_CONFIG['DB_VERSION']){ - if (!DBVersion){ - Scholar.debug('Database does not exist -- creating\n'); - return _initializeSchema(); - } - - return _migrateSchema(DBVersion); - } - else if (SCHOLAR_CONFIG['DB_REBUILD']){ - if (confirm('Erase all data and recreate database from schema?')){ - return _initializeSchema(); - } - } - } - ///////////////////////////////////////////////////////////////// @@ -333,154 +314,4 @@ Scholar.DB = new function(){ return _connection; } - - - /* - * Retrieve the DB schema version - */ - function _getDBVersion(){ - if (_getDBConnection().tableExists('version')){ - return valueQuery("SELECT version FROM version;"); - } - return false; - } - - - /* - * Load in SQL schema - * - * Returns an _array_ of SQL statements for feeding into query() - */ - function _getSchemaSQL(){ - // We pull the schema from an external file so we only have to process - // it when necessary - var file = Components.classes["@mozilla.org/extensions/manager;1"] - .getService(Components.interfaces.nsIExtensionManager) - .getInstallLocation(SCHOLAR_CONFIG['GUID']) - .getItemLocation(SCHOLAR_CONFIG['GUID']); - file.append('schema.sql'); - - // Open an input stream from file - var istream = Components.classes["@mozilla.org/network/file-input-stream;1"] - .createInstance(Components.interfaces.nsIFileInputStream); - istream.init(file, 0x01, 0444, 0); - istream.QueryInterface(Components.interfaces.nsILineInputStream); - - var line = {}, sql = '', hasmore; - - // Fetch the schema version from the first line of the file - istream.readLine(line); - var schemaVersion = line.value.match(/-- ([0-9]+)/)[1]; - - do { - hasmore = istream.readLine(line); - sql += line.value + "\n"; - } while(hasmore); - - istream.close(); - - if (schemaVersion!=SCHOLAR_CONFIG['DB_VERSION']){ - throw("Scholar config version does not match schema version"); - } - - return sql; - } - - - /* - * Retrieve the version attribute of the schema SQL XML - */ - function _getSchemaSQLVersion(){ - var file = Components.classes["@mozilla.org/extensions/manager;1"] - .getService(Components.interfaces.nsIExtensionManager) - .getInstallLocation(SCHOLAR_CONFIG['GUID']) - .getItemLocation(SCHOLAR_CONFIG['GUID']); - file.append('schema.sql'); - - // Open an input stream from file - var istream = Components.classes["@mozilla.org/network/file-input-stream;1"] - .createInstance(Components.interfaces.nsIFileInputStream); - istream.init(file, 0x01, 0444, 0); - istream.QueryInterface(Components.interfaces.nsILineInputStream); - - var line = {}; - - // Fetch the schema version from the first line of the file - istream.readLine(line); - var schemaVersion = line.value.match(/-- ([0-9]+)/)[1]; - istream.close(); - - return schemaVersion; - } - - - /* - * Create new DB schema - */ - function _initializeSchema(){ - try { - beginTransaction(); - var sql = _getSchemaSQL(); - query(sql); - query("INSERT INTO version VALUES (" + SCHOLAR_CONFIG['DB_VERSION'] + ")"); - commitTransaction(); - } - catch(e){ - alert(e); - rollbackTransaction(); - } - } - - - /* - * Migrate schema from an older version, preserving data - */ - function _migrateSchema(fromVersion){ - var toVersion = SCHOLAR_CONFIG['DB_VERSION']; - var schemaVersion = _getSchemaSQLVersion(); - - if (toVersion!=schemaVersion){ - throw("Scholar config version does not match schema version"); - } - - Scholar.debug('Updating DB from version ' + fromVersion + ' to ' + toVersion + '\n'); - - // Step through version changes until we reach the current version - // - // Each block performs the changes necessary to move from the - // previous revision to that one. - // - // N.B. Be sure to call _updateDBVersion(i) at the end of each block and - // update SCHOLAR_CONFIG['DB_VERSION'] to the target version - for (var i=parseInt(fromVersion) + 1; i<=toVersion; i++){ - - if (i==9){ - Scholar.DB.query("DROP TABLE IF EXISTS objectCreators; " - + "DROP TABLE IF EXISTS objectData; DROP TABLE IF EXISTS objectKeywords; " - + "DROP TABLE IF EXISTS objectTypeFields; DROP TABLE IF EXISTS objectTypes; " - + "DROP TABLE IF EXISTS objects; DROP TABLE IF EXISTS treeOrder;"); - _updateDBVersion(i); - } - - // For now, just wipe and recreate - if (i==13){ - Scholar.DB.query("DROP TABLE IF EXISTS folders; " - + "DROP TABLE IF EXISTS treeStructure;"); - _initializeSchema(); - } - - if (i==14){ - // do stuff - // _updateDBVersion(i); - } - } - } - - - /* - * Update the DB schema version tag of an existing database - */ - function _updateDBVersion(version){ - return query("UPDATE version SET version=" + version); - } } diff --git a/chrome/chromeFiles/content/scholar/xpcom/schema.js b/chrome/chromeFiles/content/scholar/xpcom/schema.js new file mode 100644 index 000000000..cb7f80ed4 --- /dev/null +++ b/chrome/chromeFiles/content/scholar/xpcom/schema.js @@ -0,0 +1,195 @@ +Scholar.Schema = new function(){ + var _dbVersion; + var _schemaVersion; + + this.updateSchema = updateSchema; + + /* + * Checks if the DB schema exists and is up-to-date, updating if necessary + */ + function updateSchema(){ + var dbVersion = _getDBVersion(); + var schemaVersion = _getSchemaSQLVersion(); + + if (dbVersion > schemaVersion){ + throw("Scholar DB version is newer than schema version"); + } + else if (dbVersion < schemaVersion){ + if (!dbVersion){ + Scholar.debug('Database does not exist -- creating\n'); + return _initializeSchema(); + } + + return _migrateSchema(dbVersion); + } + else if (SCHOLAR_CONFIG['DB_REBUILD']){ + if (confirm('Erase all data and recreate database from schema?')){ + return _initializeSchema(); + } + } + } + + + ///////////////////////////////////////////////////////////////// + // + // Private methods + // + ///////////////////////////////////////////////////////////////// + + /* + * Retrieve the DB schema version + */ + function _getDBVersion(){ + if (_dbVersion){ + return _dbVersion; + } + + if (Scholar.DB.tableExists('version')){ + var dbVersion = Scholar.DB.valueQuery("SELECT version FROM version;"); + _dbVersion = dbVersion; + return dbVersion; + } + return false; + } + + + /* + * Retrieve the version attribute of the schema SQL XML + */ + function _getSchemaSQLVersion(){ + if (_schemaVersion){ + return _schemaVersion; + } + + var file = Components.classes["@mozilla.org/extensions/manager;1"] + .getService(Components.interfaces.nsIExtensionManager) + .getInstallLocation(SCHOLAR_CONFIG['GUID']) + .getItemLocation(SCHOLAR_CONFIG['GUID']); + file.append('schema.sql'); + + // Open an input stream from file + var istream = Components.classes["@mozilla.org/network/file-input-stream;1"] + .createInstance(Components.interfaces.nsIFileInputStream); + istream.init(file, 0x01, 0444, 0); + istream.QueryInterface(Components.interfaces.nsILineInputStream); + + var line = {}; + + // Fetch the schema version from the first line of the file + istream.readLine(line); + var schemaVersion = line.value.match(/-- ([0-9]+)/)[1]; + istream.close(); + + _schemaVersion = schemaVersion; + return schemaVersion; + } + + + /* + * Load in SQL schema + * + * Returns an _array_ of SQL statements for feeding into query() + */ + function _getSchemaSQL(){ + // We pull the schema from an external file so we only have to process + // it when necessary + var file = Components.classes["@mozilla.org/extensions/manager;1"] + .getService(Components.interfaces.nsIExtensionManager) + .getInstallLocation(SCHOLAR_CONFIG['GUID']) + .getItemLocation(SCHOLAR_CONFIG['GUID']); + file.append('schema.sql'); + + // Open an input stream from file + var istream = Components.classes["@mozilla.org/network/file-input-stream;1"] + .createInstance(Components.interfaces.nsIFileInputStream); + istream.init(file, 0x01, 0444, 0); + istream.QueryInterface(Components.interfaces.nsILineInputStream); + + var line = {}, sql = '', hasmore; + + // Skip the first line, which contains the schema version + istream.readLine(line); + //var schemaVersion = line.value.match(/-- ([0-9]+)/)[1]; + + do { + hasmore = istream.readLine(line); + sql += line.value + "\n"; + } while(hasmore); + + istream.close(); + + return sql; + } + + + /* + * Create new DB schema + */ + function _initializeSchema(){ + try { + beginTransaction(); + var sql = _getSchemaSQL(); + query(sql); + query("INSERT INTO version VALUES (" + _getSchemaSQLVersion() + ")"); + commitTransaction(); + } + catch(e){ + alert(e); + rollbackTransaction(); + } + } + + + /* + * Migrate schema from an older version, preserving data + */ + function _migrateSchema(fromVersion){ + // + // Change this value to match the schema version + // + var toVersion = 13; + + if (toVersion != _getSchemaSQLVersion()){ + throw('Schema version does not match version in _migrateSchema()'); + } + + Scholar.debug('Updating DB from version ' + fromVersion + ' to ' + toVersion + '\n'); + + Scholar.DB.beginTransaction(); + + // Step through version changes until we reach the current version + // + // Each block performs the changes necessary to move from the + // previous revision to that one. + for (var i=parseInt(fromVersion) + 1; i<=toVersion; i++){ + if (i==9){ + Scholar.DB.query("DROP TABLE IF EXISTS objectCreators; " + + "DROP TABLE IF EXISTS objectData; DROP TABLE IF EXISTS objectKeywords; " + + "DROP TABLE IF EXISTS objectTypeFields; DROP TABLE IF EXISTS objectTypes; " + + "DROP TABLE IF EXISTS objects; DROP TABLE IF EXISTS treeOrder;"); + } + + // For now, just wipe and recreate + if (i==13){ + Scholar.DB.query("DROP TABLE IF EXISTS folders; " + + "DROP TABLE IF EXISTS treeStructure;"); + _initializeSchema(); + } + + if (i==14){ + // do stuff + } + } + + _updateDBVersion(i-1); + Scholar.DB.commitTransaction(); + } + + + /* + * Update the DB schema version tag of an existing database + */ + function _updateDBVersion(version){ + return Scholar.DB.query("UPDATE version SET version=" + version); + } +} diff --git a/chrome/chromeFiles/content/scholar/xpcom/scholar.js b/chrome/chromeFiles/content/scholar/xpcom/scholar.js index dc82a5ebf..a892c3950 100644 --- a/chrome/chromeFiles/content/scholar/xpcom/scholar.js +++ b/chrome/chromeFiles/content/scholar/xpcom/scholar.js @@ -1,7 +1,6 @@ const SCHOLAR_CONFIG = { GUID: 'scholar@chnm.gmu.edu', DB_FILE: 'scholar.sqlite', - DB_VERSION: 13, // must match version at top of schema.sql DB_REBUILD: false, // erase DB and recreate from schema DEBUG_LOGGING: true, DEBUG_TO_CONSOLE: true // dump debug messages to console rather than (much slower) Debug Logger @@ -34,7 +33,7 @@ var Scholar = new function(){ return false; } - Scholar.DB.updateSchema(); + Scholar.Schema.updateSchema(); // Load in the localization stringbundle for use by getString(name) var src = 'chrome://scholar/locale/scholar.properties'; diff --git a/components/chnmIScholarService.js b/components/chnmIScholarService.js index d6fd72e05..1c24d1b1e 100644 --- a/components/chnmIScholarService.js +++ b/components/chnmIScholarService.js @@ -18,11 +18,14 @@ Cc["@mozilla.org/moz/jssubscript-loader;1"] .getService(Ci.mozIJSSubScriptLoader) .loadSubScript("chrome://scholar/content/xpcom/scholar.js"); - Cc["@mozilla.org/moz/jssubscript-loader;1"] .getService(Ci.mozIJSSubScriptLoader) .loadSubScript("chrome://scholar/content/xpcom/db.js"); +Cc["@mozilla.org/moz/jssubscript-loader;1"] + .getService(Ci.mozIJSSubScriptLoader) + .loadSubScript("chrome://scholar/content/xpcom/schema.js"); + Cc["@mozilla.org/moz/jssubscript-loader;1"] .getService(Ci.mozIJSSubScriptLoader) .loadSubScript("chrome://scholar/content/xpcom/data_access.js"); diff --git a/schema.sql b/schema.sql index 31dd89995..10e1bfbfb 100644 --- a/schema.sql +++ b/schema.sql @@ -251,75 +251,3 @@ INSERT INTO collectionItems VALUES (6856, 13, 1); INSERT INTO collectionItems VALUES (7373, 15, 0); INSERT INTO collectionItems VALUES (1241, 12, 0); - - INSERT INTO "scrapers" VALUES(1, NULL, NULL, NULL, 'Amazon.com Scraper', 'Simon Kornblith', '^http://www.amazon.com/gp/product/', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#''; -var prefixDC = ''http://purl.org/dc/elements/1.1/''; -var prefixDCMI = ''http://purl.org/dc/dcmitype/''; -var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/''; - -var namespace = doc.documentElement.namespaceURI; -var nsResolver = namespace ? function(prefix) { -if (prefix == ''x'') return namespace; else return null; -} : null; - -var getNode = function(doc, contextNode, xpath, nsResolver) { -return doc.evaluate(xpath, contextNode, nsResolver, XPathResult.ANY_TYPE,null).iterateNext(); -} - -var cleanString = function(s) { -s = utilities.trimString(s); -return s.replace(/ +/g, " "); -} - -var uri = doc.location.href; - -model.addStatement(uri, prefixRDF + "type", prefixDCMI + "text", false); - -// Retrieve authors -var xpath = ''/html/body/table/tbody/tr/td[2]/form/div[@class="buying"]/a''; -var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver); -for (var i = 0; i < elmts.length; i++) { -var elmt = elmts[i]; - -model.addStatement(uri, prefixDC + ''creator'', cleanString(getNode(doc, elmt, ''./text()[1]'', nsResolver).nodeValue), false); // Use your own type here -} - -// Retrieve data from "Product Details" box -var xpath = ''/html/body/table/tbody/tr/td[2]/table/tbody/tr/td[@class="bucket"]/div[@class="content"]/ul/li''; -var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver); -for (var i = 0; i < elmts.length; i++) { -var elmt = elmts[i]; -var attribute = cleanString(getNode(doc, elmt, ''./B[1]/text()[1]'', nsResolver).nodeValue); -if(getNode(doc, elmt, ''./text()[1]'', nsResolver)) { -var value = cleanString(getNode(doc, elmt, ''./text()[1]'', nsResolver).nodeValue); - -if(attribute == "Publisher:") { -if(value.lastIndexOf("(") != -1) { -var date = value.substring(value.lastIndexOf("(")+1, value.length-1); -value = value.substring(0, value.lastIndexOf("(")-1); -} -if(value.lastIndexOf(";") != -1) { -var edition = value.substring(value.lastIndexOf(";")+2, value.length); -value = value.substring(0, value.lastIndexOf(";")); -} -model.addStatement(uri, prefixDC + ''publisher'', value); -model.addStatement(uri, prefixDC + ''date'', date); -model.addStatement(uri, prefixDC + ''hasVersion'', edition); -} else if(attribute == "Language:") { -model.addStatement(uri, prefixDC + ''language'', value); -} else if(attribute == "ISBN:") { -model.addStatement(uri, prefixDC + ''identifier'', ''ISBN ''+value); -} else if(value.substring(value.indexOf(" ")+1, value.length) == "pages") { -model.addStatement(uri, prefixDummy + ''pages'', value.substring(0, value.indexOf(" "))); -model.addStatement(uri, prefixDC + ''medium'', attribute.substring(0, attribute.indexOf(":"))); -} -} -} - -var xpath = ''/html/body/table/tbody/tr/td[2]/form/div[@class="buying"]/b[@class="sans"]''; -var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver); -var title = cleanString(getNode(doc, elmts[0], ''./text()[1]'', nsResolver).nodeValue); -if(title.lastIndexOf("(") != -1 && title.lastIndexOf(")") == title.length-1) { -title = title.substring(0, title.lastIndexOf("(")-1); -} -model.addStatement(uri, prefixDC + ''title'', title);');