393807b152
Integrated the scrapers with the schema update mechanism. Changed a bunch of schema methods to handle both schema.sql and scrapers.sql (or others, if need be) and altered the version table to track mu ltiple versions for different files. This theoretically should detect that the version table has changed and force a reinitialization of the DB--let me know if there are problems.
1054 lines
36 KiB
SQL
1054 lines
36 KiB
SQL
-- 1
|
|
DELETE FROM scrapers;
|
|
INSERT INTO "scrapers" VALUES(1, NULL, NULL, 20060603002000, 'Amazon.com Scraper', 'Simon Kornblith', '^http://www\.amazon\.com/gp/product/', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
var namespace = doc.documentElement.namespaceURI;
|
|
var nsResolver = namespace ? function(prefix) {
|
|
if (prefix == ''x'') return namespace; else return null;
|
|
} : null;
|
|
|
|
var getNode = function(doc, contextNode, xpath, nsResolver) {
|
|
return doc.evaluate(xpath, contextNode, nsResolver, XPathResult.ANY_TYPE,null).iterateNext();
|
|
}
|
|
|
|
var cleanString = function(s) {
|
|
s = utilities.trimString(s);
|
|
return s.replace(/ +/g, " ");
|
|
}
|
|
|
|
var uri = doc.location.href;
|
|
|
|
model.addStatement(uri, prefixRDF + "type", prefixDummy + "book", false);
|
|
|
|
// Retrieve authors
|
|
var xpath = ''/html/body/table/tbody/tr/td[2]/form/div[@class="buying"]/a'';
|
|
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
|
|
for (var i = 0; i < elmts.length; i++) {
|
|
var elmt = elmts[i];
|
|
|
|
model.addStatement(uri, prefixDC + ''creator'', cleanString(getNode(doc, elmt, ''./text()[1]'', nsResolver).nodeValue), false); // Use your own type here
|
|
}
|
|
|
|
// Retrieve data from "Product Details" box
|
|
var xpath = ''/html/body/table/tbody/tr/td[2]/table/tbody/tr/td[@class="bucket"]/div[@class="content"]/ul/li'';
|
|
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
|
|
for (var i = 0; i < elmts.length; i++) {
|
|
var elmt = elmts[i];
|
|
var attribute = cleanString(getNode(doc, elmt, ''./B[1]/text()[1]'', nsResolver).nodeValue);
|
|
if(getNode(doc, elmt, ''./text()[1]'', nsResolver)) {
|
|
var value = cleanString(getNode(doc, elmt, ''./text()[1]'', nsResolver).nodeValue);
|
|
|
|
if(attribute == "Publisher:") {
|
|
if(value.lastIndexOf("(") != -1) {
|
|
var date = value.substring(value.lastIndexOf("(")+1, value.length-1);
|
|
value = value.substring(0, value.lastIndexOf("(")-1);
|
|
}
|
|
if(value.lastIndexOf(";") != -1) {
|
|
var edition = value.substring(value.lastIndexOf(";")+2, value.length);
|
|
value = value.substring(0, value.lastIndexOf(";"));
|
|
}
|
|
model.addStatement(uri, prefixDC + ''publisher'', value);
|
|
model.addStatement(uri, prefixDC + ''date'', date);
|
|
model.addStatement(uri, prefixDC + ''hasVersion'', edition);
|
|
} else if(attribute == "Language:") {
|
|
model.addStatement(uri, prefixDC + ''language'', value);
|
|
} else if(attribute == "ISBN:") {
|
|
model.addStatement(uri, prefixDC + ''identifier'', ''ISBN ''+value);
|
|
} else if(value.substring(value.indexOf(" ")+1, value.length) == "pages") {
|
|
model.addStatement(uri, prefixDummy + ''pages'', value.substring(0, value.indexOf(" ")));
|
|
model.addStatement(uri, prefixDC + ''medium'', attribute.substring(0, attribute.indexOf(":")));
|
|
}
|
|
}
|
|
}
|
|
|
|
var xpath = ''/html/body/table/tbody/tr/td[2]/form/div[@class="buying"]/b[@class="sans"]'';
|
|
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
|
|
var title = cleanString(getNode(doc, elmts[0], ''./text()[1]'', nsResolver).nodeValue);
|
|
if(title.lastIndexOf("(") != -1 && title.lastIndexOf(")") == title.length-1) {
|
|
title = title.substring(0, title.lastIndexOf("(")-1);
|
|
}
|
|
model.addStatement(uri, prefixDC + ''title'', title);');
|
|
|
|
INSERT INTO "scrapers" VALUES(2, NULL, NULL, 20060603002000, 'WorldCat Scraper', 'Simon Kornblith', '^http://newfirstsearch\.oclc\.org/WebZ/',
|
|
'if(doc.title == ''FirstSearch: WorldCat Detailed Record'') {
|
|
return true;
|
|
}
|
|
return false;',
|
|
'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
var sessionRegexp = /(?:\?|\:)sessionid=([^?:]+)(?:\?|\:|$)/;
|
|
var numberRegexp = /(?:\?|\:)recno=([^?:]+)(?:\?|\:|$)/;
|
|
var resultsetRegexp = /(?:\?|\:)resultset=([^?:]+)(?:\?|\:|$)/;
|
|
var lineRegexp = /^([\w() ]+): *(.*)$/;
|
|
var publisherRegexp = /^(.*), (.*?),?$/;
|
|
|
|
var uri = doc.location.href;
|
|
|
|
var sMatch = sessionRegexp.exec(uri);
|
|
var sessionid = sMatch[1];
|
|
|
|
var nMatch = numberRegexp.exec(uri);
|
|
if(nMatch) {
|
|
var number = nMatch[1];
|
|
} else {
|
|
number = 1;
|
|
}
|
|
|
|
var rMatch = resultsetRegexp.exec(uri);
|
|
if(rMatch) {
|
|
var resultset = rMatch[1];
|
|
} else {
|
|
// It''s in an XPCNativeWrapper, so we have to do this black magic
|
|
resultset = doc.forms.namedItem(''main'').elements.namedItem(''resultset'').value;
|
|
}
|
|
|
|
var newUri = ''http://newfirstsearch.oclc.org/WebZ/DirectExport?numrecs=10:smartpage=directexport:entityexportnumrecs=10:entityexportresultset='' + resultset + '':entityexportrecno='' + number + '':sessionid='' + sessionid + '':entitypagenum=35:0'';
|
|
|
|
model.addStatement(uri, prefixRDF + "type", prefixDummy + "book", false);
|
|
|
|
function cleanAuthor(author) {
|
|
author = author.replace(/^[\s\.\,\/\[\]\:]+/, '''');
|
|
author = author.replace(/[\s\,\/\[\]\:\.]+$/, '''');
|
|
// Add period for initials
|
|
if(author.substring(author.length-2, author.length-1) == " ") {
|
|
author += ".";
|
|
}
|
|
var splitNames = author.split('', '');
|
|
if(splitNames.length > 1) {
|
|
author = splitNames[1]+'' ''+splitNames[0];
|
|
}
|
|
return author;
|
|
}
|
|
|
|
utilities.HTTPUtilities.doPost(newUri, ''exportselect=record&exporttype=plaintext'', null, function(text) {
|
|
var lines = text.split(''\n'');
|
|
for(var i=0;i<lines.length;i++) {
|
|
match = lineRegexp.exec(lines[i]);
|
|
if(match) {
|
|
if(match[1] == ''Title'') {
|
|
var title = match[2];
|
|
if(!lineRegexp.test(lines[i+1])) {
|
|
i++;
|
|
title += '' ''+lines[i];
|
|
}
|
|
if(title.substring(title.length-2) == " /") {
|
|
title = title.substring(0, title.length-2);
|
|
}
|
|
model.addStatement(uri, prefixDC + ''title'', title);
|
|
} else if(match[1] == ''Author(s)'') {
|
|
var authors = match[2].split('';'');
|
|
if(authors) {
|
|
model.addStatement(uri, prefixDC + ''creator'', cleanAuthor(authors[0]));
|
|
for(var j=1; j<authors.length; j+=2) {
|
|
if(authors[j-1].substring(0, 1) == ''('') {
|
|
j++;
|
|
}
|
|
model.addStatement(uri, prefixDC + ''creator'', cleanAuthor(authors[j]));
|
|
}
|
|
} else {
|
|
model.addStatement(uri, prefixDC + ''creator'', utilities.trimString(match[2]));
|
|
}
|
|
} else if(match[1] == ''Publication'') {
|
|
// Don''t even try to deal with this. The WorldCat metadata is of poor enough quality that this isn''t worth it.
|
|
match[2] = utilities.trimString(match[2]);
|
|
if(match[2].substring(match[2].length-1) == '','') {
|
|
match[2] = match[2].substring(0, match[2].length-1);
|
|
}
|
|
model.addStatement(uri, prefixDC + ''publisher'', match[2]);
|
|
} else if(match[1] == ''Language'') {
|
|
model.addStatement(uri, prefixDC + ''language'', utilities.trimString(match[2]));
|
|
} else if(match[1] == ''Standard No'') {
|
|
var identifiers = match[2].split(/ +/);
|
|
var j=0;
|
|
while(j<(identifiers.length-1)) {
|
|
var type = identifiers[j].substring(0, identifiers[j].length-1);
|
|
var lastChar;
|
|
var value;
|
|
|
|
j++;
|
|
while(j<identifiers.length && (lastChar = identifiers[j].substring(identifiers[j].length-1)) != '':'') {
|
|
if(identifiers[j].substring(0, 1) != ''('') {
|
|
if(lastChar == '';'') {
|
|
value = identifiers[j].substring(0, identifiers[j].length-1);
|
|
} else {
|
|
value = identifiers[j];
|
|
}
|
|
model.addStatement(uri, prefixDC + ''identifier'', type + '' '' + value);
|
|
}
|
|
j++;
|
|
}
|
|
}
|
|
} else if(match[1] == ''Year'') {
|
|
model.addStatement(uri, prefixDC + ''year'', match[2]);
|
|
}
|
|
}
|
|
}
|
|
|
|
done();
|
|
})
|
|
wait();');
|
|
|
|
INSERT INTO "scrapers" VALUES(3, NULL, NULL, 20060603002000, 'LOC/Voyager WebVoyage Scraper', 'Simon Kornblith', 'Pwebrecon\.cgi',
|
|
'try {
|
|
if(doc.forms.namedItem(''frm'').elements.namedItem(''RC'')) {
|
|
return false;
|
|
}
|
|
var export_options = doc.forms.namedItem(''frm'').elements.namedItem(''RD'').options;
|
|
for(i in export_options) {
|
|
if(export_options[i].text == ''Latin1 MARC''
|
|
|| export_options[i].text == ''Raw MARC''
|
|
|| export_options[i].text == ''UTF-8''
|
|
|| export_options[i].text == ''MARC (Unicode/UTF-8)''
|
|
|| export_options[i].text == ''MARC (non-Unicode/MARC-8)'') {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
} catch(e) {
|
|
return false;
|
|
}',
|
|
'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
var uri = doc.location.href;
|
|
|
|
var raw, unicode, latin1;
|
|
|
|
var form = doc.forms.namedItem(''frm'');
|
|
var newUri = form.action;
|
|
var postString = '''';
|
|
for(i in form.elements) {
|
|
if(form.elements[i].type == ''HIDDEN'' || form.elements[i].type == ''hidden'') {
|
|
postString += escape(form.elements[i].name)+''=''+escape(form.elements[i].value)+''&'';
|
|
}
|
|
}
|
|
|
|
var export_options = form.elements.namedItem(''RD'').options;
|
|
for(i in export_options) {
|
|
if(export_options[i].text == ''Raw MARC''
|
|
|| export_options[i].text == ''MARC (non-Unicode/MARC-8)'') {
|
|
raw = i;
|
|
} if(export_options[i].text == ''Latin1 MARC'') {
|
|
latin1 = i;
|
|
} else if(export_options[i].text == ''UTF-8''
|
|
|| export_options[i].text == ''MARC (Unicode/UTF-8)'') {
|
|
unicode = i;
|
|
}
|
|
}
|
|
postString += ''RD=''+i+''&MAILADDY=&SAVE=Press+to+SAVE+or+PRINT'';
|
|
|
|
model.addStatement(uri, prefixRDF + ''type'', prefixDummy + ''book'', false);
|
|
|
|
// No idea why this doesn''t work as post
|
|
utilities.HTTPUtilities.doGet(newUri+''?''+postString, null, function(text) {
|
|
var record = new MARC_Record();
|
|
record.load(text, "binary");
|
|
model = utilities.importMARCRecord(record, uri, model);
|
|
done();
|
|
})
|
|
wait();');
|
|
|
|
INSERT INTO "scrapers" VALUES(4, NULL, NULL, 20060603002000, 'JSTOR Scraper', 'Simon Kornblith', '^http://www\.jstor\.org/(?:view|browse)', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
var namespace = doc.documentElement.namespaceURI;
|
|
var nsResolver = namespace ? function(prefix) {
|
|
if (prefix == ''x'') return namespace; else return null;
|
|
} : null;
|
|
|
|
var getNode = function(doc, contextNode, xpath, nsResolver) {
|
|
return doc.evaluate(xpath, contextNode, nsResolver, XPathResult.ANY_TYPE,null).iterateNext();
|
|
}
|
|
|
|
var cleanString = function(s) {
|
|
s = utilities.trimString(s);
|
|
return s.replace(/ +/g, " ");
|
|
}
|
|
|
|
var uri = doc.location.href;
|
|
|
|
|
|
// If this is a view page, find the link to the citation
|
|
var xpath = ''/html/body/div[@class="indent"]/center/font/p/a[@class="nav"]'';
|
|
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
|
|
if(!elmts) {
|
|
var xpath = ''/html/body/div[@class="indent"]/center/p/font/a[@class="nav"]'';
|
|
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
|
|
}
|
|
if(!elmts) {
|
|
exit;
|
|
}
|
|
var saveCitation = getNode(doc, elmts[0], ''.'', nsResolver).href;
|
|
var viewSavedCitations = getNode(doc, elmts[1], ''.'', nsResolver).href;
|
|
saveCitation = saveCitation.replace(''citationAction=remove'', ''citationAction=save'');
|
|
|
|
// Parse save citation link
|
|
var importantCitationRegexp = /userID.*$/;
|
|
var match = importantCitationRegexp.exec(viewSavedCitations);
|
|
var postData = match[0]+''&citationAction=removeAll&confirmRemAll=on'';
|
|
utilities.HTTPUtilities.doPost(''http://www.jstor.org/browse'', postData, null, function() { // clear marked
|
|
utilities.HTTPUtilities.doGet(saveCitation, null, function() { // mark this
|
|
utilities.HTTPUtilities.doGet(''http://www.jstor.org/browse/citations.txt?exportAction=Save+as+Text+File&exportFormat=cm&''+match[0], null, function(text) {
|
|
// get marked
|
|
var lines = text.split("\n");
|
|
var haveStarted = false;
|
|
|
|
var data = new Object();
|
|
data[prefixDC + "title"] = new Array();
|
|
data[prefixDC + "creator"] = new Array();
|
|
data[prefixDummy + "publication"] = new Array();
|
|
data[prefixDummy + "volume"] = new Array();
|
|
data[prefixDummy + "number"] = new Array();
|
|
data[prefixDummy + "series"] = new Array();
|
|
data[prefixDC + "date"] = new Array();
|
|
data[prefixDummy + "pages"] = new Array();
|
|
data[prefixDC + "identifier"] = new Array();
|
|
data[prefixDC + "publisher"] = new Array();
|
|
|
|
var stableURL;
|
|
|
|
for(i in lines) {
|
|
if(haveStarted) {
|
|
var fieldCode = lines[i].substring(0, 2);
|
|
var fieldContent = cleanString(lines[i].substring(5));
|
|
|
|
if(lines[i].substring(2, 5) != " : ") {
|
|
break;
|
|
}
|
|
|
|
if(fieldCode == "TI") {
|
|
data[prefixDC + "title"].push(fieldContent);
|
|
} else if(fieldCode == "AU") {
|
|
var authors = fieldContent.split(";");
|
|
for(j in authors) {
|
|
var author = authors[j];
|
|
var splitNames = author.split('', '');
|
|
if(splitNames) {
|
|
author = splitNames[1]+'' ''+splitNames[0];
|
|
}
|
|
data[prefixDC + "creator"].push(author);
|
|
}
|
|
} else if(fieldCode == "SO") {
|
|
data[prefixDummy + "publication"].push(fieldContent);
|
|
} else if(fieldCode == "VO") {
|
|
data[prefixDummy + "volume"].push(fieldContent);
|
|
} else if(fieldCode == "NO") {
|
|
data[prefixDummy + "number"].push(fieldContent);
|
|
} else if(fieldCode == "SE") {
|
|
data[prefixDummy + "series"].push(fieldContent);
|
|
} else if(fieldCode == "DA") {
|
|
data[prefixDC + "date"].push(fieldContent);
|
|
} else if(fieldCode == "PP") {
|
|
data[prefixDummy + "pages"].push(fieldContent);
|
|
} else if(fieldCode == "EI") {
|
|
stableURL = fieldContent;
|
|
} else if(fieldCode == "IN") {
|
|
data[prefixDC + "identifier"].push("ISSN "+fieldContent);
|
|
} else if(fieldCode == "PB") {
|
|
data[prefixDC + "publisher"].push(fieldContent);
|
|
}
|
|
}
|
|
if(lines[i].substring(0,3) == "<1>") {
|
|
haveStarted = true;
|
|
}
|
|
}
|
|
|
|
// Loop through again so that we can add with the stableURL
|
|
model.addStatement(stableURL, prefixRDF + "type", prefixDummy + "journal", false);
|
|
for(i in data) {
|
|
if(data[i].length) {
|
|
for(j in data[i]) {
|
|
model.addStatement(stableURL, i, data[i][j]);
|
|
}
|
|
}
|
|
}
|
|
|
|
done();
|
|
})
|
|
})
|
|
});
|
|
|
|
wait();');
|
|
|
|
INSERT INTO "scrapers" VALUES(5, NULL, NULL, 20060603002000, 'History Cooperative Scraper', 'Simon Kornblith', '^http://www\.historycooperative\.org/journals/.+/.+/.+\.html', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
var uri = doc.location.href;
|
|
|
|
model.addStatement(uri, prefixRDF + "type", prefixDummy + "journal", false);
|
|
|
|
var month, year;
|
|
|
|
var metaTags = doc.getElementsByTagName("meta");
|
|
|
|
function associateMeta(field, rdfUri) {
|
|
var field = metaTags.namedItem(field);
|
|
if(field) {
|
|
model.addStatement(uri, rdfUri, field.getAttribute("content"), false);
|
|
}
|
|
}
|
|
|
|
associateMeta("Title", prefixDC + "title");
|
|
associateMeta("Journal", prefixDummy + "publication");
|
|
associateMeta("Volume", prefixDummy + "volume");
|
|
associateMeta("Issue", prefixDummy + "number");
|
|
|
|
var author = metaTags.namedItem("Author");
|
|
if(author) {
|
|
var authors = author.getAttribute("content").split(" and ");
|
|
for(j in authors) {
|
|
model.addStatement(uri, prefixDC + "creator", authors[j], false);
|
|
}
|
|
}
|
|
|
|
var month = metaTags.namedItem("PublicationMonth");
|
|
var year = metaTags.namedItem("PublicationYear");
|
|
if(month && year) {
|
|
model.addStatement(uri, prefixDC + "date", month.getAttribute("content")+" "+year.getAttribute("content"), false);
|
|
}
|
|
');
|
|
|
|
INSERT INTO "scrapers" VALUES(6, NULL, NULL, 20060603002000, 'InnoPAC Scraper', 'Simon Kornblith', '^http://[^/]+/search/[^/]+/[^/]+/1\%2C[^/]+/frameset\&FF=', NULL,
|
|
'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
var uri = doc.location.href;
|
|
|
|
var matchRegexp = new RegExp(''^(http://[^/]+/search/[^/]+/[^/]+/1\%2C[^/]+/)frameset(.+)$'');
|
|
var m = matchRegexp.exec(uri);
|
|
var newUri = m[1]+''marc''+m[2];
|
|
|
|
utilities.loadDocument(newUri, browser, function(newBrowser) {
|
|
newDoc = newBrowser.contentDocument;
|
|
|
|
var namespace = newDoc.documentElement.namespaceURI;
|
|
var nsResolver = namespace ? function(prefix) {
|
|
if (prefix == ''x'') return namespace; else return null;
|
|
} : null;
|
|
|
|
var getNode = function(doc, contextNode, xpath, nsResolver) {
|
|
return doc.evaluate(xpath, contextNode, nsResolver, XPathResult.ANY_TYPE,null).iterateNext();
|
|
}
|
|
|
|
var xpath = ''//pre'';
|
|
var elmts = utilities.gatherElementsOnXPath(newDoc, newDoc, xpath, nsResolver);
|
|
|
|
var text = getNode(doc, elmts[0], ''./text()[1]'', nsResolver).nodeValue;
|
|
|
|
var record = new MARC_Record();
|
|
record.load(text, "MARC_PAC");
|
|
model = utilities.importMARCRecord(record, uri, model);
|
|
done();
|
|
}, function() {})
|
|
|
|
wait();');
|
|
|
|
INSERT INTO "scrapers" VALUES(7, NULL, NULL, 20060603002000, 'SIRSI Scraper', 'Simon Kornblith', '/uhtbin/cgisirsi',
|
|
'var namespace = doc.documentElement.namespaceURI;
|
|
var nsResolver = namespace ? function(prefix) {
|
|
if (prefix == ''x'') return namespace; else return null;
|
|
} : null;
|
|
|
|
var xpath = ''//tr[th[@class="viewmarctags"]][td[@class="viewmarctags"]]'';
|
|
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
|
|
if(elmts.length) {
|
|
return true;
|
|
}
|
|
return false;',
|
|
'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
var namespace = doc.documentElement.namespaceURI;
|
|
var nsResolver = namespace ? function(prefix) {
|
|
if (prefix == ''x'') return namespace; else return null;
|
|
} : null;
|
|
|
|
var getNode = function(doc, contextNode, xpath, nsResolver) {
|
|
return doc.evaluate(xpath, contextNode, nsResolver, XPathResult.ANY_TYPE,null).iterateNext();
|
|
}
|
|
|
|
function stringTrimmer(x) {
|
|
var x = x.replace(/^[^\w(]+/, "");
|
|
return x.replace(/[^\w)]+$/, "");
|
|
}
|
|
|
|
function getAnyNumber(x) {
|
|
var re = /[0-9]+/;
|
|
var m = re.exec(x);
|
|
if(m) {
|
|
return m[0];
|
|
}
|
|
}
|
|
|
|
function getISBN(x) {
|
|
var re = /^[0-9](?:[0-9X]+)/;
|
|
var m = re.exec(x);
|
|
if(m) {
|
|
return m[0];
|
|
}
|
|
}
|
|
|
|
function cleanAuthor(author) {
|
|
author = author.replace(/^[\s\.\,\/\[\]\:]+/, '''');
|
|
author = author.replace(/[\s\,\/\[\]\:\.]+$/, '''');
|
|
// Add period for initials
|
|
if(author.substring(author.length-2, author.length-1) == " ") {
|
|
author += ".";
|
|
}
|
|
var splitNames = author.split('', '');
|
|
if(splitNames.length > 1) {
|
|
author = splitNames[1]+'' ''+splitNames[0];
|
|
}
|
|
return author;
|
|
}
|
|
|
|
var uri = doc.location.href;
|
|
var data = new Object();
|
|
|
|
var xpath = ''//tr[th[@class="viewmarctags"]][td[@class="viewmarctags"]]'';
|
|
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
|
|
for (var i = 0; i < elmts.length; i++) {
|
|
var elmt = elmts[i];
|
|
try {
|
|
var node = getNode(doc, elmt, ''./TD[1]/A[1]/text()[1]'', nsResolver);
|
|
if(!node) {
|
|
var node = getNode(doc, elmt, ''./TD[1]/text()[1]'', nsResolver);
|
|
}
|
|
if(node) {
|
|
var field = stringTrimmer(getNode(doc, elmt, ''./TH[1]/text()[1]'', nsResolver).nodeValue);
|
|
field = field.toLowerCase();
|
|
var value = stringTrimmer(node.nodeValue);
|
|
var rdfUri = null;
|
|
if(field == "publisher") {
|
|
rdfUri = prefixDC + ''publisher'';
|
|
} else if(field == "pub date") {
|
|
rdfUri = prefixDC + ''date'';
|
|
value = getAnyNumber(value);
|
|
} else if(field == "isbn") {
|
|
rdfUri = prefixDC + ''identifier'';
|
|
value = ''ISBN ''+getISBN(value);
|
|
} else if(field == "title") {
|
|
rdfUri = prefixDC + ''title'';
|
|
var titleParts = value.split(" / ");
|
|
value = titleParts[0];
|
|
} else if(field == "publication info") {
|
|
rdfUri = prefixDummy + ''place'';
|
|
var pubParts = value.split(" : ");
|
|
value = pubParts[0];
|
|
} else if(field == "personal author") {
|
|
rdfUri = prefixDC + ''creator'';
|
|
value = cleanAuthor(node.nodeValue);
|
|
} else if(field == "added author") {
|
|
rdfUri = prefixDC + ''contributor'';
|
|
value = cleanAuthor(node.nodeValue);
|
|
} else if(field == "corporate author") {
|
|
rdfUri = prefixDC + ''creator'';
|
|
}
|
|
if(rdfUri) {
|
|
var insert = true;
|
|
if(data && data[rdfUri]) {
|
|
for(j in data[rdfUri]) {
|
|
if(data[rdfUri][j] == value) {
|
|
insert = false;
|
|
break;
|
|
}
|
|
}
|
|
} else if(!data[rdfUri]) {
|
|
data[rdfUri] = new Array();
|
|
}
|
|
if(insert) {
|
|
data[rdfUri].push(value);
|
|
model.addStatement(uri, rdfUri, value, true);
|
|
}
|
|
}
|
|
}
|
|
} catch (e) {}
|
|
|
|
}
|
|
');
|
|
|
|
INSERT INTO "scrapers" VALUES(8, NULL, NULL, 20060603002000, 'ProQuest Scraper', 'Simon Kornblith', 'http://proquest\.umi\.com/pqdweb\?(?:.*\&)?did=', '',
|
|
'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
var namespace = doc.documentElement.namespaceURI;
|
|
var nsResolver = namespace ? function(prefix) {
|
|
if (prefix == ''x'') return namespace; else return null;
|
|
} : null;
|
|
|
|
var getNode = function(doc, contextNode, xpath, nsResolver) {
|
|
return doc.evaluate(xpath, contextNode, nsResolver, XPathResult.ANY_TYPE,null).iterateNext();
|
|
}
|
|
|
|
function stringTrimmer(x) {
|
|
var x = x.replace(/^[^\w(]+/, "");
|
|
return x.replace(/[^\w)]+$/, "");
|
|
}
|
|
|
|
function getPageRange(x) {
|
|
var re = /[0-9\-]+/;
|
|
var m = re.exec(x);
|
|
if(m) {
|
|
return m[0];
|
|
}
|
|
}
|
|
|
|
function cleanAuthor(author) {
|
|
author = author.replace(/^[\s\.\,\/\[\]\:]+/, '''');
|
|
author = author.replace(/[\s\,\/\[\]\:\.]+$/, '''');
|
|
// Add period for initials
|
|
if(author.substring(author.length-2, author.length-1) == " ") {
|
|
author += ".";
|
|
}
|
|
var splitNames = author.split('', '');
|
|
if(splitNames.length > 1) {
|
|
author = splitNames[1]+'' ''+splitNames[0];
|
|
}
|
|
return author;
|
|
}
|
|
|
|
var uri = doc.location.href;
|
|
var data = new Object();
|
|
|
|
// Title
|
|
var xpath = ''/html/body/span[@class="textMedium"]/table/tbody/tr/td[@class="headerBlack"]/strong//text()'';
|
|
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
|
|
var title = "";
|
|
for (var i = 0; i < elmts.length; i++) {
|
|
var elmt = elmts[i];
|
|
title += elmt.nodeValue;
|
|
}
|
|
if(title) {
|
|
model.addStatement(uri, prefixDC + ''title'', title, true);
|
|
}
|
|
|
|
// Authors
|
|
var xpath = ''/html/body/span[@class="textMedium"]/table/tbody/tr/td[@class="textMedium"]/a/em'';
|
|
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
|
|
for (var i = 0; i < elmts.length; i++) {
|
|
var elmt = elmts[i];
|
|
|
|
// Dirty hack to fix highlighted words
|
|
var xpath = ''.//text()'';
|
|
var author = "";
|
|
var authorElmts = utilities.gatherElementsOnXPath(doc, elmt, xpath, nsResolver);
|
|
for (var j = 0; j < authorElmts.length; j++) {
|
|
var authorElmt = authorElmts[j];
|
|
author += authorElmt.nodeValue;
|
|
}
|
|
model.addStatement(uri, prefixDC + ''creator'', cleanAuthor(author), true);
|
|
}
|
|
|
|
// Other info
|
|
var xpath = ''/html/body/span[@class="textMedium"]/font/table/tbody/tr'';
|
|
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
|
|
for (var i = 0; i < elmts.length; i++) {
|
|
var elmt = elmts[i];
|
|
var field = stringTrimmer(getNode(doc, elmt, ''./TD[1]/text()[1]'', nsResolver).nodeValue).toLowerCase();
|
|
if(field == "publication title") {
|
|
var publication = getNode(doc, elmt, ''./TD[2]/A[1]/text()[1]'', nsResolver);
|
|
if(publication.nodeValue) {
|
|
model.addStatement(uri, prefixDummy + ''publication'', stringTrimmer(publication.nodeValue), true);
|
|
}
|
|
var place = getNode(doc, elmt, ''./TD[2]/text()[1]'', nsResolver);
|
|
if(place.nodeValue) {
|
|
model.addStatement(uri, prefixDummy + ''place'', stringTrimmer(place.nodeValue), true);
|
|
}
|
|
var date = getNode(doc, elmt, ''./TD[2]/A[2]/text()[1]'', nsResolver);
|
|
if(date.nodeValue) {
|
|
model.addStatement(uri, prefixDC + ''date'', stringTrimmer(date.nodeValue), true);
|
|
}
|
|
var moreInfo = getNode(doc, elmt, ''./TD[2]/text()[2]'', nsResolver);
|
|
if(moreInfo.nodeValue) {
|
|
moreInfo = stringTrimmer(moreInfo.nodeValue);
|
|
var parts = moreInfo.split(";\xA0");
|
|
|
|
var issueRegexp = /^(\w+)\.(?: |\xA0)?(.+)$/
|
|
var issueInfo = parts[0].split(",\xA0");
|
|
for(j in issueInfo) {
|
|
var m = issueRegexp.exec(issueInfo[j]);
|
|
var info = m[1].toLowerCase();
|
|
if(info == "vol") {
|
|
model.addStatement(uri, prefixDummy + ''volume'', stringTrimmer(m[2]), true);
|
|
} else if(info == "iss" || info == "no") {
|
|
model.addStatement(uri, prefixDummy + ''number'', stringTrimmer(m[2]), true);
|
|
}
|
|
}
|
|
if(parts[1] && stringTrimmer(parts[1]).substring(0, 3).toLowerCase() == "pg.") {
|
|
var pages = getPageRange(parts[1]);
|
|
if(pages) {
|
|
model.addStatement(uri, prefixDummy + ''pages'', pages, true);
|
|
}
|
|
}
|
|
}
|
|
} else if(field == "source type") {
|
|
var value = getNode(doc, elmt, ''./TD[2]/text()[1]'', nsResolver);
|
|
if(value.nodeValue) {
|
|
value = stringTrimmer(value.nodeValue).toLowerCase();
|
|
|
|
if(value == "newspaper" || value == "periodical") {
|
|
model.addStatement(uri, prefixRDF + "type", prefixDummy + "journal", false);
|
|
} else {
|
|
model.addStatement(uri, prefixRDF + "type", prefixDummy + "book", false);
|
|
}
|
|
}
|
|
} else if(field == "isbn" || field == "issn" || field == "issn/isbn") {
|
|
var value = getNode(doc, elmt, ''./TD[2]/text()[1]'', nsResolver);
|
|
if(value) {
|
|
var type;
|
|
value = stringTrimmer(value.nodeValue);
|
|
if(value.length == 10 || value.length == 13) {
|
|
type = "ISBN";
|
|
} else if(value.length == 8) {
|
|
type = "ISSN";
|
|
}
|
|
if(type) {
|
|
model.addStatement(uri, prefixDC + "identifier", type+" "+value, false);
|
|
}
|
|
}
|
|
}
|
|
}');
|
|
|
|
INSERT INTO "scrapers" VALUES(9, NULL, NULL, 20060603002000, 'InfoTrac Scraper', 'Simon Kornblith', '^http://infotrac-college\.thomsonlearning\.com/itw/infomark/',
|
|
'if(doc.title.substring(0, 8) == "Article ") {
|
|
return true;
|
|
}
|
|
return false;',
|
|
'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
var namespace = doc.documentElement.namespaceURI;
|
|
var nsResolver = namespace ? function(prefix) {
|
|
if (prefix == ''x'') return namespace; else return null;
|
|
} : null;
|
|
|
|
function cleanAuthor(author) {
|
|
author = author.replace(/^[\s\.\,\/\[\]\:]+/, '''');
|
|
author = author.replace(/[\s\,\/\[\]\:\.]+$/, '''');
|
|
// Add period for initials
|
|
if(author.substring(author.length-2, author.length-1) == " ") {
|
|
author += ".";
|
|
}
|
|
var splitNames = author.split('', '');
|
|
if(splitNames.length > 1) {
|
|
author = splitNames[1]+'' ''+splitNames[0];
|
|
}
|
|
return author;
|
|
}
|
|
|
|
var uri = doc.location.href;
|
|
|
|
var xpath = ''/html/body//comment()'';
|
|
var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver);
|
|
for (var i = 0; i < elmts.length; i++) {
|
|
var elmt = elmts[i];
|
|
var colon = elmt.nodeValue.indexOf(":");
|
|
var field = elmt.nodeValue.substring(1, colon).toLowerCase();
|
|
var value = elmt.nodeValue.substring(colon+1, elmt.nodeValue.length-1);
|
|
if(field == "title") {
|
|
model.addStatement(uri, prefixDC + "title", value, false);
|
|
} else if(field == "journal") {
|
|
model.addStatement(uri, prefixDummy + "publication", value, false);
|
|
} else if(field == "pi") {
|
|
parts = value.split(" ");
|
|
var date = "";
|
|
var isDate = true;
|
|
var rdfUri;
|
|
for(j in parts) {
|
|
firstChar = parts[j].substring(0, 1);
|
|
rdfUri = false;
|
|
|
|
if(firstChar == "v") {
|
|
rdfUri = prefixDummy + "volume";
|
|
} else if(firstChar == "i") {
|
|
rdfUri = prefixDummy + "issue";
|
|
} else if(firstChar == "p") {
|
|
rdfUri = prefixDummy + "pages";
|
|
var pagesRegexp = /p(\w+)\((\w+)\)/;
|
|
var match = pagesRegexp.exec(parts[j]);
|
|
if(match) {
|
|
var finalPage = parseInt(match[1])+parseInt(match[2])
|
|
parts[j] = "p"+match[1]+"-"+finalPage.toString();
|
|
}
|
|
}
|
|
|
|
if(rdfUri) {
|
|
isDate = false;
|
|
if(parts[j] != "pNA") { // not a real page number
|
|
var content = parts[j].substring(1);
|
|
model.addStatement(uri, rdfUri, content, true);
|
|
}
|
|
} else if(isDate) {
|
|
date += " "+parts[j];
|
|
}
|
|
}
|
|
if(date != "") {
|
|
model.addStatement(uri, prefixDC + "date", date.substring(1), false);
|
|
}
|
|
} else if(field == "author") {
|
|
model.addStatement(uri, prefixDC + "creator", cleanAuthor(value), false);
|
|
}
|
|
}
|
|
model.addStatement(uri, prefixRDF + "type", prefixDummy + "journal", false);');
|
|
|
|
INSERT INTO "scrapers" VALUES(10, NULL, NULL, 20060603002000, 'LexisNexis Scraper', 'Simon Kornblith', '^http://web\.lexis-nexis\.com/universe/document', NULL,
|
|
'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
function clearTags(x) {
|
|
x = x.replace(/<br[^>]*>/gi, "\n");
|
|
return x.replace(/<[^>]+>/g, "");
|
|
}
|
|
|
|
var uri = doc.location.href;
|
|
|
|
var citationDataDiv;
|
|
var divs = doc.getElementsByTagName("div");
|
|
for(i in divs) {
|
|
if(divs[i].className == "bodytext") {
|
|
citationDataDiv = divs[i];
|
|
break;
|
|
}
|
|
}
|
|
|
|
centerElements = citationDataDiv.getElementsByTagName("center");
|
|
var elementParts = centerElements[0].innerHTML.split(/<br[^>]*>/gi);
|
|
model.addStatement(uri, prefixDummy + "publication", elementParts[elementParts.length-1], true);
|
|
|
|
var dateRegexp = /<br[^>]*><b>([A-Z][a-z]+)<\/b> ([0-9]+, [0-9]{4})/;
|
|
var m = dateRegexp.exec(centerElements[centerElements.length-1].innerHTML);
|
|
if(m) {
|
|
model.addStatement(uri, prefixDC + "date", m[1]+" "+m[2], true);
|
|
} else {
|
|
var elementParts = centerElements[centerElements.length-1].innerHTML.split(/<br[^>]*>/gi);
|
|
model.addStatement(uri, prefixDC + "date", elementParts[1], true);
|
|
}
|
|
|
|
var cutIndex = citationDataDiv.innerHTML.indexOf("<b>BODY:</b>");
|
|
if(cutIndex < 0) {
|
|
cutIndex = citationDataDiv.innerHTML.indexOf("<b>TEXT:</b>");
|
|
}
|
|
if(cutIndex > 0) {
|
|
citationData = citationDataDiv.innerHTML.substring(0, cutIndex);
|
|
} else {
|
|
citationData = citationDataDiv.innerHTML;
|
|
}
|
|
|
|
citationData = clearTags(citationData);
|
|
|
|
var headlineRegexp = /\n(?:HEADLINE|TITLE|ARTICLE): ([^\n]+)\n/;
|
|
var m = headlineRegexp.exec(citationData);
|
|
if(m) {
|
|
model.addStatement(uri, prefixDC + "title", clearTags(m[1]), true);
|
|
}
|
|
|
|
var bylineRegexp = /\nBYLINE: *(\w[\w\- ]+)/;
|
|
var m = bylineRegexp.exec(citationData);
|
|
if(m) {
|
|
utilities.debugPrint(m[1].substring(0, 3).toLowerCase());
|
|
if(m[1].substring(0, 3).toLowerCase() == "by ") {
|
|
m[1] = m[1].substring(3);
|
|
}
|
|
model.addStatement(uri, prefixDC + "creator", m[1], true);
|
|
}
|
|
|
|
var authorRegexp = /\n(?:AUTHOR|NAME): ([^\n]+)\n/;
|
|
var m = authorRegexp.exec(citationData);
|
|
if(m) {
|
|
var authors = m[1].split(/, (?:and )?/);
|
|
for(i in authors) {
|
|
model.addStatement(uri, prefixDC + "creator", authors[i].replace(" *", ""), true);
|
|
}
|
|
}
|
|
|
|
model.addStatement(uri, prefixRDF + "type", prefixDummy + "journal", false);
|
|
|
|
utilities.debugPrint(citationData);');
|
|
|
|
INSERT INTO "scrapers" VALUES(11, NULL, NULL, 20060603002000, 'Aleph Scraper', 'Simon Kornblith', 'func=full-set-set.*\&format=999', NULL,
|
|
'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
var uri = doc.location.href;
|
|
|
|
var newUri = uri.replace("&format=999", "&format=001");
|
|
utilities.debugPrint(newUri);
|
|
|
|
function stringTrimmer(x) {
|
|
var x = x.replace(/^[^\w(]+/, "");
|
|
return x.replace(/[^\w)]+$/, "");
|
|
}
|
|
|
|
utilities.loadDocument(newUri, browser, function(newBrowser) {
|
|
newDoc = newBrowser.contentDocument;
|
|
|
|
var namespace = newDoc.documentElement.namespaceURI;
|
|
var nsResolver = namespace ? function(prefix) {
|
|
if (prefix == ''x'') return namespace; else return null;
|
|
} : null;
|
|
|
|
var getNode = function(doc, contextNode, xpath, nsResolver) {
|
|
return doc.evaluate(xpath, contextNode, nsResolver, XPathResult.ANY_TYPE,null).iterateNext();
|
|
}
|
|
|
|
var xpath = ''/html/body/table/tbody/tr[td[1][@class="td1"][@id="bold"]][td[2][@class="td1"]]'';
|
|
var elmts = utilities.gatherElementsOnXPath(newDoc, newDoc, xpath, nsResolver);
|
|
var record = new MARC_Record();
|
|
for(var i=0; i<elmts.length; i++) {
|
|
var elmt = elmts[i];
|
|
var field = stringTrimmer(getNode(doc, elmt, ''./TD[1]/text()[1]'', nsResolver).nodeValue);
|
|
var value = getNode(doc, elmt, ''./TD[2]/text()[1]'', nsResolver).nodeValue;
|
|
var value = value.replace(/\|([a-z]) /g, record.subfield_delimiter+"$1");
|
|
|
|
if(field != "FMT" && field != "LDR") {
|
|
var ind1 = "";
|
|
var ind2 = "";
|
|
var code = field.substring(0, 3);
|
|
if(field.length > 3) {
|
|
var ind1 = field.charAt(3);
|
|
if(field.length > 4) {
|
|
var ind2 = field.charAt(4);
|
|
}
|
|
}
|
|
record.add_field(code, ind1, ind2, value);
|
|
}
|
|
}
|
|
|
|
model = utilities.importMARCRecord(record, uri, model);
|
|
done();
|
|
}, function() {})
|
|
|
|
wait();');
|
|
|
|
INSERT INTO "scrapers" VALUES(12, NULL, NULL, 20060603002000, 'Dynix Scraper', 'Simon Kornblith', 'ipac\.jsp\?.*uri=full=[0-9]', NULL,
|
|
'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
var uri = doc.location.href;
|
|
var newUri = uri+''&fullmarc=true'';
|
|
utilities.debugPrint(newUri);
|
|
|
|
function stringTrimmer(x) {
|
|
var x = x.replace(/^[^\w(]+/, "");
|
|
return x.replace(/[^\w)]+$/, "");
|
|
}
|
|
|
|
var getNode = function(doc, contextNode, xpath, nsResolver) {
|
|
return doc.evaluate(xpath, contextNode, nsResolver, XPathResult.ANY_TYPE,null).iterateNext();
|
|
}
|
|
|
|
var getNodeString = function(doc, contextNode, xpath, nsResolver) {
|
|
var elmts = utilities.gatherElementsOnXPath(doc, contextNode, xpath, nsResolver);
|
|
var returnVar = "";
|
|
for(var i=0; i<elmts.length; i++) {
|
|
returnVar += elmts[i].nodeValue;
|
|
}
|
|
return returnVar;
|
|
}
|
|
|
|
utilities.loadDocument(newUri, browser, function(newBrowser) {
|
|
newDoc = newBrowser.contentDocument;
|
|
|
|
var namespace = newDoc.documentElement.namespaceURI;
|
|
var nsResolver = namespace ? function(prefix) {
|
|
if (prefix == ''x'') return namespace; else return null;
|
|
} : null;
|
|
|
|
var xpath = ''//form/table[@class="tableBackground"]/tbody/tr/td/table[@class="tableBackground"]/tbody/tr[td[1]/a[@class="normalBlackFont1"]]'';
|
|
var elmts = utilities.gatherElementsOnXPath(newDoc, newDoc, xpath, nsResolver);
|
|
var record = new MARC_Record();
|
|
for(var i=0; i<elmts.length; i++) {
|
|
var elmt = elmts[i];
|
|
var field = stringTrimmer(getNode(doc, elmt, ''./TD[1]/A[1]/text()[1]'', nsResolver).nodeValue);
|
|
var value = getNodeString(doc, elmt, ''./TD[2]/TABLE[1]/TBODY[1]/TR[1]/TD[1]/A[1]//text()'', nsResolver);
|
|
value = value.replace(/\$([a-z]) /g, record.subfield_delimiter+"$1");
|
|
|
|
if(field != "FMT" && field != "LDR") {
|
|
var ind1 = "";
|
|
var ind2 = "";
|
|
var valRegexp = /^([0-9])([0-9])? (.*)$/;
|
|
var m = valRegexp.exec(value);
|
|
if(m) {
|
|
ind1 = m[1];
|
|
if(ind2) {
|
|
ind2 = m[2]
|
|
}
|
|
value = m[3];
|
|
}
|
|
record.add_field(field, ind1, ind2, value);
|
|
}
|
|
}
|
|
|
|
model = utilities.importMARCRecord(record, uri, model);
|
|
done();
|
|
}, function() {})
|
|
|
|
wait();');
|
|
|
|
INSERT INTO "scrapers" VALUES(13, NULL, NULL, 20060603002000, 'VTLS Scraper', 'Simon Kornblith', 'chameleon\?.*function=(?:CARDSCR|INITREQ)', NULL,
|
|
'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
|
|
var prefixDC = ''http://purl.org/dc/elements/1.1/'';
|
|
var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
|
|
var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
|
|
|
|
var uri = doc.location.href;
|
|
var newUri = uri.replace(/function=[A-Z]{7}/, "function=MARCSCR");
|
|
utilities.debugPrint(newUri);
|
|
|
|
var getNode = function(doc, contextNode, xpath, nsResolver) {
|
|
return doc.evaluate(xpath, contextNode, nsResolver, XPathResult.ANY_TYPE,null).iterateNext();
|
|
}
|
|
|
|
utilities.loadDocument(newUri, browser, function(newBrowser) {
|
|
newDoc = newBrowser.contentDocument;
|
|
|
|
var namespace = newDoc.documentElement.namespaceURI;
|
|
var nsResolver = namespace ? function(prefix) {
|
|
if (prefix == ''x'') return namespace; else return null;
|
|
} : null;
|
|
|
|
var xpath = ''//table[@class="outertable"]/tbody/tr[td[4]]'';
|
|
var elmts = utilities.gatherElementsOnXPath(newDoc, newDoc, xpath, nsResolver);
|
|
var record = new MARC_Record();
|
|
for(var i=0; i<elmts.length; i++) {
|
|
var elmt = elmts[i];
|
|
var field = getNode(doc, elmt, ''./TD[1]/text()[1]'', nsResolver).nodeValue;
|
|
var ind1 = getNode(doc, elmt, ''./TD[2]/text()[1]'', nsResolver).nodeValue;
|
|
var ind2 = getNode(doc, elmt, ''./TD[3]/text()[1]'', nsResolver).nodeValue;
|
|
var value = getNode(doc, elmt, ''./TD[4]/text()[1]'', nsResolver).nodeValue;
|
|
value = value.replace(/\\([a-z]) /g, record.subfield_delimiter+"$1");
|
|
|
|
record.add_field(field, ind1, ind2, value);
|
|
}
|
|
|
|
model = utilities.importMARCRecord(record, uri, model);
|
|
done();
|
|
}, function() {})
|
|
|
|
wait();'); |