diff --git a/scrapers.sql b/scrapers.sql index 15cd30e1a..01c51de4b 100644 --- a/scrapers.sql +++ b/scrapers.sql @@ -1,7 +1,7 @@ --- 18 +-- 19 -- Set the following timestamp to the most recent scraper update date -REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-06-25 12:11:00')); +REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-06-25 14:16:00')); REPLACE INTO "scrapers" VALUES('96b9f483-c44d-5784-cdad-ce21b984fe01', '2006-06-22 22:58:00', 'Amazon.com Scraper', 'Simon Kornblith', '^http://www\.amazon\.com/(?:gp/(?:product|search)/|exec/obidos/search-handle-url/)', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#''; var prefixDC = ''http://purl.org/dc/elements/1.1/''; @@ -384,12 +384,17 @@ utilities.HTTPUtilities.doGet(newUri+''?''+postString, null, function(text) { }) wait();'); -REPLACE INTO "scrapers" VALUES('d921155f-0186-1684-615c-ca57682ced9b', '2006-06-18 11:02:00', 'JSTOR Scraper', 'Simon Kornblith', '^http://www\.jstor\.org/(?:view|browse)', +REPLACE INTO "scrapers" VALUES('d921155f-0186-1684-615c-ca57682ced9b', '2006-06-25 14:16:00', 'JSTOR Scraper', 'Simon Kornblith', '^http://www\.jstor\.org/(?:view|browse|search/)', 'var namespace = doc.documentElement.namespaceURI; var nsResolver = namespace ? function(prefix) { if (prefix == ''x'') return namespace; else return null; } : null; +// See if this is a seach results page +if(doc.title == "JSTOR: Search Results") { + return true; +} + // If this is a view page, find the link to the citation var xpath = ''/html/body/div[@class="indent"]/center/font/p/a[@class="nav"]''; var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver); @@ -411,52 +416,130 @@ var nsResolver = namespace ? function(prefix) { } : null; var uri = doc.location.href; +var saveCitations = new Array(); -// If this is a view page, find the link to the citation -var xpath = ''/html/body/div[@class="indent"]/center/font/p/a[@class="nav"]''; -var elmts = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver); -if(!elmts.length) { - var xpath = ''/html/body/div[@class="indent"]/center/p/font/a[@class="nav"]''; +if(doc.title == "JSTOR: Search Results") { + var availableItems = new Object(); + + // Require link to match this + var tagRegexp = new RegExp(); + tagRegexp.compile(''citationAction=''); + + var tableRows = utilities.gatherElementsOnXPath(doc, doc, ''/html/body/div[@class="indent"]/table/tbody/tr[td/span[@class="printDownloadSaveLinks"]]'', nsResolver); + // Go through table rows + for(var i=0; i/; + var stableURL, ISSN; for(i in lines) { - if(haveStarted) { - var fieldCode = lines[i].substring(0, 2); - var fieldContent = utilities.cleanString(lines[i].substring(5)); - - if(lines[i].substring(2, 5) != " : ") { - break; + if(lines[i].substring(0,3) == "<1>") { + haveStarted = true; + } else if(newItemRe.test(lines[i])) { + utilities.debugPrint("new item!"); + if(!stableURL) { + if(ISSN) { + stableURL = "http://www.jstor.org/browse/"+ISSN; + } else { // Just make sure it''s unique + stableURL = k; + k++; + } } + model.addStatement(stableURL, prefixRDF + "type", prefixDummy + "journalArticle", false); + for(i in data) { + if(data[i].length) { + for(j in data[i]) { + model.addStatement(stableURL, i, data[i][j]); + } + } + } + var data = newDataObject(); + delete ISSN; + delete stableURL; + } else if(lines[i].substring(2, 5) == " : " && haveStarted) { + var fieldCode = lines[i].substring(0, 2); + var fieldContent = utilities.cleanString(lines[i].substring(5)) if(fieldCode == "TI") { data[prefixDC + "title"].push(fieldContent); @@ -493,16 +576,22 @@ utilities.HTTPUtilities.doPost(''http://www.jstor.org/browse'', postData, null, stableURL = fieldContent; } else if(fieldCode == "IN") { data[prefixDC + "identifier"].push("ISSN "+fieldContent); + ISSN = fieldContent; } else if(fieldCode == "PB") { data[prefixDC + "publisher"].push(fieldContent); } } - if(lines[i].substring(0,3) == "<1>") { - haveStarted = true; - } } // Loop through again so that we can add with the stableURL + if(!stableURL) { + if(ISSN) { + stableURL = "http://www.jstor.org/browse/"+ISSN; + } else { // Just make sure it''s unique + stableURL = k; + k++; + } + } model.addStatement(stableURL, prefixRDF + "type", prefixDummy + "journalArticle", false); for(i in data) { if(data[i].length) { @@ -513,8 +602,8 @@ utilities.HTTPUtilities.doPost(''http://www.jstor.org/browse'', postData, null, } done(); - }) - }) + }); + }, function() {}); }); wait();');