From fd2052e63c289d08a4e018169460c89cace90710 Mon Sep 17 00:00:00 2001
From: Simon Kornblith <simon@simonster.com>
Date: Sat, 24 Jun 2006 17:33:35 +0000
Subject: [PATCH] Search results scraping for PubMed and Google Books. This
 marks the end of what I can do with respect to #15 until I'm at home or CHNM,
 where I'll have access to the gated collections.

---
 scrapers.sql | 96 +++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 72 insertions(+), 24 deletions(-)

diff --git a/scrapers.sql b/scrapers.sql
index 7a4614075..3bef2614e 100644
--- a/scrapers.sql
+++ b/scrapers.sql
@@ -1,7 +1,7 @@
--- 15
+-- 16
 
 -- Set the following timestamp to the most recent scraper update date
-REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-06-24 11:22:00'));
+REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-06-24 13:31:00'));
 
 REPLACE INTO "scrapers" VALUES('96b9f483-c44d-5784-cdad-ce21b984fe01', '2006-06-22 22:58:00', 'Amazon.com Scraper', 'Simon Kornblith', '^http://www\.amazon\.com/(?:gp/(?:product|search)/|exec/obidos/search-handle-url/)', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
 var prefixDC = ''http://purl.org/dc/elements/1.1/'';
@@ -1747,23 +1747,52 @@ for(i in elmts) {
 
 model.addStatement(uri, prefixRDF + "type", prefixDummy + "journal", false);');
 
-REPLACE INTO "scrapers" VALUES('fcf41bed-0cbc-3704-85c7-8062a0068a7a', '2006-06-18 11:19:00', 'PubMed Scraper', 'Simon Kornblith', '^http://www\.ncbi\.nlm\.nih\.gov/entrez/query\.fcgi\?(?:.*db=PubMed.*list_uids=[0-9]|.*list_uids=[0-9].*db=PubMed)', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
+REPLACE INTO "scrapers" VALUES('fcf41bed-0cbc-3704-85c7-8062a0068a7a', '2006-06-24 13:17:00', 'PubMed Scraper', 'Simon Kornblith', '^http://www\.ncbi\.nlm\.nih\.gov/entrez/query\.fcgi\?(?:.*db=PubMed.*list_uids=[0-9]|.*list_uids=[0-9].*db=PubMed|.*db=PubMed.*CMD=search|.*CMD=search.*db=PubMed)', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
 var prefixDC = ''http://purl.org/dc/elements/1.1/'';
 var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
 var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
 
-function mapRDF(text, rdfUri) {
+function mapRDF(uri, text, rdfUri) {
 	if(text != "") {
 		model.addStatement(uri, rdfUri, text, true);
 	}
 }
 
 var uri = doc.location.href;
-var newUri = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=PubMed&retmode=xml&rettype=citation&id=";
+var ids = new Array();
 var idRegexp = /[\?\&]list_uids=([0-9\,]+)/;
-var m = idRegexp.exec(uri);
-newUri += m[1];
 
+var m = idRegexp.exec(uri);
+if(m) {
+	ids.push(m[1]);
+} else {
+	var namespace = doc.documentElement.namespaceURI;
+	var nsResolver = namespace ? function(prefix) {
+		if (prefix == ''x'') return namespace; else return null;
+	} : null;
+	
+	var items = new Array();
+	var tableRows = utilities.gatherElementsOnXPath(doc, doc, ''//div[@class="ResultSet"]/table/tbody'', nsResolver);
+	// Go through table rows
+	for(var i=0; i<tableRows.length; i++) {
+		var link = utilities.getNode(doc, tableRows[i], ''.//a'', nsResolver);
+		var article = utilities.getNode(doc, tableRows[i], ''./tr[2]/td[2]/text()[1]'', nsResolver);
+		items[link.href] = article.nodeValue;
+	}
+	
+	items = utilities.selectItems(items);
+	
+	if(!items) {
+		return true;
+	}
+	
+	for(i in items) {
+		var m = idRegexp.exec(i);
+		ids.push(m[1]);
+	}
+}
+
+var newUri = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=PubMed&retmode=xml&rettype=citation&id="+ids.join(",");
 utilities.HTTPUtilities.doGet(newUri, null, function(text) {
 	// Remove xml parse instruction and doctype
 	text = text.replace(/<!DOCTYPE[^>]*>/, "").replace(/<\?xml[^>]*\?>/, "");
@@ -1773,6 +1802,7 @@ utilities.HTTPUtilities.doGet(newUri, null, function(text) {
 	for(var i=0; i<xml.PubmedArticle.length(); i++) {
 		var citation = xml.PubmedArticle[i].MedlineCitation;
 		
+		var uri = "http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=pubmed&cmd=Retrieve&list_uids="+citation.PMID.text();
 		if(citation.PMID.length()) {
 			model.addStatement(uri, prefixDC + "identifier", "PMID "+citation.PMID.text(), true);
 		}
@@ -1793,14 +1823,14 @@ utilities.HTTPUtilities.doGet(newUri, null, function(text) {
 			}
 			
 			if(article.Journal.Title.length()) {
-				model.addStatement(uri, prefixDummy + "publication", utilities.superCleanString(article.Journal.Title.text()), true);
+				model.addStatement(uri, prefixDummy + "publication", utilities.superCleanString(article.Journal.Title.text().toString()), true);
 			} else if(citation.MedlineJournalInfo.MedlineTA.length()) {
-				model.addStatement(uri, prefixDummy + "publication", utilities.superCleanString(citation.MedlineJournalInfo.MedlineTA.text()), true);
+				model.addStatement(uri, prefixDummy + "publication", utilities.superCleanString(citation.MedlineJournalInfo.MedlineTA.text().toString()), true);
 			}
 			
 			if(article.Journal.JournalIssue.length()) {
-				mapRDF(article.Journal.JournalIssue.Volume.text(), prefixDummy + "volume");
-				mapRDF(article.Journal.JournalIssue.Issue.text(), prefixDummy + "number");
+				mapRDF(uri, article.Journal.JournalIssue.Volume.text(), prefixDummy + "volume");
+				mapRDF(uri, article.Journal.JournalIssue.Issue.text(), prefixDummy + "number");
 				if(article.Journal.JournalIssue.PubDate.length()) {
 					model.addStatement(uri, prefixDC + "date", article.Journal.JournalIssue.PubDate.Day.text()+" "+article.Journal.JournalIssue.PubDate.Month.text()+" "+article.Journal.JournalIssue.PubDate.Year.text(), true);
 				}
@@ -1810,22 +1840,22 @@ utilities.HTTPUtilities.doGet(newUri, null, function(text) {
 		if(article.AuthorList.length() && article.AuthorList.Author.length()) {
 			var authors = article.AuthorList.Author;
 			for(var j=0; j<authors.length(); j++) {
-				var lastName = authors[j].LastName.text();
-				var firstName = authors[j].FirstName.text();
+				var lastName = authors[j].LastName.text().toString();
+				var firstName = authors[j].FirstName.text().toString();
 				if(firstName == "") {
-					var firstName = authors[j].ForeName.text();
+					var firstName = authors[j].ForeName.text().toString();
 				}
 				if(firstName && lastName) {
 					model.addStatement(uri, prefixDC + "creator", firstName + " " + lastName);
 				}
 			}
 		}
+		model.addStatement(uri, prefixRDF + "type", prefixDummy + "journal", false);
 	}
 
 	done();
 })
 
-model.addStatement(uri, prefixRDF + "type", prefixDummy + "journal", false);
 wait();');
 
 REPLACE INTO "scrapers" VALUES('951c027d-74ac-47d4-a107-9c3069ab7b48', '2006-06-20 10:52:00', 'Scraper for Dublin Core expressed as HTML META elements', 'Simon Kornblith', NULL,
@@ -1862,21 +1892,41 @@ for(var i=0; i<metaTags.length; i++) {
 	}
 }');
 
-REPLACE INTO "scrapers" VALUES('3e684d82-73a3-9a34-095f-19b112d88bbf', '2006-06-21 10:28:00', 'Google Books Scraper', 'Simon Kornblith', 'http://books\.google\.com/books\?vid=.*\&id=.*', NULL,
+REPLACE INTO "scrapers" VALUES('3e684d82-73a3-9a34-095f-19b112d88bbf', '2006-06-24 13:31:00', 'Google Books Scraper', 'Simon Kornblith', 'http://books\.google\.com/books\?(.*vid=.*\&id=.*|.*q=.*)', NULL,
 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#'';
 var prefixDC = ''http://purl.org/dc/elements/1.1/'';
 var prefixDCMI = ''http://purl.org/dc/dcmitype/'';
 var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/'';
 
 var uri = doc.location.href;
+var newUris = new Array();
+
 var re = new RegExp(''http://books\\.google\\.com/books\\?vid=([^&]+).*\\&id=([^&]+)'', ''i'');
-var urlParts = re.exec(uri);
-var newUri = ''http://books.google.com/books?vid=''+urlParts[1]+''&id=''+urlParts[2];
+var m = re.exec(uri);
+if(m) {
+	newUris.push(''http://books.google.com/books?vid=''+m[1]+''&id=''+m[2]);
+} else {
+	var items = utilities.getItemArray(doc, doc, ''http://books\\.google\\.com/books\\?vid=([^&]+).*\\&id=([^&]+)'', ''^(?:All matching pages|About this Book)'');
 
-utilities.debugPrint(newUri);
+	// Drop " - Page" thing
+	for(i in items) {
+		items[i] = items[i].replace(/- Page [0-9]+\s*$/, "");
+	}
+	items = utilities.selectItems(items);
+	
+	if(!items) {
+		return true;
+	}
+	
+	for(i in items) {
+		var m = re.exec(i);
+		newUris.push(''http://books.google.com/books?vid=''+m[1]+''&id=''+m[2]);
+	}
+}
 
-utilities.loadDocument(newUri, browser, function(newBrowser) {
-	newDoc = newBrowser.contentDocument;
+utilities.processDocuments(browser, null, newUris, function(newBrowser) {
+	var newDoc = newBrowser.contentDocument;
+	var uri = newDoc.location.href;
 	
 	var namespace = newDoc.documentElement.namespaceURI;
 	var nsResolver = namespace ? function(prefix) {
@@ -1923,8 +1973,6 @@ utilities.loadDocument(newUri, browser, function(newBrowser) {
 		}
 	}
 	model.addStatement(uri, prefixRDF + "type", prefixDummy + "book", false);
-	
-	done();
-}, function() {});
+}, function() { done(); }, function() {});
 
 wait();');
\ No newline at end of file