From 5e73dcdd2e4f6b920fe020a39fd99a9590af792b Mon Sep 17 00:00:00 2001 From: Simon Kornblith Date: Sun, 25 Jun 2006 16:13:47 +0000 Subject: [PATCH] - Search results scraping for WorldCat. - Make scraperJavaScript run on reload again, because it makes debugging easier - There's not actually a memory leak in the proxyMonitor code. --- .../content/scholar/ingester/browser.js | 4 +- .../content/scholar/xpcom/ingester.js | 1 - scrapers.sql | 115 +++++++++++++----- 3 files changed, 85 insertions(+), 35 deletions(-) diff --git a/chrome/chromeFiles/content/scholar/ingester/browser.js b/chrome/chromeFiles/content/scholar/ingester/browser.js index 647d0ef5a..0475a11fc 100644 --- a/chrome/chromeFiles/content/scholar/ingester/browser.js +++ b/chrome/chromeFiles/content/scholar/ingester/browser.js @@ -196,11 +196,11 @@ Scholar_Ingester_Interface._setDocument = function(browser) { } // Only re-load the scraper if it's a new document - if(Scholar_Ingester_Interface.browserUris[key] != browser.contentDocument.location.href) { + //if(Scholar_Ingester_Interface.browserUris[key] != browser.contentDocument.location.href) { Scholar_Ingester_Interface.browserUris[key] = browser.contentDocument.location.href; Scholar_Ingester_Interface.browserDocuments[key] = new Scholar.Ingester.Document(browser, window); Scholar_Ingester_Interface.browserDocuments[key].retrieveScraper(); - } + //} } /* diff --git a/chrome/chromeFiles/content/scholar/xpcom/ingester.js b/chrome/chromeFiles/content/scholar/xpcom/ingester.js index 33744adbd..370f1b774 100644 --- a/chrome/chromeFiles/content/scholar/xpcom/ingester.js +++ b/chrome/chromeFiles/content/scholar/xpcom/ingester.js @@ -59,7 +59,6 @@ Scholar.Ingester.ProxyMonitor.proxyToProper = function(url) { url = unescape(url); } - // FIXME - potential memory leak Scholar.Ingester.ProxyMonitor._now = true; Scholar.Ingester.ProxyMonitor._url = url; Scholar.Ingester.ProxyMonitor._host = host; diff --git a/scrapers.sql b/scrapers.sql index e072d5b03..15cd30e1a 100644 --- a/scrapers.sql +++ b/scrapers.sql @@ -1,7 +1,7 @@ --- 17 +-- 18 -- Set the following timestamp to the most recent scraper update date -REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-06-25 00:56:00')); +REPLACE INTO "version" VALUES ('repository', STRFTIME('%s', '2006-06-25 12:11:00')); REPLACE INTO "scrapers" VALUES('96b9f483-c44d-5784-cdad-ce21b984fe01', '2006-06-22 22:58:00', 'Amazon.com Scraper', 'Simon Kornblith', '^http://www\.amazon\.com/(?:gp/(?:product|search)/|exec/obidos/search-handle-url/)', NULL, 'var prefixRDF = ''http://www.w3.org/1999/02/22-rdf-syntax-ns#''; var prefixDC = ''http://purl.org/dc/elements/1.1/''; @@ -71,7 +71,7 @@ function scrape(doc) { model.addStatement(uri, prefixRDF + "type", prefixDummy + "book", false); } -var searchRe = new RegExp(''http://www\.amazon\.com/(gp/search/|exec/obidos/search-handle-url/)''); +var searchRe = new RegExp(''^http://www\.amazon\.com/(gp/search/|exec/obidos/search-handle-url/)''); var m = searchRe.exec(doc.location.href) if(m) { // Why can''t amazon use the same stylesheets @@ -83,7 +83,7 @@ if(m) { } var searchresults = utilities.gatherElementsOnXPath(doc, doc, xpath, nsResolver); - var items = utilities.getItemArray(doc, searchresults, ''http://www\.amazon\.com/(gp/product/|exec/obidos/tg/detail/)'', ''^(Buy new|Hardcover|Paperback|Digital)$''); + var items = utilities.getItemArray(doc, searchresults, ''^http://www\.amazon\.com/(gp/product/|exec/obidos/tg/detail/)'', ''^(Buy new|Hardcover|Paperback|Digital)$''); items = utilities.selectItems(items); if(!items) { @@ -103,8 +103,8 @@ if(m) { scrape(doc); }'); -REPLACE INTO "scrapers" VALUES('838d8849-4ffb-9f44-3d0d-aa8a0a079afe', '2006-06-18 11:02:00', 'WorldCat Scraper', 'Simon Kornblith', '^http://newfirstsearch\.oclc\.org/WebZ/', -'if(doc.title == ''FirstSearch: WorldCat Detailed Record'') { +REPLACE INTO "scrapers" VALUES('838d8849-4ffb-9f44-3d0d-aa8a0a079afe', '2006-06-25 12:11:00', 'WorldCat Scraper', 'Simon Kornblith', '^http://(?:new)?firstsearch\.oclc\.org/WebZ/', +'if(doc.title == ''FirstSearch: WorldCat Detailed Record'' || doc.title == ''FirstSearch: WorldCat List of Records'') { return true; } return false;', @@ -116,36 +116,88 @@ var prefixDummy = ''http://chnm.gmu.edu/firefox-scholar/''; var sessionRegexp = /(?:\?|\:)sessionid=([^?:]+)(?:\?|\:|$)/; var numberRegexp = /(?:\?|\:)recno=([^?:]+)(?:\?|\:|$)/; var resultsetRegexp = /(?:\?|\:)resultset=([^?:]+)(?:\?|\:|$)/; -var lineRegexp = /^([\w() ]+): *(.*)$/; -var publisherRegexp = /^(.*), (.*?),?$/; +var hostRegexp = new RegExp("http://([^/]+)/"); var uri = doc.location.href; - + var sMatch = sessionRegexp.exec(uri); var sessionid = sMatch[1]; -var nMatch = numberRegexp.exec(uri); -if(nMatch) { - var number = nMatch[1]; +var hMatch = hostRegexp.exec(uri); +var host = hMatch[1]; + +var newUri, exportselect; + +if(doc.title == ''FirstSearch: WorldCat Detailed Record'') { + var publisherRegexp = /^(.*), (.*?),?$/; + + var nMatch = numberRegexp.exec(uri); + if(nMatch) { + var number = nMatch[1]; + } else { + number = 1; + } + + var rMatch = resultsetRegexp.exec(uri); + if(rMatch) { + var resultset = rMatch[1]; + } else { + // It''s in an XPCNativeWrapper, so we have to do this black magic + resultset = doc.forms.namedItem(''main'').elements.namedItem(''resultset'').value; + } + + exportselect = ''record''; + newUri = ''http://''+host+''/WebZ/DirectExport?numrecs=10:smartpage=directexport:entityexportnumrecs=10:entityexportresultset='' + resultset + '':entityexportrecno='' + number + '':sessionid='' + sessionid + '':entitypagenum=35:0''; + + var uris = new Array(newUri); } else { - number = 1; + var items = utilities.getItemArray(doc, doc, ''/WebZ/FSFETCH\\?fetchtype=fullrecord'', ''^(See more details for locating this item|Detailed Record)$''); + items = utilities.selectItems(items); + + if(!items) { + return true; + } + + // Set BookMark cookie + for(i in items) { // Hack to get first item + var myCookie = sessionid+":"; + var rMatch = resultsetRegexp.exec(i); + var resultset = rMatch[1]; + break; + } + var uris = new Array(); + for(i in items) { + var nMatch = numberRegexp.exec(i); + myCookie += resultset+"_"+nMatch[1]+","; + uris.push(i); + } + myCookie = myCookie.substr(0, myCookie.length-1); + doc.cookie = "BookMark="+myCookie; + + exportselect = ''marked''; + newUri = ''http://''+host+''/WebZ/DirectExport?numrecs=10:smartpage=directexport:entityexportnumrecs=10:entityexportresultset='' + resultset + '':entityexportrecno=1:sessionid='' + sessionid + '':entitypagenum=29:0''; } -var rMatch = resultsetRegexp.exec(uri); -if(rMatch) { - var resultset = rMatch[1]; -} else { - // It''s in an XPCNativeWrapper, so we have to do this black magic - resultset = doc.forms.namedItem(''main'').elements.namedItem(''resultset'').value; -} - -var newUri = ''http://newfirstsearch.oclc.org/WebZ/DirectExport?numrecs=10:smartpage=directexport:entityexportnumrecs=10:entityexportresultset='' + resultset + '':entityexportrecno='' + number + '':sessionid='' + sessionid + '':entitypagenum=35:0''; - -utilities.HTTPUtilities.doPost(newUri, ''exportselect=record&exporttype=plaintext'', null, function(text) { +utilities.HTTPUtilities.doPost(newUri, ''exportselect=''+exportselect+''&exporttype=plaintext'', null, function(text) { + var lineRegexp = new RegExp(); + lineRegexp.compile("^([\\w() ]+): *(.*)$"); + + var k = 0; + var uri = uris[k]; + model.addStatement(uri, prefixRDF + "type", prefixDummy + "book", false); + var lines = text.split(''\n''); for(var i=0;i