From 58150885866c1afba9e3c5519306228f945b543b Mon Sep 17 00:00:00 2001 From: Martynas Bagdonas Date: Thu, 18 Jan 2018 12:23:27 +0200 Subject: [PATCH 1/4] Fix PDF tools usage in tests --- chrome/content/zotero/xpcom/fulltext.js | 42 ++++++++++++++++--------- test/content/runtests.js | 28 +++++++++++++++++ test/runtests.sh | 9 ++++++ 3 files changed, 64 insertions(+), 15 deletions(-) diff --git a/chrome/content/zotero/xpcom/fulltext.js b/chrome/content/zotero/xpcom/fulltext.js index 45efcc6db..9c7e65688 100644 --- a/chrome/content/zotero/xpcom/fulltext.js +++ b/chrome/content/zotero/xpcom/fulltext.js @@ -26,8 +26,6 @@ Zotero.Fulltext = Zotero.FullText = new function(){ this.isCachedMIMEType = isCachedMIMEType; - this.__defineGetter__("pdfConverterName", function() { return 'pdftotext'; }); - this.__defineGetter__("pdfInfoName", function() { return 'pdfinfo'; }); this.__defineGetter__("pdfConverterCacheFile", function () { return '.zotero-ft-cache'; }); this.__defineGetter__("pdfInfoCacheFile", function () { return '.zotero-ft-info'; }); @@ -54,11 +52,9 @@ Zotero.Fulltext = Zotero.FullText = new function(){ const kWbClassHWKatakanaLetter = 6; const kWbClassThaiLetter = 7; - var _pdfConverterFileName = null; var _pdfConverter = null; // nsIFile to executable - var _pdfInfoFileName = null; var _pdfInfo = null; // nsIFile to executable - var _popplerDatadir = null; + var _pdfData = null; var _idleObserverIsRegistered = false; var _idleObserverDelay = 30; @@ -74,19 +70,19 @@ Zotero.Fulltext = Zotero.FullText = new function(){ this.decoder = Components.classes["@mozilla.org/intl/utf8converterservice;1"]. getService(Components.interfaces.nsIUTF8ConverterService); - _pdfConverterFileName = this.pdfConverterName; - _pdfInfoFileName = this.pdfInfoName; + let pdfConverterFileName = "pdftotext"; + let pdfInfoFileName = "pdfinfo"; if (Zotero.isWin) { - _pdfConverterFileName += '.exe'; - _pdfInfoFileName += '.exe'; + pdfConverterFileName += '.exe'; + pdfInfoFileName += '.exe'; } let dir = FileUtils.getFile('AChrom', []).parent; - _popplerDatadir = dir.clone(); - _popplerDatadir.append('poppler-data'); - _popplerDatadir = _popplerDatadir.path; + _pdfData = dir.clone(); + _pdfData.append('poppler-data'); + _pdfData = _pdfData.path; _pdfConverter = dir.clone(); _pdfInfo = dir.clone(); @@ -99,8 +95,8 @@ Zotero.Fulltext = Zotero.FullText = new function(){ _pdfInfo.append('MacOS'); } - _pdfConverter.append(_pdfConverterFileName); - _pdfInfo.append(_pdfInfoFileName); + _pdfConverter.append(pdfConverterFileName); + _pdfInfo.append(pdfInfoFileName); Zotero.uiReadyPromise.delay(30000).then(() => { this.registerContentProcessor(); @@ -135,6 +131,22 @@ Zotero.Fulltext = Zotero.FullText = new function(){ }); + this.setPDFConverterPath = function(path) { + _pdfConverter = Zotero.File.pathToFile(path); + }; + + + this.setPDFInfoPath = function(path) { + _pdfInfo = Zotero.File.pathToFile(path); + + }; + + + this.setPDFDataPath = function(path) { + _pdfData = path; + }; + + this.getLibraryVersion = function (libraryID) { if (!libraryID) throw new Error("libraryID not provided"); return Zotero.DB.valueQueryAsync( @@ -201,7 +213,7 @@ Zotero.Fulltext = Zotero.FullText = new function(){ this.getPDFConverterExecAndArgs = function () { return { exec: _pdfConverter, - args: ['-datadir', _popplerDatadir] + args: ['-datadir', _pdfData] } }; diff --git a/test/content/runtests.js b/test/content/runtests.js index 82aa700b6..712dc6910 100644 --- a/test/content/runtests.js +++ b/test/content/runtests.js @@ -293,6 +293,34 @@ if(run) { window.onload = function() { Zotero.spawn(function* () { yield Zotero.Schema.schemaUpdatePromise; + + // Init paths for PDF tools and data + let pdfConvertedFileName = 'pdftotext'; + let pdfInfoFileName = 'pdfinfo'; + + if (Zotero.isWin) { + pdfConvertedFileName += '-win.exe'; + pdfInfoFileName += '-win.exe'; + } + else if (Zotero.isMac) { + pdfConvertedFileName += '-mac'; + pdfInfoFileName += '-mac'; + } + else { + let cpu = Zotero.platform.split(' ')[1]; + pdfConvertedFileName += '-linux-' + cpu; + pdfInfoFileName += '-linux-' + cpu; + } + + let pdfToolsPath = OS.Path.join(Zotero.Profile.dir, 'pdftools'); + let pdfConverterPath = OS.Path.join(pdfToolsPath, pdfConvertedFileName); + let pdfInfoPath = OS.Path.join(pdfToolsPath, pdfInfoFileName); + let pdfDataPath = OS.Path.join(pdfToolsPath, 'poppler-data'); + + Zotero.FullText.setPDFConverterPath(pdfConverterPath); + Zotero.FullText.setPDFInfoPath(pdfInfoPath); + Zotero.FullText.setPDFDataPath(pdfDataPath); + return mocha.run(); }) }; diff --git a/test/runtests.sh b/test/runtests.sh index 99a147806..5914f9115 100755 --- a/test/runtests.sh +++ b/test/runtests.sh @@ -125,6 +125,15 @@ echo "$ZOTERO_UNIT_PATH" > "$PROFILE/extensions/zotero-unit@zotero.org" # Create data directory mkdir "$TEMPDIR/Zotero" +# Download PDF tools +PDF_TOOLS_VERSION="0.0.1" +PDF_TOOLS_URL="https://zotero-download.s3.amazonaws.com/pdftools/pdftools-$PDF_TOOLS_VERSION.tar.gz" + +PDF_TOOLS_DIR="$PROFILE/pdftools" +mkdir $PDF_TOOLS_DIR +curl -o "$PDF_TOOLS_DIR/pdftools.tar.gz" $PDF_TOOLS_URL +tar -zxf "$PDF_TOOLS_DIR/pdftools.tar.gz" -C $PDF_TOOLS_DIR + cat < "$PROFILE/prefs.js" user_pref("app.update.enabled", false); user_pref("extensions.autoDisableScopes", 0); From 723b4d32e5eb5116d3b81f38eaab73870789e5a0 Mon Sep 17 00:00:00 2001 From: Martynas Bagdonas Date: Thu, 18 Jan 2018 16:48:44 +0200 Subject: [PATCH 2/4] Fix Zotero.Fulltext tests --- test/content/runtests.js | 27 +-------------------------- test/content/support.js | 31 +++++++++++++++++++++++++++++++ test/tests/fulltextTest.js | 3 ++- test/tests/recognizePDFTest.js | 1 + 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/test/content/runtests.js b/test/content/runtests.js index 712dc6910..b06ffcafd 100644 --- a/test/content/runtests.js +++ b/test/content/runtests.js @@ -294,32 +294,7 @@ if(run) { Zotero.spawn(function* () { yield Zotero.Schema.schemaUpdatePromise; - // Init paths for PDF tools and data - let pdfConvertedFileName = 'pdftotext'; - let pdfInfoFileName = 'pdfinfo'; - - if (Zotero.isWin) { - pdfConvertedFileName += '-win.exe'; - pdfInfoFileName += '-win.exe'; - } - else if (Zotero.isMac) { - pdfConvertedFileName += '-mac'; - pdfInfoFileName += '-mac'; - } - else { - let cpu = Zotero.platform.split(' ')[1]; - pdfConvertedFileName += '-linux-' + cpu; - pdfInfoFileName += '-linux-' + cpu; - } - - let pdfToolsPath = OS.Path.join(Zotero.Profile.dir, 'pdftools'); - let pdfConverterPath = OS.Path.join(pdfToolsPath, pdfConvertedFileName); - let pdfInfoPath = OS.Path.join(pdfToolsPath, pdfInfoFileName); - let pdfDataPath = OS.Path.join(pdfToolsPath, 'poppler-data'); - - Zotero.FullText.setPDFConverterPath(pdfConverterPath); - Zotero.FullText.setPDFInfoPath(pdfInfoPath); - Zotero.FullText.setPDFDataPath(pdfDataPath); + initPDFToolsPath(); return mocha.run(); }) diff --git a/test/content/support.js b/test/content/support.js index 8bd5a778a..9ef2c6a84 100644 --- a/test/content/support.js +++ b/test/content/support.js @@ -473,6 +473,37 @@ function getPromiseError(promise) { return promise.thenReturn(false).catch(e => e); } +/** + * Init paths for PDF tools and data + */ +function initPDFToolsPath() { + let pdfConvertedFileName = 'pdftotext'; + let pdfInfoFileName = 'pdfinfo'; + + if (Zotero.isWin) { + pdfConvertedFileName += '-win.exe'; + pdfInfoFileName += '-win.exe'; + } + else if (Zotero.isMac) { + pdfConvertedFileName += '-mac'; + pdfInfoFileName += '-mac'; + } + else { + let cpu = Zotero.platform.split(' ')[1]; + pdfConvertedFileName += '-linux-' + cpu; + pdfInfoFileName += '-linux-' + cpu; + } + + let pdfToolsPath = OS.Path.join(Zotero.Profile.dir, 'pdftools'); + let pdfConverterPath = OS.Path.join(pdfToolsPath, pdfConvertedFileName); + let pdfInfoPath = OS.Path.join(pdfToolsPath, pdfInfoFileName); + let pdfDataPath = OS.Path.join(pdfToolsPath, 'poppler-data'); + + Zotero.FullText.setPDFConverterPath(pdfConverterPath); + Zotero.FullText.setPDFInfoPath(pdfInfoPath); + Zotero.FullText.setPDFDataPath(pdfDataPath); +} + /** * Returns the nsIFile corresponding to the test data directory * (i.e., test/tests/data) diff --git a/test/tests/fulltextTest.js b/test/tests/fulltextTest.js index 6024a1e9d..0c9b371a9 100644 --- a/test/tests/fulltextTest.js +++ b/test/tests/fulltextTest.js @@ -5,6 +5,7 @@ describe("Zotero.Fulltext", function () { // Hidden browser, which requires a browser window, needed for charset detection // (until we figure out a better way) win = yield loadBrowserWindow(); + initPDFToolsPath(); }); after(function () { if (win) { @@ -133,7 +134,7 @@ describe("Zotero.Fulltext", function () { toSync.push({ item: pdfAttachment, content: "Zotero [zoh-TAIR-oh] is a free, easy-to-use tool to help you collect, " - + "organize, cite, and share your research sources.\n\n", + + "organize, cite, and share\nyour research sources.\n\n", indexedChars: 0, indexedPages: 1 }); diff --git a/test/tests/recognizePDFTest.js b/test/tests/recognizePDFTest.js index 9d80a1294..5384bd9a8 100644 --- a/test/tests/recognizePDFTest.js +++ b/test/tests/recognizePDFTest.js @@ -9,6 +9,7 @@ describe("PDF Recognition", function() { yield Zotero.Promise.all([ loadZoteroPane().then(w => win = w) ]); + initPDFToolsPath(); }); beforeEach(function* () { From db2ddfd49321a5b7bec1a46c0fcf694819ec18ea Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Thu, 18 Jan 2018 18:53:01 -0500 Subject: [PATCH 3/4] Fix Zotero.Fulltext.setItemContent() test --- test/tests/fulltextTest.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/tests/fulltextTest.js b/test/tests/fulltextTest.js index 0c9b371a9..b8937917c 100644 --- a/test/tests/fulltextTest.js +++ b/test/tests/fulltextTest.js @@ -176,6 +176,16 @@ describe("Zotero.Fulltext", function () { }) describe("#setItemContent()", function () { + before(() => { + // Disable PDF indexing + Zotero.Prefs.set('fulltext.pdfMaxPages', 0); + }); + + after(() => { + // Re-enable PDF indexing + Zotero.Prefs.clear('fulltext.pdfMaxPages'); + }); + it("should store data in .zotero-ft-unprocessed file", function* () { var item = yield importFileAttachment('test.pdf'); From 37eb597ee86fe64d3c4571227108799e7e39351a Mon Sep 17 00:00:00 2001 From: Dan Stillman Date: Thu, 18 Jan 2018 19:20:24 -0500 Subject: [PATCH 4/4] Cache PDF tools in the source directory between test runs This avoids having to download a 13 MB file on every test run. --- .gitignore | 3 ++- test/runtests.sh | 26 ++++++++++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 323a4f034..c469e12a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .DS_Store node_modules build -.signatures.json \ No newline at end of file +.signatures.json +tmp diff --git a/test/runtests.sh b/test/runtests.sh index 5914f9115..16b284881 100755 --- a/test/runtests.sh +++ b/test/runtests.sh @@ -1,5 +1,6 @@ #!/bin/bash -CWD="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +ROOT_DIR="$( cd "$( dirname "$SCRIPT_DIR" )" && pwd )" case "$OSTYPE" in msys*|mingw*|cygwin*) IS_CYGWIN=1 ;; @@ -116,7 +117,7 @@ TEMPDIR="`mktemp -d 2>/dev/null || mktemp -d -t 'zotero-unit'`" PROFILE="$TEMPDIR/profile" mkdir -p "$PROFILE/extensions" -makePath ZOTERO_PATH "`dirname "$CWD"`/build" +makePath ZOTERO_PATH "$ROOT_DIR/build" echo "$ZOTERO_PATH" > "$PROFILE/extensions/zotero@chnm.gmu.edu" makePath ZOTERO_UNIT_PATH "$ZOTERO_PATH/test" @@ -125,14 +126,23 @@ echo "$ZOTERO_UNIT_PATH" > "$PROFILE/extensions/zotero-unit@zotero.org" # Create data directory mkdir "$TEMPDIR/Zotero" -# Download PDF tools +# Download PDF tools if not cached in the source directory and copy to profile directory PDF_TOOLS_VERSION="0.0.1" PDF_TOOLS_URL="https://zotero-download.s3.amazonaws.com/pdftools/pdftools-$PDF_TOOLS_VERSION.tar.gz" - +PDF_TOOLS_CACHE_DIR="$ROOT_DIR/tmp/pdftools" PDF_TOOLS_DIR="$PROFILE/pdftools" -mkdir $PDF_TOOLS_DIR -curl -o "$PDF_TOOLS_DIR/pdftools.tar.gz" $PDF_TOOLS_URL -tar -zxf "$PDF_TOOLS_DIR/pdftools.tar.gz" -C $PDF_TOOLS_DIR +if [ ! -f "$PDF_TOOLS_CACHE_DIR/$PDF_TOOLS_VERSION" ]; then + echo "Fetching PDF tools version $PDF_TOOLS_VERSION" + echo + rm -rf "$PDF_TOOLS_CACHE_DIR" + mkdir -p "$PDF_TOOLS_CACHE_DIR" + curl -o "$PDF_TOOLS_CACHE_DIR/pdftools.tar.gz" $PDF_TOOLS_URL + tar -zxf "$PDF_TOOLS_CACHE_DIR/pdftools.tar.gz" -C $PDF_TOOLS_CACHE_DIR + rm "$PDF_TOOLS_CACHE_DIR/pdftools.tar.gz" + touch "$PDF_TOOLS_CACHE_DIR/$PDF_TOOLS_VERSION" + echo +fi +cp -R $PDF_TOOLS_CACHE_DIR $PDF_TOOLS_DIR cat < "$PROFILE/prefs.js" user_pref("app.update.enabled", false); @@ -179,7 +189,7 @@ trap "{ rm -rf \"$TEMPDIR\"; }" EXIT if [[ "$TRAVIS" != true ]] && ! ps | grep scripts/build.js | grep -v grep > /dev/null; then echo echo "Running JS build process" - cd "$CWD/.." + cd "$ROOT_DIR" npm run build || exit $? echo fi