From 5ecbcf680871fca66a8330a54a0700641923dfae Mon Sep 17 00:00:00 2001 From: Martin von Gagern Date: Fri, 6 Nov 2015 10:23:19 +0100 Subject: [PATCH 1/2] Check screenshots on Travis using Docker and Selenium Thanks to the docker service provided by Travis CI, we should be able to download and use the Selenium docker images in order to run our screenshotter and check whether all the screenshots match the images from the repository. --- .travis.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 78e1c0146..51cfabef4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,13 @@ language: node_js node_js: -- "0.11" -- "0.10" + - stable +sudo: required +services: + - docker +before_script: + - docker pull selenium/standalone-firefox:2.46.0 + - docker pull selenium/standalone-chrome:2.46.0 +script: + - npm test + - dockers/Screenshotter/screenshotter.sh + - git diff --name-only --exit-code From 9fb04dbb2e3fb9bae62bf1ad08ab02506914d2db Mon Sep 17 00:00:00 2001 From: Martin von Gagern Date: Fri, 6 Nov 2015 12:43:34 +0100 Subject: [PATCH 2/2] Add verification mode to screenshotter Now Travis can run the screenshotter in verification mode. The files in the repository will be seen as the expected outcome, and if the actual result differs from that, it might be attempted four more times before the test case is actually deemed failed. A timeout between page load and screenshot should allow any possible font issues to settle down. --- .travis.yml | 3 +- dockers/Screenshotter/screenshotter.js | 101 ++++++++++++++++++++----- 2 files changed, 83 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index 51cfabef4..5c5d06c92 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,5 +9,4 @@ before_script: - docker pull selenium/standalone-chrome:2.46.0 script: - npm test - - dockers/Screenshotter/screenshotter.sh - - git diff --name-only --exit-code + - dockers/Screenshotter/screenshotter.sh --verify diff --git a/dockers/Screenshotter/screenshotter.js b/dockers/Screenshotter/screenshotter.js index ddb27cc70..fa9b3218c 100644 --- a/dockers/Screenshotter/screenshotter.js +++ b/dockers/Screenshotter/screenshotter.js @@ -63,6 +63,10 @@ var opts = require("nomnom") abbr: "x", help: "Comma-separated list of test cases to exclude" }) + .option("verify", { + flag: true, + help: "Check whether screenshot matches current file content" + }) .parse(); var listOfCases; @@ -204,7 +208,9 @@ function buildDriver() { builder.usingServer(seleniumURL); } driver = builder.build(); - setSize(targetW, targetH); + driver.manage().timeouts().setScriptTimeout(3000).then(function() { + setSize(targetW, targetH); + }); } ////////////////////////////////////////////////////////////////////// @@ -243,6 +249,9 @@ function imageDimensions(img) { var countdown = listOfCases.length; +var exitStatus = 0; +var listOfFailed = []; + function takeScreenshots() { listOfCases.forEach(takeScreenshot); } @@ -253,9 +262,27 @@ function takeScreenshot(key) { console.error("Test case " + key + " not known!"); return; } + + var file = path.join(dstDir, key + "-" + opts.browser + ".png"); + var retry = 0; + var loadExpected = null; + if (opts.verify) { + loadExpected = promisify(fs.readFile, file); + } + var url = katexURL + "test/screenshotter/test.html?" + itm.query; driver.get(url); - driver.takeScreenshot().then(function haveScreenshot(img) { + driver.takeScreenshot().then(haveScreenshot).then(function() { + if (--countdown === 0) { + if (listOfFailed.length) { + console.error("Failed: " + listOfFailed.join(" ")); + } + // devServer.close(cb) will take too long. + process.exit(exitStatus); + } + }, check); + + function haveScreenshot(img) { img = imageDimensions(img); if (img.width !== targetW || img.height !== targetH) { throw new Error("Excpected " + targetW + " x " + targetH + @@ -270,27 +297,63 @@ function takeScreenshot(key) { * output file name for one of these cases, we accept both. */ key += "_alt"; + file = path.join(dstDir, key + "-" + opts.browser + ".png"); + if (loadExpected) { + loadExpected = promisify(fs.readFile, file); + } } - var file = path.join(dstDir, key + "-" + opts.browser + ".png"); - var deferred = new selenium.promise.Deferred(); var opt = new jspngopt.Optimizer({ pako: pako }); var buf = opt.bufferSync(img.buf); - fs.writeFile(file, buf, function(err) { - if (err) { - deferred.reject(err); - } - else { - deferred.fulfill(); - } - }); - return deferred.promise; - }).then(function() { - console.log(key); - if (--countdown === 0) { - // devServer.close(cb) will take too long. - process.exit(0); + if (loadExpected) { + return loadExpected.then(function(expected) { + if (!buf.equals(expected)) { + if (++retry === 5) { + console.error("FAIL! " + key); + listOfFailed.push(key); + exitStatus = 3; + } else { + console.log("error " + key); + driver.get(url); + browserSideWait(500 * retry); + return driver.takeScreenshot().then(haveScreenshot); + } + } else { + console.log("* ok " + key); + } + }); + } else { + return promisify(fs.writeFile, file, buf).then(function() { + console.log(key); + }); } - }, check); + } +} + +// Wait using a timeout call in the browser, to ensure that the wait +// time doesn't start before the page has reportedly been loaded. +function browserSideWait(milliseconds) { + // The last argument (arguments[1] here) is the callback to selenium + return driver.executeAsyncScript( + "window.setTimeout(arguments[1], arguments[0]);", + milliseconds); +} + +// Turn node callback style into a call returning a promise, +// like Q.nfcall but using Selenium promises instead of Q ones. +// Second and later arguments are passed to the function named in the +// first argument, and a callback is added as last argument. +function promisify(f) { + var args = Array.prototype.slice.call(arguments, 1); + var deferred = new selenium.promise.Deferred(); + args.push(function(err, val) { + if (err) { + deferred.reject(err); + } else { + deferred.fulfill(val); + } + }); + f.apply(null, args); + return deferred.promise; }