Merge pull request #81 from groupon-testium/before-hook-failure

allow before hook failures to auto save screenshots

README.md

@@ -98,7 +98,7 @@ testOptions =
     timeout: 60000
     connectTimeout: 20000
   mochaOptions:
-    reporter: 'spec'
+    reporter: 'spec' # the screenshot on failure future will stop working if you set this
     timeout: 20000
     slow: 4000
 

lib/test_runner/index.js

@@ -32,11 +32,12 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 // Generated by CoffeeScript 2.0.0-beta7
 void function () {
-  var BROWSERS, cache$, compact, extend, files, Mocha, passValuesToTestiumBeforeTestFiles, path, store, validateBrowser;
+  var BROWSERS, cache$, compact, extend, files, Mocha, passValuesToTestiumBeforeTestFiles, path, Reporter, store, validateBrowser;
   Mocha = require('mocha');
   path = require('path');
   store = require('../test_setup/store');
   files = require('./files');
+  Reporter = require('./reporter');
   cache$ = require('underscore');
   compact = cache$.compact;
   extend = cache$.extend;
@@ -85,7 +86,7 @@ void function () {
     runMocha = function (testFiles) {
       var defaults, mocha;
       defaults = {
-        reporter: 'spec',
+        reporter: Reporter(options.screenshotDirectory),
         timeout: 2e4,
         slow: 2e3
       };

lib/test_runner/reporter.js

@@ -0,0 +1,94 @@
+/*
+Copyright (c) 2014, Groupon, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+Neither the name of GROUPON nor the names of its contributors may be
+used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+// Generated by CoffeeScript 2.0.0-beta7
+void function () {
+  var Base, color, cursor, getBrowser, takeScreenshotOnFailure;
+  Base = require('mocha').reporters.Base;
+  cursor = Base.cursor;
+  color = Base.color;
+  getBrowser = require('../index').getBrowser;
+  takeScreenshotOnFailure = require('../test_setup/screenshot').takeScreenshotOnFailure;
+  module.exports = function (screenshotDirectory) {
+    var Spec;
+    Spec = function (runner) {
+      var indent, indents, n, self, stats;
+      Base.call(this, runner);
+      self = this;
+      stats = this.stats;
+      indents = 0;
+      n = 0;
+      indent = function () {
+        return Array(indents).join('  ');
+      };
+      runner.on('start', function () {
+        return console.log();
+      });
+      runner.on('suite', function (suite) {
+        ++indents;
+        return console.log(color('suite', '%s%s'), indent(), suite.title);
+      });
+      runner.on('suite end', function (suite) {
+        --indents;
+        if (1 === indents)
+          return console.log();
+      });
+      runner.on('pending', function (test) {
+        var fmt;
+        fmt = indent() + color('pending', '  - %s');
+        return console.log(fmt, test.title);
+      });
+      runner.on('pass', function (test) {
+        var fmt;
+        if ('fast' === test.speed) {
+          fmt = indent() + color('checkmark', '  ' + Base.symbols.ok) + color('pass', ' %s ');
+          cursor.CR();
+          return console.log(fmt, test.title);
+        } else {
+          fmt = indent() + color('checkmark', '  ' + Base.symbols.ok) + color('pass', ' %s ') + color(test.speed, '(%dms)');
+          cursor.CR();
+          return console.log(fmt, test.title, test.duration);
+        }
+      });
+      runner.on('fail', function (test, err) {
+        var browser;
+        browser = getBrowser();
+        takeScreenshotOnFailure(screenshotDirectory, test, browser);
+        cursor.CR();
+        return console.log(indent() + color('fail', '  %d) %s'), ++n, test.title);
+      });
+      return runner.on('end', self.epilogue.bind(self));
+    };
+    Spec.prototype.__proto__ = Base.prototype;
+    return Spec;
+  };
+}.call(this);

lib/test_setup/index.js

@@ -32,7 +32,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 // Generated by CoffeeScript 2.0.0-beta7
 void function () {
-  var browser, exit, getBrowser, logError, store, takeScreenshotOnFailure;
+  var browser, exit, getBrowser, logError;
   logError = require('../log/error');
   getBrowser = require('./browser').getBrowser;
   browser = getBrowser();
@@ -61,15 +61,8 @@ void function () {
     }, 1e3);
   };
   process.on('uncaughtException', exit);
-  takeScreenshotOnFailure = require('./screenshot').takeScreenshotOnFailure;
-  store = require('./store');
   before(function () {
     return browser.navigateTo('/testium-priming-load');
   });
-  afterEach(function () {
-    var screenshotDirectory;
-    screenshotDirectory = store.get().screenshotDirectory;
-    return takeScreenshotOnFailure(screenshotDirectory, this.currentTest, browser);
-  });
   after(global.exitMocha);
 }.call(this);

src/test_runner/index.coffee

@@ -34,6 +34,7 @@ Mocha = require 'mocha'
 path = require 'path'
 store = require '../test_setup/store'
 files = require './files'
+Reporter = require './reporter'
 {compact, extend} = require 'underscore'
 
 require('coffee-script-redux/register')
@@ -72,7 +73,7 @@ process.on 'message', (options) ->
 
   runMocha = (testFiles) ->
     defaults =
-      reporter: 'spec'
+      reporter: Reporter(options.screenshotDirectory)
       timeout: 20000
       slow: 2000
     options = extend {}, defaults, mochaOptions

src/test_runner/reporter.coffee

@@ -0,0 +1,95 @@
+###
+Copyright (c) 2014, Groupon, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+Neither the name of GROUPON nor the names of its contributors may be
+used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+###
+
+# Based on Mocha's Spec reporter.
+#
+# Adds ability to ensure we
+# capture failures as they happen,
+# even if it's in a before hook.
+
+Base = require('mocha').reporters.Base
+cursor = Base.cursor
+color = Base.color
+
+{getBrowser} = require '../index'
+{takeScreenshotOnFailure} = require '../test_setup/screenshot'
+
+module.exports = (screenshotDirectory) ->
+  Spec = (runner) ->
+    Base.call this, runner
+
+    self = this
+    stats = @stats
+    indents = 0
+    n = 0
+
+    indent = ->
+      Array(indents).join "  "
+
+    runner.on "start", ->
+      console.log()
+
+    runner.on "suite", (suite) ->
+      ++indents
+      console.log color("suite", "%s%s"), indent(), suite.title
+
+    runner.on "suite end", (suite) ->
+      --indents
+      console.log()  if 1 is indents
+
+    runner.on "pending", (test) ->
+      fmt = indent() + color("pending", "  - %s")
+      console.log fmt, test.title
+
+    runner.on "pass", (test) ->
+      if "fast" is test.speed
+        fmt = indent() + color("checkmark", "  " + Base.symbols.ok) + color("pass", " %s ")
+        cursor.CR()
+        console.log fmt, test.title
+      else
+        fmt = indent() + color("checkmark", "  " + Base.symbols.ok) + color("pass", " %s ") + color(test.speed, "(%dms)")
+        cursor.CR()
+        console.log fmt, test.title, test.duration
+
+    runner.on "fail", (test, err) ->
+      browser = getBrowser()
+      takeScreenshotOnFailure(screenshotDirectory, test, browser)
+
+      cursor.CR()
+      console.log indent() + color("fail", "  %d) %s"), ++n, test.title
+
+    runner.on "end", self.epilogue.bind(self)
+
+  Spec.prototype.__proto__ = Base.prototype
+
+  return Spec
+

src/test_setup/index.coffee

@@ -58,18 +58,12 @@ exit = (error) ->
 
 process.on 'uncaughtException', exit
 
-{takeScreenshotOnFailure} = require './screenshot'
-store = require './store'
-
 before ->
   # first page load is guranteed
   # by the proxy to be a success
   # this allows us to set cookies
   # right away in tests
   browser.navigateTo '/testium-priming-load'
 
-afterEach ->
-  {screenshotDirectory} = store.get()
-  takeScreenshotOnFailure(screenshotDirectory, @currentTest, browser)
 after(global.exitMocha)
 

test/integration_runner.coffee

@@ -4,6 +4,7 @@ testApp = require './app'
 testium = require '../lib/index'
 {series} = require 'async'
 
+APP_DIRECTORY = "#{__dirname}/.."
 TEST_DIRECTORY = "#{__dirname}/integration"
 LOG_DIRECTORY = "#{__dirname}/integration_log"
 SCREENSHOT_DIRECTORY = "#{LOG_DIRECTORY}/screenshots"
@@ -26,13 +27,16 @@ ensureEmpty = (path) ->
   mkdirp.sync path
 
 runTests = (callback) ->
+  tests = APP_DIRECTORY + '/' + process.env.TESTS
+  tests ?= TEST_DIRECTORY
+
   testBrowser = (browser) -> (browserTested) ->
     console.log "\nTesting against: #{browser}\n"
     options =
-      tests: TEST_DIRECTORY
+      tests: tests
       screenshotDirectory: SCREENSHOT_DIRECTORY
       logDirectory: LOG_DIRECTORY
-      appDirectory: "#{__dirname}/.."
+      appDirectory: APP_DIRECTORY
       applicationPort: 4003
       browser: browser
       http: