feat: Do not fail on empty test suite

Add option to configuration-file and cli-args to server (single run) and runner,
for disabling failure on empty test suite. Will instead display a warning.

Closes #926

lib/cli.js

@@ -37,6 +37,10 @@ var processArgs = function (argv, options, fs, path) {
     options.colors = options.colors === 'true'
   }
 
+  if (helper.isString(options.failOnEmptyTestSuite)) {
+    options.failOnEmptyTestSuite = options.failOnEmptyTestSuite === 'true'
+  }
+
   if (helper.isString(options.logLevel)) {
     var logConstant = constant['LOG_' + options.logLevel.toUpperCase()]
     if (helper.isDefined(logConstant)) {
@@ -165,6 +169,8 @@ var describeStart = function () {
     .describe('single-run', 'Run the test when browsers captured and exit.')
     .describe('no-single-run', 'Disable single-run.')
     .describe('report-slower-than', '<integer> Report tests that are slower than given time [ms].')
+    .describe('fail-on-empty-test-suite', 'Fail on empty test suite.')
+    .describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.')
     .describe('help', 'Print usage and options.')
 }
 
@@ -176,6 +182,8 @@ var describeRun = function () {
       '  $0 run [<configFile>] [<options>] [ -- <clientArgs>]')
     .describe('port', '<integer> Port where the server is listening.')
     .describe('no-refresh', 'Do not re-glob all the patterns.')
+    .describe('fail-on-empty-test-suite', 'Fail on empty test suite.')
+    .describe('no-fail-on-empty-test-suite', 'Do not fail on empty test suite.')
     .describe('help', 'Print usage.')
 }
 

lib/config.js

@@ -260,6 +260,7 @@ var Config = function () {
   this.browserDisconnectTolerance = 0
   this.browserNoActivityTimeout = 10000
   this.concurrency = Infinity
+  this.failOnEmptyTestSuite = true
 }
 
 var CONFIG_SYNTAX_HELP = '  module.exports = function(config) {\n' +

lib/middleware/runner.js

@@ -41,7 +41,8 @@ var createRunnerMiddleware = function (emitter, fileList, capturedBrowsers, repo
         // clean up, close runner response
         emitter.once('run_complete', function (browsers, results) {
           reporter.removeAdapter(responseWrite)
-          response.end(constant.EXIT_CODE + results.exitCode)
+          var emptyTestSuite = (results.failed + results.success) === 0 ? 0 : 1
+          response.end(constant.EXIT_CODE + emptyTestSuite + results.exitCode)
         })
       })
 

lib/runner.js

@@ -3,9 +3,11 @@ var http = require('http')
 var constant = require('./constants')
 var helper = require('./helper')
 var cfg = require('./config')
+var logger = require('./logger')
+var log = logger.create('runner')
 
-var parseExitCode = function (buffer, defaultCode) {
-  var tailPos = buffer.length - Buffer.byteLength(constant.EXIT_CODE) - 1
+var parseExitCode = function (buffer, defaultCode, failOnEmptyTestSuite) {
+  var tailPos = buffer.length - Buffer.byteLength(constant.EXIT_CODE) - 2
 
   if (tailPos < 0) {
     return defaultCode
@@ -14,9 +16,15 @@ var parseExitCode = function (buffer, defaultCode) {
   // tail buffer which might contain the message
   var tail = buffer.slice(tailPos)
   var tailStr = tail.toString()
-  if (tailStr.substr(0, tailStr.length - 1) === constant.EXIT_CODE) {
+  if (tailStr.substr(0, tailStr.length - 2) === constant.EXIT_CODE) {
     tail.fill('\x00')
-    return parseInt(tailStr.substr(-1), 10)
+    var emptyInt = parseInt(tailStr.substr(-2, 1), 10)
+    var exitCode = parseInt(tailStr.substr(-1), 10)
+    if (failOnEmptyTestSuite === false && emptyInt === 0) {
+      log.warn('Test suite was empty.')
+      return 0
+    }
+    return exitCode
   }
 
   return defaultCode
@@ -40,7 +48,7 @@ exports.run = function (config, done) {
 
   var request = http.request(options, function (response) {
     response.on('data', function (buffer) {
-      exitCode = parseExitCode(buffer, exitCode)
+      exitCode = parseExitCode(buffer, exitCode, config.failOnEmptyTestSuite)
       process.stdout.write(buffer)
     })
 
@@ -51,7 +59,7 @@ exports.run = function (config, done) {
 
   request.on('error', function (e) {
     if (e.code === 'ECONNREFUSED') {
-      console.error('There is no server listening on port %d', options.port)
+      log.error('There is no server listening on port %d', options.port)
       done(1, e.code)
     } else {
       throw e

lib/server.js

@@ -251,8 +251,10 @@ Server.prototype._start = function (config, launcher, preprocess, fileList, webS
       var results = singleRunBrowsers.getResults()
       if (singleRunBrowserNotCaptured) {
         results.exitCode = 1
+      } else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {
+        results.exitCode = 0
+        self.log.warn('Test suite was empty.')
       }
-
       self.emit('run_complete', singleRunBrowsers, results)
     }
   }

test/unit/middleware/runner.spec.js

@@ -73,7 +73,7 @@ describe('middleware.runner', () => {
 
     response.once('end', () => {
       expect(nextSpy).to.not.have.been.called
-      expect(response).to.beServedAs(200, 'result\x1FEXIT0')
+      expect(response).to.beServedAs(200, 'result\x1FEXIT10')
       done()
     })
 
@@ -83,6 +83,54 @@ describe('middleware.runner', () => {
     emitter.emit('run_complete', capturedBrowsers, {exitCode: 0})
   })
 
+  it('should set the empty to 0 if empty results', (done) => {
+    capturedBrowsers.add(new Browser())
+    sinon.stub(capturedBrowsers, 'areAllReady', () => true)
+
+    response.once('end', () => {
+      expect(nextSpy).to.not.have.been.called
+      expect(response).to.beServedAs(200, 'result\x1FEXIT00')
+      done()
+    })
+
+    handler(new HttpRequestMock('/__run__'), response, nextSpy)
+
+    mockReporter.write('result')
+    emitter.emit('run_complete', capturedBrowsers, {exitCode: 0, success: 0, failed: 0})
+  })
+
+  it('should set the empty to 1 if successful tests', (done) => {
+    capturedBrowsers.add(new Browser())
+    sinon.stub(capturedBrowsers, 'areAllReady', () => true)
+
+    response.once('end', () => {
+      expect(nextSpy).to.not.have.been.called
+      expect(response).to.beServedAs(200, 'result\x1FEXIT10')
+      done()
+    })
+
+    handler(new HttpRequestMock('/__run__'), response, nextSpy)
+
+    mockReporter.write('result')
+    emitter.emit('run_complete', capturedBrowsers, {exitCode: 0, success: 3, failed: 0})
+  })
+
+  it('should set the empty to 1 if failed tests', (done) => {
+    capturedBrowsers.add(new Browser())
+    sinon.stub(capturedBrowsers, 'areAllReady', () => true)
+
+    response.once('end', () => {
+      expect(nextSpy).to.not.have.been.called
+      expect(response).to.beServedAs(200, 'result\x1FEXIT10')
+      done()
+    })
+
+    handler(new HttpRequestMock('/__run__'), response, nextSpy)
+
+    mockReporter.write('result')
+    emitter.emit('run_complete', capturedBrowsers, {exitCode: 0, success: 0, failed: 6})
+  })
+
   it('should not run if there is no browser captured', (done) => {
     sinon.stub(fileListMock, 'refresh')
 

test/unit/runner.spec.js

@@ -12,14 +12,14 @@ describe('runner', () => {
     var EXIT = constant.EXIT_CODE
 
     it('should return 0 exit code if present in the buffer', () => {
-      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}0`))).to.equal(0)
+      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}10`))).to.equal(0)
     })
 
     it('should null the exit code part of the buffer', () => {
-      var buffer = new Buffer(`some${EXIT}1`)
+      var buffer = new Buffer(`some${EXIT}01`)
       m.parseExitCode(buffer)
 
-      expect(buffer.toString()).to.equal('some\0\0\0\0\0\0')
+      expect(buffer.toString()).to.equal('some\0\0\0\0\0\0\0')
     })
 
     it('should not touch buffer without exit code and return default', () => {
@@ -41,8 +41,21 @@ describe('runner', () => {
     })
 
     it('should parse any single digit exit code', () => {
-      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}1`))).to.equal(1)
-      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}7`))).to.equal(7)
+      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}01`))).to.equal(1)
+      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}17`))).to.equal(7)
+    })
+
+    it('should return exit code 0 if failOnEmptyTestSuite is false and and non-empty int is 0', () => {
+      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}01`), undefined, false)).to.equal(0)
+    })
+
+    it('should return exit code if failOnEmptyTestSuite is true', () => {
+      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}00`), undefined, true)).to.equal(0)
+      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}01`), undefined, true)).to.equal(1)
+      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}07`), undefined, true)).to.equal(7)
+      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}10`), undefined, true)).to.equal(0)
+      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}11`), undefined, true)).to.equal(1)
+      expect(m.parseExitCode(new Buffer(`something\nfake${EXIT}17`), undefined, true)).to.equal(7)
     })
   })
 })