feat(config): add failOnSkippedTests option. (#3374)

Recently jasmine added support for reporting pending (incl due to fit()) tests.
This appears in results as skipped tests and this option allows us to fail tests
with skipped tests'

docs/config/01-configuration-file.md

@@ -383,6 +383,16 @@ The server can be stopped using the `karma stop` command.
 **Description:** Enable or disable failure on running empty test-suites. If disabled the program
 will return exit-code `0` and display a warning.
 
+## failOnSkippedTests
+**Type:** Boolean
+
+**Default:** `false`
+
+**CLI:** `--fail-on-skipped-tests`, `--no-fail-on-skipped-tests`
+
+**Description:** Enable or disable failure on tests deliberately disabled, eg fit() or xit() tests in jasmine.
+Use this to prevent accidental disabling tests needed to validate production.
+
 ## failOnFailingTestSuite
 **Type:** Boolean
 

lib/browser_collection.js

@@ -38,30 +38,37 @@ class BrowserCollection {
     return this.browsers.map((browser) => browser.serialize())
   }
 
-  calculateExitCode (results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
+  calculateExitCode (results, singleRunBrowserNotCaptured, config) {
+    config = config || {}
     if (results.disconnected || singleRunBrowserNotCaptured) {
       return 1
-    } else if (results.success + results.failed === 0 && !failOnEmptyTestSuite) {
+    }
+    if (results.skipped && config.failOnSkippedTests) {
+      return 1
+    }
+    if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {
       return 0
-    } else if (results.error) {
+    }
+    if (results.error) {
       return 1
-    } else if (failOnFailingTestSuite === false) {
+    }
+    if (config.failOnFailingTestSuite === false) {
       return 0 // Tests executed without infrastructure error, exit with 0 independent of test status.
-    } else {
-      return results.failed ? 1 : 0
     }
+    return results.failed ? 1 : 0
   }
 
-  getResults (singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
-    const results = { success: 0, failed: 0, error: false, disconnected: false, exitCode: 0 }
+  getResults (singleRunBrowserNotCaptured, config) {
+    const results = { success: 0, failed: 0, skipped: 0, error: false, disconnected: false, exitCode: 0 }
     this.browsers.forEach((browser) => {
       results.success += browser.lastResult.success
       results.failed += browser.lastResult.failed
+      results.skipped += browser.lastResult.skipped
       results.error = results.error || browser.lastResult.error
       results.disconnected = results.disconnected || browser.lastResult.disconnected
     })
 
-    results.exitCode = this.calculateExitCode(results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite)
+    results.exitCode = this.calculateExitCode(results, singleRunBrowserNotCaptured, config)
     return results
   }
 

test/unit/browser_collection.spec.js

@@ -281,15 +281,15 @@ describe('BrowserCollection', () => {
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
       it('shall pass if failOnEmptyTestSuite is false', () => {
-        const res = collection.calculateExitCode(results, false, false)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: false })
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
       it('shall fail if failOnEmptyTestSuite is true', () => {
-        const res = collection.calculateExitCode(results, false, true)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true })
         expect(res).to.be.equal(EXIT_CODE_ERROR)
       })
       it('shall fail if failOnFailingTestSuite is set', () => {
-        const res = collection.calculateExitCode(results, false, true, true)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: true })
         expect(res).to.be.equal(EXIT_CODE_ERROR)
       })
     })
@@ -308,15 +308,15 @@ describe('BrowserCollection', () => {
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
       it('shall pass if failOnEmptyTestSuite not is set', () => {
-        const res = collection.calculateExitCode(results, false, false)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: false })
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
       it('shall pass if failOnFailingTestSuite is true', () => {
-        const res = collection.calculateExitCode(results, false, true, true)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: true })
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
       it('shall pass if failOnFailingTestSuite is false', () => {
-        const res = collection.calculateExitCode(results, false, true, false)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: false })
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
     })
@@ -335,15 +335,34 @@ describe('BrowserCollection', () => {
         expect(res).to.be.equal(EXIT_CODE_ERROR)
       })
       it('shall fail if failOnEmptyTestSuite not is set', () => {
-        const res = collection.calculateExitCode(results, false, false)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: false })
         expect(res).to.be.equal(EXIT_CODE_ERROR)
       })
       it('shall fail if failOnFailingTestSuite is true', () => {
-        const res = collection.calculateExitCode(results, false, true, true)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: true })
         expect(res).to.be.equal(EXIT_CODE_ERROR)
       })
       it('shall pass if failOnFailingTestSuite is false', () => {
-        const res = collection.calculateExitCode(results, false, true, false)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: false })
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+    })
+    describe('all test passed, with skipped tests', () => {
+      const results = {
+        success: 10,
+        skipped: 5,
+        error: false
+      }
+      it('shall fail if singleRunBrowserNotCaptured is true', () => {
+        const res = collection.calculateExitCode(results, true)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall fail if failOnSkippedTests is true', () => {
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnSkippedTests: true })
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall pass if failOnSkippedTests is false', () => {
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnSkippedTests: false })
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
     })