feat(config): add failOnPendingTests option.

Recently jasmine added support for reporting pending (incl due to fit()) tests. This appears in results as skipped tests and this option allows us to fail tests with skipped tests' '
karma-runner · Oct 1, 2019 · 1a20132 · 1a20132
1 parent 6235e68
commit 1a20132
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 15 deletions.
diff --git a/docs/config/01-configuration-file.md b/docs/config/01-configuration-file.md
@@ -383,6 +383,16 @@ The server can be stopped using the `karma stop` command.
 **Description:** Enable or disable failure on running empty test-suites. If disabled the program
 will return exit-code `0` and display a warning.
 
+## failOnPendingTests
+**Type:** Boolean
+
+**Default:** `false`
+
+**CLI:** `--fail-on-pending-tests`, `--no-fail-on-pending-tests`
+
+**Description:** Enable or disable failure on tests deliberately disabled, eg fit() or xit() tests in jasmine.
+Use this to prevent accidental disabling tests needed to validate production.
+
 ## files
 **Type:** Array
 

diff --git a/lib/browser_collection.js b/lib/browser_collection.js
@@ -38,30 +38,35 @@ class BrowserCollection {
     return this.browsers.map((browser) => browser.serialize())
   }
 
-  calculateExitCode (results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
+  calculateExitCode (results, singleRunBrowserNotCaptured, config) {
+    config = config || {}
+    console.log('config', config)
     if (results.disconnected || singleRunBrowserNotCaptured) {
       return 1
-    } else if (results.success + results.failed === 0 && !failOnEmptyTestSuite) {
+    } else if (results.skipped && config.failOnPendingTests) {
+      return 1
+    } else if (results.success + results.failed === 0 && !config.failOnEmptyTestSuite) {
       return 0
     } else if (results.error) {
       return 1
-    } else if (failOnFailingTestSuite === false) {
+    } else if (config.failOnFailingTestSuite === false) {
       return 0 // Tests executed without infrastructure error, exit with 0 independent of test status.
     } else {
       return results.failed ? 1 : 0
     }
   }
 
-  getResults (singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite) {
-    const results = { success: 0, failed: 0, error: false, disconnected: false, exitCode: 0 }
+  getResults (singleRunBrowserNotCaptured, config) {
+    const results = { success: 0, failed: 0, skipped: 0, error: false, disconnected: false, exitCode: 0 }
     this.browsers.forEach((browser) => {
       results.success += browser.lastResult.success
       results.failed += browser.lastResult.failed
+      results.skipped += browser.lastResult.skipped
       results.error = results.error || browser.lastResult.error
       results.disconnected = results.disconnected || browser.lastResult.disconnected
     })
 
-    results.exitCode = this.calculateExitCode(results, singleRunBrowserNotCaptured, failOnEmptyTestSuite, failOnFailingTestSuite)
+    results.exitCode = this.calculateExitCode(results, singleRunBrowserNotCaptured, config)
     return results
   }
 

diff --git a/test/unit/browser_collection.spec.js b/test/unit/browser_collection.spec.js
@@ -281,15 +281,15 @@ describe('BrowserCollection', () => {
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
       it('shall pass if failOnEmptyTestSuite is false', () => {
-        const res = collection.calculateExitCode(results, false, false)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: false })
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
       it('shall fail if failOnEmptyTestSuite is true', () => {
-        const res = collection.calculateExitCode(results, false, true)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true })
         expect(res).to.be.equal(EXIT_CODE_ERROR)
       })
       it('shall fail if failOnFailingTestSuite is set', () => {
-        const res = collection.calculateExitCode(results, false, true, true)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: true })
         expect(res).to.be.equal(EXIT_CODE_ERROR)
       })
     })
@@ -308,15 +308,15 @@ describe('BrowserCollection', () => {
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
       it('shall pass if failOnEmptyTestSuite not is set', () => {
-        const res = collection.calculateExitCode(results, false, false)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: false })
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
       it('shall pass if failOnFailingTestSuite is true', () => {
-        const res = collection.calculateExitCode(results, false, true, true)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: true })
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
       it('shall pass if failOnFailingTestSuite is false', () => {
-        const res = collection.calculateExitCode(results, false, true, false)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: false })
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
     })
@@ -335,15 +335,34 @@ describe('BrowserCollection', () => {
         expect(res).to.be.equal(EXIT_CODE_ERROR)
       })
       it('shall fail if failOnEmptyTestSuite not is set', () => {
-        const res = collection.calculateExitCode(results, false, false)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: false })
         expect(res).to.be.equal(EXIT_CODE_ERROR)
       })
       it('shall fail if failOnFailingTestSuite is true', () => {
-        const res = collection.calculateExitCode(results, false, true, true)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: true })
         expect(res).to.be.equal(EXIT_CODE_ERROR)
       })
       it('shall pass if failOnFailingTestSuite is false', () => {
-        const res = collection.calculateExitCode(results, false, true, false)
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnFailingTestSuite: false })
+        expect(res).to.be.equal(EXIT_CODE_SUCCESS)
+      })
+    })
+    describe('all test passed, with skipped tests', () => {
+      const results = {
+        success: 10,
+        skipped: 5,
+        error: false
+      }
+      it('shall fail if singleRunBrowserNotCaptured is true', () => {
+        const res = collection.calculateExitCode(results, true)
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall fail if failOnPendingTests is true', () => {
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnPendingTests: true })
+        expect(res).to.be.equal(EXIT_CODE_ERROR)
+      })
+      it('shall pass if failOnPendingTests is false', () => {
+        const res = collection.calculateExitCode(results, false, { failOnEmptyTestSuite: true, failOnPendingTests: false })
         expect(res).to.be.equal(EXIT_CODE_SUCCESS)
       })
     })