Issue 3416: Support multiple errors in the Allure-reporter (#3746)

* Issue 3416 make the final set of updates for the allure reporter, passing all errors combined to allure

* @wdio/allure-reporter:  Polish docs code for #3746

packages/wdio-allure-reporter/src/compoundError.js

@@ -0,0 +1,20 @@
+function indentAll(lines) {
+    return lines.split('\n').map(x => '    ' + x).join('\n')
+}
+
+/**
+ * An error that encapsulates more than one error, to support soft-assertions from Jasmine
+ * even though Allure's API assumes one error-per test
+ */
+export default class CompoundError extends Error {
+    constructor(...innerErrors) {
+        const message = ['CompoundError: One or more errors occurred. ---'].
+            concat(innerErrors.map(x => {
+                if (x.stack) return `${indentAll(x.stack)}\n--- End of stack trace ---`
+                else return `   ${x.message}\n--- End of error message ---`
+            })).join('\n')
+
+        super(message)
+        this.innerErrors = innerErrors
+    }
+}

packages/wdio-allure-reporter/src/index.js

@@ -1,7 +1,7 @@
 import WDIOReporter from '@wdio/reporter'
 import Allure from 'allure-js-commons'
 import Step from 'allure-js-commons/beans/step'
-import { getTestStatus, isEmpty, tellReporter, isMochaEachHooks } from './utils'
+import { getTestStatus, isEmpty, tellReporter, isMochaEachHooks, getErrorFromFailedTest } from './utils'
 import { events, stepStatuses, testStatuses } from './constants'
 
 class AllureReporter extends WDIOReporter {
@@ -87,7 +87,7 @@ class AllureReporter extends WDIOReporter {
             this.allure.endStep(status)
         }
 
-        this.allure.endCase(status, test.error)
+        this.allure.endCase(status, getErrorFromFailedTest(test))
     }
 
     onTestSkip(test) {

packages/wdio-allure-reporter/src/utils.js

@@ -1,5 +1,7 @@
 import process from 'process'
+import CompoundError from './compoundError'
 import { testStatuses, mochaEachHooks } from './constants'
+
 /**
  * Get allure test status by TestStat object
  * @param test {Object} - TestStat object
@@ -44,3 +46,16 @@ export const isMochaEachHooks = title => mochaEachHooks.some(hook => title.inclu
 export const tellReporter = (event, msg = {}) => {
     process.emit(event, msg)
 }
+
+/**
+ * Properly format error from different test runners
+ * @param {Object} test - TestStat object
+ * @returns {Object} - error object
+ * @private
+ */
+export const getErrorFromFailedTest = (test) => {
+    if (test.errors && Array.isArray(test.errors)) {
+        return test.errors.length === 1 ? test.errors[0] : new CompoundError(...test.errors)
+    }
+    return test.error
+}

packages/wdio-allure-reporter/tests/__fixtures__/testState.js

@@ -32,6 +32,21 @@ export function testFailed() {
     return Object.assign(testState(), { error, state: 'failed', end: '2018-05-14T15:17:21.631Z', _duration: 2730 })
 }
 
+export function testFailedWithMultipleErrors() {
+    const errors =
+    [
+        {
+            message: 'ReferenceError: All is Dust',
+            stack: 'ReferenceError: All is Dust'
+        },
+        {
+            message: 'InternalError: Abandon Hope',
+            stack: 'InternalError: Abandon Hope'
+        }
+    ]
+    return Object.assign(testState(), { errors, state: 'failed', end: '2018-05-14T15:17:21.631Z', _duration: 2730 })
+}
+
 export function testPending() {
     return Object.assign(testState(), { state: 'pending', end: '2018-05-14T15:17:21.631Z', _duration: 0 })
 }

packages/wdio-allure-reporter/tests/compoundError.test.js

@@ -0,0 +1,87 @@
+import CompoundError from '../src/compoundError'
+
+describe('CompoundError', () => {
+    let e1
+    let e2
+
+    beforeEach(() => {
+        try {
+            throw new Error('Everything is awful')
+        } catch (e) {
+            e1 = e
+        }
+        try {
+            throw new Error('I am so sad')
+        } catch (e) {
+            e2 = e
+        }
+    })
+
+    it('should have a message header', () => {
+        const compoundErr = new CompoundError(e1, e2)
+        const lines = compoundErr.message.split('\n')
+        expect(lines[0]).toBe('CompoundError: One or more errors occurred. ---')
+    })
+
+    it('should combine error messages from each error', () => {
+        const compoundErr = new CompoundError(e1, e2)
+        const lines = compoundErr.message.split('\n')
+        expect(lines).toContain('    Error: Everything is awful')
+        expect(lines).toContain('    Error: I am so sad')
+    })
+
+    it('should include stack traces from the errors', () => {
+        const compoundErr = new CompoundError(e1, e2)
+        const lines = compoundErr.message.split('\n').map(x => x.substr(4))
+
+        // This is a little dense, but essentially, CompoundError's messages look like
+        //
+        // IntroMessage
+        // EndOfStackMessage
+        // Seperator
+        // SecondStack
+        // EndOfStackMessage
+
+        // So we split both the final CompoundError message
+        // and the traces that compose it on line seperators and then test to make sure that
+        // the split traces are in the appropriate places in the CompoundError message.
+        // We do this rather than hardcoding strings, so we can use actual error stacks (which might be slightly)
+        // different depending on how we run the tests.
+
+        const e1split = e1.stack.split('\n')
+        const e2split = e2.stack.split('\n')
+        const startOfFirstStack = 1
+        const endOfFirstStack = e1split.length + startOfFirstStack
+        const startOfSecondStack = endOfFirstStack + 1
+        const endOfSecondStack = startOfSecondStack + e2split.length
+        expect(lines.slice(startOfFirstStack, endOfFirstStack)).toEqual(e1split)
+        expect(lines.slice(startOfSecondStack, endOfSecondStack)).toEqual(e2split)
+    })
+
+    it('should include delimiters to indicate where stack traces end', () => {
+        const compoundErr = new CompoundError(e1, e2)
+        const lines = compoundErr.message.split('\n')
+
+        expect(lines).toContain('--- End of stack trace ---')
+    })
+
+    it('should not explode if the stack property is undefined one an error', () => {
+        e1 = { message: 'goodbye' }
+        e2 = { message: 'hello' }
+
+        expect(() => new CompoundError(e1, e2)).not.toThrow()
+    })
+
+    it('should combine messages if stacks are not available for some reason', () => {
+        e1 = { message: 'goodbye' }
+        e2 = { message: 'hello' }
+        const error = new CompoundError(e1, e2)
+        const lines = error.message.split('\n')
+
+        expect(lines[0]).toBe('CompoundError: One or more errors occurred. ---')
+        expect(lines[1]).toBe('   goodbye')
+        expect(lines[2]).toBe('--- End of error message ---')
+        expect(lines[3]).toBe('   hello')
+        expect(lines[4]).toBe('--- End of error message ---')
+    })
+})

packages/wdio-allure-reporter/tests/suite.test.js

@@ -3,7 +3,7 @@ import AllureReporter from '../src/'
 import { clean, getResults } from './helper'
 import { runnerEnd, runnerStart } from './__fixtures__/runner'
 import { suiteEnd, suiteStart } from './__fixtures__/suite'
-import { testFailed, testPassed, testPending, testStart } from './__fixtures__/testState'
+import { testFailed, testPassed, testPending, testStart, testFailedWithMultipleErrors } from './__fixtures__/testState'
 import { commandStart, commandEnd, commandEndScreenShot, commandStartScreenShot } from './__fixtures__/command'
 
 let processOn
@@ -164,6 +164,33 @@ describe('Failed tests', () => {
         expect(allureXml('test-case').attr('status')).toEqual('failed')
     })
 
+    it('should detect failed test case with multiple errors', () => {
+        const reporter = new AllureReporter({ stdout: true, outputDir } )
+
+        const runnerEvent = runnerStart()
+        runnerEvent.config.framework = 'jasmine'
+        delete runnerEvent.config.capabilities.browserName
+        delete runnerEvent.config.capabilities.version
+
+        reporter.onRunnerStart(runnerEvent)
+        reporter.onSuiteStart(suiteStart())
+        reporter.onTestStart(testStart())
+        reporter.onTestFail(testFailedWithMultipleErrors())
+        reporter.onSuiteEnd(suiteEnd())
+        reporter.onRunnerEnd(runnerEnd())
+
+        const results = getResults(outputDir)
+        expect(results).toHaveLength(1)
+
+        allureXml = results[0]
+        expect(allureXml('test-case > name').text()).toEqual('should can do something')
+        expect(allureXml('test-case').attr('status')).toEqual('failed')
+        const message = allureXml('message').text()
+        const lines = message.split('\n')
+        expect(lines[0]).toBe('CompoundError: One or more errors occurred. ---')
+        expect(lines[1].trim()).toBe('ReferenceError: All is Dust')
+        expect(lines[3].trim()).toBe('InternalError: Abandon Hope')
+    })
 })
 
 describe('Pending tests', () => {

packages/wdio-allure-reporter/tests/utils.test.js

@@ -1,76 +1,118 @@
 import process from 'process'
-import { getTestStatus, isEmpty, tellReporter, isMochaEachHooks } from '../src/utils'
+import CompoundError from '../src/compoundError'
+import { getTestStatus, isEmpty, tellReporter, isMochaEachHooks, getErrorFromFailedTest } from '../src/utils'
 import { testStatuses } from '../src/constants'
 
-let processEmit
-beforeAll(() => {
-    processEmit = ::process.emit
-    process.emit = jest.fn()
-})
-
-afterAll(() => {
-    process.emit = processEmit
-})
-
-describe('utils#getTestStatus', () => {
-    it('return  status for jasmine', () => {
-        expect(getTestStatus({}, { framework: 'jasmine' })).toEqual(testStatuses.FAILED)
+describe('utils', () => {
+    let processEmit
+    beforeAll(() => {
+        processEmit = ::process.emit
+        process.emit = jest.fn()
     })
 
-    it('failed for AssertionError', () => {
-        const config = { framework: 'mocha' }
-        const test = { error: { name: 'AssertionError' } }
-        expect(getTestStatus(test, config)).toEqual(testStatuses.FAILED)
+    afterAll(() => {
+        process.emit = processEmit
     })
 
-    it('failed for AssertionError stacktrace', () => {
-        const config = { framework: 'mocha' }
-        const test = { error: { stack: 'AssertionError' } }
-        expect(getTestStatus(test, config)).toEqual(testStatuses.FAILED)
-    })
+    describe('getTestStatus', () => {
+        it('return  status for jasmine', () => {
+            expect(getTestStatus({}, { framework: 'jasmine' })).toEqual(testStatuses.FAILED)
+        })
 
-    it('broken for not AssertionError', () => {
-        const config = { framework: 'mocha' }
-        const test = { error: { name: 'MyError' } }
-        expect(getTestStatus(test, config)).toEqual(testStatuses.BROKEN)
-    })
+        it('failed for AssertionError', () => {
+            const config = { framework: 'mocha' }
+            const test = { error: { name: 'AssertionError' } }
+            expect(getTestStatus(test, config)).toEqual(testStatuses.FAILED)
+        })
+
+        it('failed for AssertionError stacktrace', () => {
+            const config = { framework: 'mocha' }
+            const test = { error: { stack: 'AssertionError' } }
+            expect(getTestStatus(test, config)).toEqual(testStatuses.FAILED)
+        })
 
-    it('failed status for not AssertionError stacktrace', () => {
-        const config = { framework: 'mocha' }
-        const test = { error: { stack: 'MyError stack trace' } }
-        expect(getTestStatus(test, config)).toEqual(testStatuses.BROKEN)
+        it('broken for not AssertionError', () => {
+            const config = { framework: 'mocha' }
+            const test = { error: { name: 'MyError' } }
+            expect(getTestStatus(test, config)).toEqual(testStatuses.BROKEN)
+        })
+
+        it('failed status for not AssertionError stacktrace', () => {
+            const config = { framework: 'mocha' }
+            const test = { error: { stack: 'MyError stack trace' } }
+            expect(getTestStatus(test, config)).toEqual(testStatuses.BROKEN)
+        })
     })
-})
 
-describe('utils', () => {
     it('isMochaEachHooks filter hook by title', () => {
         expect(isMochaEachHooks('"before all" hook')).toEqual(false)
         expect(isMochaEachHooks('"after all" hook')).toEqual(false)
         expect(isMochaEachHooks('"before each" hook')).toEqual(true)
         expect(isMochaEachHooks('"after each" hook')).toEqual(true)
     })
 
-    it('isEmpty filter empty objects', () => {
-        expect(isEmpty({})).toEqual(true)
-        expect(isEmpty([])).toEqual(true)
-        expect(isEmpty(undefined)).toEqual(true)
-        expect(isEmpty(null)).toEqual(true)
-        expect(isEmpty('')).toEqual(true)
+    describe('isEmpty', () => {
+        it('should filter empty objects', () => {
+            expect(isEmpty({})).toEqual(true)
+            expect(isEmpty([])).toEqual(true)
+            expect(isEmpty(undefined)).toEqual(true)
+            expect(isEmpty(null)).toEqual(true)
+            expect(isEmpty('')).toEqual(true)
+        })
     })
-})
 
-describe('utils#tellReporter', () => {
-    afterEach(() => {
-        process.emit.mockClear()
+    describe('isMochaHooks', () => {
+        it('should filter hook by title', () => {
+            expect(isMochaEachHooks('"before all" hook')).toEqual(false)
+            expect(isMochaEachHooks('"after all" hook')).toEqual(false)
+            expect(isMochaEachHooks('"before each" hook')).toEqual(true)
+            expect(isMochaEachHooks('"after each" hook')).toEqual(true)
+        })
     })
-    it('should accept message', () => {
-        tellReporter('foo', { bar: 'baz' })
-        expect(process.emit).toHaveBeenCalledTimes(1)
-        expect(process.emit).toHaveBeenCalledWith('foo', { bar: 'baz' })
+
+    describe('tellReporter', () => {
+        afterEach(() => {
+            process.emit.mockClear()
+        })
+
+        it('should accept message', () => {
+            tellReporter('foo', { bar: 'baz' })
+            expect(process.emit).toHaveBeenCalledTimes(1)
+            expect(process.emit).toHaveBeenCalledWith('foo', { bar: 'baz' })
+        })
+
+        it('should accept no message', () => {
+            tellReporter('foo')
+            expect(process.emit).toHaveBeenCalledTimes(1)
+            expect(process.emit).toHaveBeenCalledWith('foo', {})
+        })
     })
-    it('should accept no message', () => {
-        tellReporter('foo')
-        expect(process.emit).toHaveBeenCalledTimes(1)
-        expect(process.emit).toHaveBeenCalledWith('foo', {})
+
+    describe('getErrorFromFailedTest', () => {
+        // wdio-mocha-framework returns a single 'error', while wdio-jasmine-framework returns an array of 'errors'
+        it('should return just the error property when there is no errors property', () => {
+            const testStat = {
+                error: new Error('Everything is Broken Forever')
+            }
+            expect(getErrorFromFailedTest(testStat).message).toBe('Everything is Broken Forever')
+        })
+
+        it('should return a single error when there is an errors array with one error', () => {
+            const testStat = {
+                errors: [new Error('Everything is Broken Forever')],
+                error: new Error('Everything is Broken Forever')
+            }
+            expect(getErrorFromFailedTest(testStat).message).toBe('Everything is Broken Forever')
+        })
+
+        it('should return a CompoundError of the errors when there is more than one error', () => {
+            const testStat = {
+                errors: [new Error('Everything is Broken Forever'), new Error('Additional things are broken')],
+                error: new Error('Everything is Broken Forever')
+            }
+            expect(getErrorFromFailedTest(testStat) instanceof CompoundError).toBe(true)
+            expect(getErrorFromFailedTest(testStat).innerErrors).toEqual(testStat.errors)
+        })
     })
 })
+