fix(middleware/runner): handle file list rejections (#3400)

Add error handling for rejections on the file list methods in the `runner` middleware. As discussed in #3396 it does this by handling an error the same way an error in a run is handled.

Fixes #3396

lib/executor.js

@@ -9,6 +9,7 @@ class Executor {
     this.emitter = emitter
 
     this.executionScheduled = false
+    this.errorsScheduled = []
     this.pendingCount = 0
     this.runningBrowsers = null
 
@@ -37,10 +38,42 @@ class Executor {
     }
   }
 
+  /**
+   * Schedule an error to be reported
+   * @param {string} errorMessage
+   * @returns {boolean} a boolean indicating whether or not the error was handled synchronously
+   */
+  scheduleError (errorMessage) {
+    // We don't want to interfere with any running test.
+    // Verify that no test is running before reporting the error.
+    if (this.capturedBrowsers.areAllReady()) {
+      log.warn(errorMessage)
+      const errorResult = {
+        success: 0,
+        failed: 0,
+        skipped: 0,
+        error: errorMessage,
+        exitCode: 1
+      }
+      const noBrowsersStartedTests = []
+      this.emitter.emit('run_start', noBrowsersStartedTests) // A run cannot complete without being started
+      this.emitter.emit('run_complete', noBrowsersStartedTests, errorResult)
+      return true
+    } else {
+      this.errorsScheduled.push(errorMessage)
+      return false
+    }
+  }
+
   onRunComplete () {
     if (this.executionScheduled) {
       this.schedule()
     }
+    if (this.errorsScheduled.length) {
+      const errorsToReport = this.errorsScheduled
+      this.errorsScheduled = []
+      errorsToReport.forEach((error) => this.scheduleError(error))
+    }
   }
 
   onBrowserComplete () {

lib/middleware/runner.js

@@ -35,19 +35,18 @@ function createRunnerMiddleware (emitter, fileList, capturedBrowsers, reporter,
       }
 
       const data = request.body
-      emitter.once('run_start', function () {
-        const responseWrite = response.write.bind(response)
-        responseWrite.colors = data.colors
-        reporter.addAdapter(responseWrite)
 
-        // clean up, close runner response
-        emitter.once('run_complete', function (browsers, results) {
-          reporter.removeAdapter(responseWrite)
-          const emptyTestSuite = (results.failed + results.success) === 0 ? 0 : 1
-          response.end(constant.EXIT_CODE + emptyTestSuite + results.exitCode)
-        })
+      updateClientArgs(data)
+      handleRun(data)
+      refreshFileList(data).then(() => {
+        executor.schedule()
+      }).catch((error) => {
+        const errorMessage = `Error during refresh file list. ${error.stack || error}`
+        executor.scheduleError(errorMessage)
       })
+    })
 
+    function updateClientArgs (data) {
       helper.restoreOriginalArgs(config)
       if (_.isEmpty(data.args)) {
         log.debug('Ignoring empty client.args from run command')
@@ -59,43 +58,52 @@ function createRunnerMiddleware (emitter, fileList, capturedBrowsers, reporter,
         log.warn('Replacing client.args with ', data.args, ' as their types do not match.')
         config.client.args = data.args
       }
+    }
 
+    async function refreshFileList (data) {
       let fullRefresh = true
 
       if (helper.isArray(data.changedFiles)) {
-        data.changedFiles.forEach(function (filepath) {
-          fileList.changeFile(path.resolve(config.basePath, filepath))
+        await Promise.all(data.changedFiles.map(async function (filepath) {
+          await fileList.changeFile(path.resolve(config.basePath, filepath))
           fullRefresh = false
-        })
+        }))
       }
 
       if (helper.isArray(data.addedFiles)) {
-        data.addedFiles.forEach(function (filepath) {
-          fileList.addFile(path.resolve(config.basePath, filepath))
+        await Promise.all(data.addedFiles.map(async function (filepath) {
+          await fileList.addFile(path.resolve(config.basePath, filepath))
           fullRefresh = false
-        })
+        }))
       }
 
       if (helper.isArray(data.removedFiles)) {
-        data.removedFiles.forEach(function (filepath) {
-          fileList.removeFile(path.resolve(config.basePath, filepath))
+        await Promise.all(data.removedFiles.map(async function (filepath) {
+          await fileList.removeFile(path.resolve(config.basePath, filepath))
           fullRefresh = false
-        })
+        }))
       }
 
       if (fullRefresh && data.refresh !== false) {
         log.debug('Refreshing all the files / patterns')
-        fileList.refresh().then(function () {
-          // Wait for the file list refresh to complete before starting test run,
-          // otherwise the context.html generation might not see new/updated files.
-          if (!config.autoWatch) {
-            executor.schedule()
-          }
-        })
-      } else {
-        executor.schedule()
+        await fileList.refresh()
       }
-    })
+    }
+
+    function handleRun (data) {
+      emitter.once('run_start', function () {
+        const responseWrite = response.write.bind(response)
+        responseWrite.colors = data.colors
+        reporter.addAdapter(responseWrite)
+
+        // clean up, close runner response
+        emitter.once('run_complete', function (_browsers, results) {
+          reporter.removeAdapter(responseWrite)
+          const emptyTestSuite = (results.failed + results.success) === 0 ? 0 : 1
+          response.end(constant.EXIT_CODE + emptyTestSuite + results.exitCode)
+        })
+      })
+    }
   }
 }
 

test/unit/executor.spec.js

@@ -5,6 +5,8 @@ const BrowserCollection = require('../../lib/browser_collection')
 const EventEmitter = require('../../lib/events').EventEmitter
 const Executor = require('../../lib/executor')
 
+const log = require('../../lib/logger').create()
+
 describe('executor', () => {
   let emitter
   let capturedBrowsers
@@ -21,36 +23,85 @@ describe('executor', () => {
     executor.socketIoSockets = new EventEmitter()
 
     spy = {
-      onRunStart: () => null,
-      onSocketsExecute: () => null
+      onRunStart: sinon.stub(),
+      onSocketsExecute: sinon.stub(),
+      onRunComplete: sinon.stub()
     }
-
-    sinon.spy(spy, 'onRunStart')
-    sinon.spy(spy, 'onSocketsExecute')
+    sinon.stub(log, 'warn')
 
     emitter.on('run_start', spy.onRunStart)
+    emitter.on('run_complete', spy.onRunComplete)
     executor.socketIoSockets.on('execute', spy.onSocketsExecute)
   })
 
-  it('should start the run and pass client config', () => {
-    capturedBrowsers.areAllReady = () => true
+  describe('schedule', () => {
+    it('should start the run and pass client config', () => {
+      capturedBrowsers.areAllReady = () => true
+
+      executor.schedule()
+      expect(spy.onRunStart).to.have.been.called
+      expect(spy.onSocketsExecute).to.have.been.calledWith(config.client)
+    })
+
+    it('should wait for all browsers to finish', () => {
+      capturedBrowsers.areAllReady = () => false
 
-    executor.schedule()
-    expect(spy.onRunStart).to.have.been.called
-    expect(spy.onSocketsExecute).to.have.been.calledWith(config.client)
+      // they are not ready yet
+      executor.schedule()
+      expect(spy.onRunStart).not.to.have.been.called
+      expect(spy.onSocketsExecute).not.to.have.been.called
+
+      capturedBrowsers.areAllReady = () => true
+      emitter.emit('run_complete')
+      expect(spy.onRunStart).to.have.been.called
+      expect(spy.onSocketsExecute).to.have.been.called
+    })
   })
 
-  it('should wait for all browsers to finish', () => {
-    capturedBrowsers.areAllReady = () => false
+  describe('scheduleError', () => {
+    it('should return `true` if scheduled synchronously', () => {
+      const result = executor.scheduleError('expected error')
+      expect(result).to.be.true
+    })
+
+    it('should emit both "run_start" and "run_complete"', () => {
+      executor.scheduleError('expected error')
+      expect(spy.onRunStart).to.have.been.called
+      expect(spy.onRunComplete).to.have.been.called
+      expect(spy.onRunStart).to.have.been.calledBefore(spy.onRunComplete)
+    })
+
+    it('should report the error', () => {
+      const expectedError = 'expected error'
+      executor.scheduleError(expectedError)
+      expect(spy.onRunComplete).to.have.been.calledWith([], {
+        success: 0,
+        failed: 0,
+        skipped: 0,
+        error: expectedError,
+        exitCode: 1
+      })
+    })
+
+    it('should wait for scheduled runs to end before reporting the error', () => {
+      // Arrange
+      let browsersAreReady = true
+      const expectedError = 'expected error'
+      capturedBrowsers.areAllReady = () => browsersAreReady
+      executor.schedule()
+      browsersAreReady = false
 
-    // they are not ready yet
-    executor.schedule()
-    expect(spy.onRunStart).not.to.have.been.called
-    expect(spy.onSocketsExecute).not.to.have.been.called
+      // Act
+      const result = executor.scheduleError(expectedError)
+      browsersAreReady = true
 
-    capturedBrowsers.areAllReady = () => true
-    emitter.emit('run_complete')
-    expect(spy.onRunStart).to.have.been.called
-    expect(spy.onSocketsExecute).to.have.been.called
+      // Assert
+      expect(result).to.be.false
+      expect(spy.onRunComplete).to.not.have.been.called
+      emitter.emit('run_complete')
+      expect(spy.onRunComplete).to.have.been.calledWith([], sinon.match({
+        error: expectedError
+      }))
+    })
   })
 })