diff --git a/test/cli/multiple-configs/_config.js b/test/cli/multiple-configs/_config.js
index 1a2122b4d47..844064e6cd1 100644
--- a/test/cli/multiple-configs/_config.js
+++ b/test/cli/multiple-configs/_config.js
@@ -1,4 +1,10 @@
+const assert = require('assert');
+
 module.exports = {
 	description: 'generates output file when multiple configurations are specified and one build fails',
-	command: 'rollup -c'
+	command: 'rollup -c',
+	error: err => {
+		assert.ok(/Unexpected Exception/.test(err.message));
+		return true;
+	}
 };
diff --git a/test/test.js b/test/test.js
index cddd9d540f4..6441ce28541 100644
--- a/test/test.js
+++ b/test/test.js
@@ -543,8 +543,8 @@ describe( 'rollup', function () {
 					exec( command, {}, ( err, code, stderr ) => {
 						if ( err ) {
 							if ( config.error ) {
-								config.error( err );
-								return done();
+								const shouldContinue = config.error( err );
+								if (!shouldContinue) return done();
 							} else {
 								throw err;
 							}