Run failing tests to confirm their status

jsdom · Jun 12, 2018 · a5a7785 · a5a7785
1 parent a9d590a
commit a5a7785
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 13 deletions.
diff --git a/test/web-platform-tests/run-single-wpt.js b/test/web-platform-tests/run-single-wpt.js
@@ -15,7 +15,7 @@ module.exports = urlPrefixFactory => {
     };
   }
 
-  return (testPath, title = testPath) => {
+  return (testPath, title = testPath, expectFail) => {
     specify({
       title,
       expectPromise: true,
@@ -24,13 +24,13 @@ module.exports = urlPrefixFactory => {
       slow: 10000,
       skipIfBrowser: true,
       fn() {
-        return createJSDOM(urlPrefixFactory(), testPath);
+        return createJSDOM(urlPrefixFactory(), testPath, expectFail);
       }
     });
   };
 };
 
-function createJSDOM(urlPrefix, testPath) {
+function createJSDOM(urlPrefix, testPath, expectFail) {
   const reporterPathname = "/resources/testharnessreport.js";
   const unhandledExceptions = [];
   const doneErrors = [];
@@ -42,7 +42,13 @@ function createJSDOM(urlPrefix, testPath) {
     virtualConsole.on("jsdomError", e => {
       if (e.type === "unhandled exception" && !allowUnhandledExceptions) {
         unhandledExceptions.push(e);
-        console.error(e.detail.stack);
+
+        // Some failing tests make a lot of noise.
+        // There's no need to log these messages
+        // for errors we're already aware of.
+        if (!expectFail) {
+          console.error(e.detail.stack);
+        }
       }
     });
 
@@ -118,9 +124,16 @@ function createJSDOM(urlPrefix, testPath) {
           errors.push(...doneErrors);
           errors.push(...unhandledExceptions);
 
-          if (errors.length === 1) {
+          if (errors.length === 0 && expectFail) {
+            reject(new Error(`
+            Hey, did you fix a bug? This test used to be failing, but during
+            this run there were no errors. If you have fixed the issue covered
+            by this test, you can edit the "to-run.yaml" file and remove the line
+            containing this test. Thanks!
+            `));
+          } else if (errors.length === 1 && !expectFail) {
             reject(new Error(errors[0]));
-          } else if (errors.length) {
+          } else if (errors.length && !expectFail) {
             reject(new Error(`${errors.length} errors in test:\n\n${errors.join("\n")}`));
           } else {
             resolve();

diff --git a/test/web-platform-tests/run-wpts.js b/test/web-platform-tests/run-wpts.js
@@ -14,7 +14,8 @@ const validReasons = new Set([
   "mutates-globals",
   "needs-await",
   "needs-node8",
-  "fails-node10"
+  "fails-node10",
+  "timeout-node6" // For tests that timeout in Node.js v6, but pass in later versions
 ]);
 
 let supportsAwait = true;
@@ -25,7 +26,7 @@ try {
 }
 
 const hasNode8 = Number(process.versions.node.split(".")[0]) >= 8;
-const notNode10 = Number(process.versions.node.split(".")[0]) !== 10;
+const isNode10 = Number(process.versions.node.split(".")[0]) === 10;
 
 const manifestFilename = path.resolve(__dirname, "wpt-manifest.json");
 const manifest = readManifest(manifestFilename);
@@ -59,13 +60,17 @@ describe("web-platform-tests", () => {
 
           const testFile = stripPrefix(testFilePath, toRunDoc.DIR + "/");
           const reason = matchingPattern && toRunDoc[matchingPattern][0];
-          const shouldRunAnyway = (reason === "needs-await" && supportsAwait) ||
-                                  (reason === "needs-node8" && hasNode8) ||
-                                  (reason === "fails-node10" && notNode10);
-          if (matchingPattern && !shouldRunAnyway) {
+          const shouldSkip = ["timeout", "flaky", "mutates-globals"].includes(reason) ||
+                             (reason === "timeout-node6" && !hasNode8);
+          const expectFail = (reason === "fail") ||
+                             (reason === "needs-await" && !supportsAwait) ||
+                             (reason === "needs-node8" && !hasNode8) ||
+                             (reason === "fails-node10" && isNode10);
+
+          if (matchingPattern && shouldSkip) {
             specify.skip(`[${reason}] ${testFile}`);
           } else {
-            runSingleWPT(testFilePath, testFile);
+            runSingleWPT(testFilePath, testFile, expectFail);
           }
         }
       }