From 7195f0d8cfb90d762c2dc5d1dbd02e373d4c813f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Ribaudo?= Date: Mon, 16 Sep 2019 00:07:10 +0200 Subject: [PATCH] Unify logic for running parser tests from external suites (#10444) --- Makefile | 8 +- package.json | 1 + scripts/tests/flow/index.js | 132 ++++++++ .../tests/flow/run_babel_parser_flow_tests.js | 311 ------------------ ...flow_tests_whitelist.txt => whitelist.txt} | 25 +- scripts/tests/test262/index.js | 193 +++++++++++ .../tests/test262/run_babel_parser_test262.js | 126 ------- .../test262/run_babel_parser_test262_utils.js | 297 ----------------- .../{test262_whitelist.txt => whitelist.txt} | 2 +- scripts/tests/utils/parser-test-runner.js | 237 +++++++++++++ yarn.lock | 5 + 11 files changed, 581 insertions(+), 756 deletions(-) create mode 100644 scripts/tests/flow/index.js delete mode 100644 scripts/tests/flow/run_babel_parser_flow_tests.js rename scripts/tests/flow/{flow_tests_whitelist.txt => whitelist.txt} (67%) create mode 100644 scripts/tests/test262/index.js delete mode 100644 scripts/tests/test262/run_babel_parser_test262.js delete mode 100644 scripts/tests/test262/run_babel_parser_test262_utils.js rename scripts/tests/test262/{test262_whitelist.txt => whitelist.txt} (99%) create mode 100644 scripts/tests/utils/parser-test-runner.js diff --git a/Makefile b/Makefile index bc69709bdd19..896425533d55 100644 --- a/Makefile +++ b/Makefile @@ -159,13 +159,13 @@ bootstrap-flow: cd build/flow && git checkout $(FLOW_COMMIT) test-flow: - node scripts/tests/flow/run_babel_parser_flow_tests.js + node scripts/tests/flow test-flow-ci: build-bundle-ci bootstrap-flow $(MAKE) test-flow test-flow-update-whitelist: - node scripts/tests/flow/run_babel_parser_flow_tests.js --update-whitelist + node scripts/tests/flow --update-whitelist bootstrap-test262: rm -rf build/test262 @@ -174,13 +174,13 @@ bootstrap-test262: cd build/test262 && git checkout $(TEST262_COMMIT) test-test262: - node scripts/tests/test262/run_babel_parser_test262.js + node scripts/tests/test262 test-test262-ci: build-bundle-ci bootstrap-test262 $(MAKE) test-test262 test-test262-update-whitelist: - node scripts/tests/test262/run_babel_parser_test262.js --update-whitelist + node scripts/tests/test262 --update-whitelist # Does not work on Windows clone-license: diff --git a/package.json b/package.json index a204c5e72859..efdd7916d469 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,7 @@ "lerna-changelog": "^0.5.0", "lint-staged": "^9.2.0", "lodash": "^4.17.13", + "mergeiterator": "^1.2.5", "output-file-sync": "^2.0.0", "prettier": "^1.19.1", "pump": "^3.0.0", diff --git a/scripts/tests/flow/index.js b/scripts/tests/flow/index.js new file mode 100644 index 000000000000..9b12bf07053e --- /dev/null +++ b/scripts/tests/flow/index.js @@ -0,0 +1,132 @@ +const fs = require("fs").promises; +const path = require("path"); +const merge = require("mergeiterator"); +const TestRunner = require("../utils/parser-test-runner"); + +const flowOptionsMapping = { + esproposal_class_instance_fields: "classProperties", + esproposal_class_static_fields: "classProperties", + esproposal_export_star_as: "exportNamespaceFrom", + esproposal_decorators: "decorators-legacy", + esproposal_nullish_coalescing: "nullishCoalescingOperator", + esproposal_optional_chaining: "optionalChaining", + types: "flowComments", + intern_comments: false, +}; + +function getPlugins(test) { + const plugins = [ + "dynamicImport", + ["flow", { all: true }], + "flowComments", + "jsx", + "classProperties", + "classPrivateProperties", + "classPrivateMethods", + "bigInt", + "numericSeparator", + ]; + + if (!test.options) return plugins; + + for (const [option, enabled] of Object.entries(test.options)) { + if (!enabled) { + const idx = plugins.indexOf(flowOptionsMapping[option]); + if (idx !== -1) plugins.splice(idx, 1); + } else if (!(option in flowOptionsMapping)) { + throw new Error("Parser options not mapped " + option); + } else if (flowOptionsMapping[option]) { + plugins.push(flowOptionsMapping[option]); + } + } + + return plugins; +} + +async function* readdirRecursive(root, dir = ".") { + const names = await fs.readdir(path.join(root, dir)); + + const dirs = []; + + for (const name of names) { + const file = path.join(dir, name); + const stats = await fs.stat(path.join(root, file)); + if (!stats.isDirectory()) { + if (!file) continue; + yield file; + } else { + dirs.push(readdirRecursive(root, file)); + } + } + + yield* merge(dirs); +} + +async function* loadTests(root) { + for await (const file of readdirRecursive(root)) { + if (file.slice(-3) === ".js") { + const noExt = path.join(root, file).slice(0, -3); + + const [contents, tree, options] = await Promise.all([ + fs.readFile(noExt + ".js", "utf8"), + fs.readFile(noExt + ".tree.json", "utf8").catch(() => null), + fs.readFile(noExt + ".options.json", "utf8").catch(() => null), + ]); + + yield { + file, + contents, + tree: JSON.parse(tree), + options: JSON.parse(options), + }; + } + } +} + +const runner = new TestRunner({ + testDir: path.join(__dirname, "../../../build/flow/src/parser/test/flow"), + whitelist: path.join(__dirname, "whitelist.txt"), + shouldUpdate: process.argv.includes("--update-whitelist"), + + async *getTests() { + for await (const test of loadTests(this.testDir)) { + const shouldSuccess = + test.tree && (!test.tree.errors || !test.tree.errors.length); + + yield { + contents: test.contents, + fileName: test.file, + id: test.file, + expectedError: !shouldSuccess, + plugins: getPlugins(test), + }; + } + }, + + parse(test, parser) { + try { + parser(test.contents, { + sourceType: "module", + plugins: test.plugins, + }); + } catch (e) { + // lets retry in script mode + if (!test.expectedError) { + try { + parser(test.contents, { + sourceType: "script", + plugins: test.plugins, + }); + return; + } catch {} + } + + throw e; + } + }, +}); + +runner.run().catch(err => { + console.error(err); + process.exitCode = 1; +}); diff --git a/scripts/tests/flow/run_babel_parser_flow_tests.js b/scripts/tests/flow/run_babel_parser_flow_tests.js deleted file mode 100644 index f087fb6614e3..000000000000 --- a/scripts/tests/flow/run_babel_parser_flow_tests.js +++ /dev/null @@ -1,311 +0,0 @@ -"use strict"; - -const path = require("path"); -const fs = require("fs"); -const chalk = require("chalk"); -const parse = require("../../../packages/babel-parser").parse; - -const TESTS_FOLDER = path.join( - __dirname, - "../../../build/flow/src/parser/test/flow" -); -const WHITELIST_PATH = path.join(__dirname, "./flow_tests_whitelist.txt"); - -const shouldUpdateWhitelist = process.argv.indexOf("--update-whitelist") > 0; - -function map_get_default(map, key, defaultConstructor) { - if (map.has(key)) { - return map.get(key); - } - const value = new defaultConstructor(); - map.set(key, value); - return value; -} - -function get_whitelist(filename) { - return fs - .readFileSync(filename, "utf8") - .split("\n") - .map(line => line.replace(/#.*$/, "").trim()) - .filter(Boolean); -} - -function list_files(root, dir) { - const files = fs.readdirSync(dir ? path.join(root, dir) : root); - let result = []; - for (let i = 0; i < files.length; i++) { - const file = dir ? path.join(dir, files[i]) : files[i]; - const stats = fs.statSync(path.join(root, file)); - if (stats.isDirectory()) { - result = result.concat(list_files(root, file)); - } else { - result.push(file); - } - } - return result.sort(); -} - -function get_tests(root_dir) { - const files = list_files(root_dir); - const tests = new Map(); - for (let i = 0; i < files.length; i++) { - const file = files[i]; - const test_name = path.dirname(file); - const case_parts = path.basename(file).split("."); - const case_name = case_parts[0]; - - // Hack to ignore hidden files. - if (case_name === "") { - continue; - } - - const cases = map_get_default(tests, test_name, Map); - const case_ = map_get_default(cases, case_name, Object); - - const content = fs.readFileSync(path.join(root_dir, file), "utf8"); - const ext = case_parts[case_parts.length - 1]; - const kind = - case_parts.length > 2 ? case_parts[case_parts.length - 2] : null; - - if (ext === "js") { - case_.file = file; - case_.content = content; - } else if (ext === "json" && kind === "tree") { - case_.expected_ast = JSON.parse(content); - } else if (ext === "json" && kind === "options") { - case_.options = JSON.parse(content); - } - } - return tests; -} - -function update_whitelist(summary) { - const contains = (tests, file) => - tests.some(({ test }) => test.file === file); - - const disallowed = summary.disallowed.success.concat( - summary.disallowed.failure - ); - - const oldLines = fs - .readFileSync(WHITELIST_PATH, "utf8") - .trim() - .split("\n") - .filter(line => { - const file = line.replace(/#.*$/, "").trim(); - return ( - !contains(disallowed, file) && summary.unrecognized.indexOf(file) === -1 - ); - }); - - const newLines = summary.disallowed.failure - .map(({ test }) => test.file) - .filter(test => oldLines.indexOf(test) === -1); - - const result = oldLines.concat(newLines).join("\n") + "\n"; - - fs.writeFileSync(WHITELIST_PATH, result); -} - -const options = { - plugins: [ - "dynamicImport", - ["flow", { all: true }], - "flowComments", - "jsx", - "classProperties", - "classPrivateProperties", - "classPrivateMethods", - "bigInt", - "numericSeparator", - ], - sourceType: "module", - ranges: true, -}; - -const flowOptionsMapping = { - esproposal_class_instance_fields: "classProperties", - esproposal_class_static_fields: "classProperties", - esproposal_export_star_as: "exportNamespaceFrom", - esproposal_decorators: "decorators-legacy", - esproposal_nullish_coalescing: "nullishCoalescingOperator", - esproposal_optional_chaining: "optionalChaining", - types: "flowComments", - intern_comments: false, -}; - -const summary = { - passed: true, - allowed: { - success: [], - failure: [], - }, - disallowed: { - success: [], - failure: [], - }, - unrecognized: [], -}; - -const tests = get_tests(TESTS_FOLDER); -const whitelist = get_whitelist(WHITELIST_PATH); - -const unrecognized = new Set(whitelist); - -tests.forEach(section => { - section.forEach(test => { - const shouldSuccess = - test.expected_ast && - (!Array.isArray(test.expected_ast.errors) || - test.expected_ast.errors.length === 0); - const inWhitelist = whitelist.indexOf(test.file) > -1; - - const babelParserOptions = Object.assign({}, options); - babelParserOptions.plugins = babelParserOptions.plugins.slice(); - - if (test.options) { - Object.keys(test.options).forEach(option => { - if (!test.options[option]) { - const idx = babelParserOptions.plugins.indexOf( - flowOptionsMapping[option] - ); - if (idx) { - babelParserOptions.plugins.splice(idx, 1); - } - return; - } - if (!(option in flowOptionsMapping)) { - throw new Error("Parser options not mapped " + option); - } - if (flowOptionsMapping[option]) { - babelParserOptions.plugins.push(flowOptionsMapping[option]); - } - }); - } - - let failed = false; - let exception = null; - try { - parse(test.content, babelParserOptions); - } catch (e) { - exception = e; - failed = true; - - // lets retry in script mode - if (shouldSuccess) { - try { - parse( - test.content, - Object.assign({}, babelParserOptions, { sourceType: "script" }) - ); - exception = null; - failed = false; - } catch (e) {} - } - } - - const isSuccess = shouldSuccess !== failed; - const isAllowed = isSuccess !== inWhitelist; - - summary[isAllowed ? "allowed" : "disallowed"][ - isSuccess ? "success" : "failure" - ].push({ test, exception, shouldSuccess, babelParserOptions }); - summary.passed &= isAllowed; - - unrecognized.delete(test.file); - - process.stdout.write(chalk.gray(".")); - }); -}); - -summary.unrecognized = Array.from(unrecognized); -summary.passed &= summary.unrecognized.length === 0; - -// This is needed because, after the dots written using -// `process.stdout.write(".")` there is no final newline -console.log(); - -if (summary.disallowed.failure.length || summary.disallowed.success.length) { - console.log("\n-- FAILED TESTS --"); - summary.disallowed.failure.forEach( - ({ test, shouldSuccess, exception, babelParserOptions }) => { - console.log(chalk.red(`✘ ${test.file}`)); - if (shouldSuccess) { - console.log(chalk.yellow(" Should parse successfully, but did not")); - console.log(chalk.yellow(` Failed with: \`${exception.message}\``)); - } else { - console.log(chalk.yellow(" Should fail parsing, but did not")); - } - console.log( - chalk.yellow( - ` Active plugins: ${JSON.stringify(babelParserOptions.plugins)}` - ) - ); - } - ); - summary.disallowed.success.forEach( - ({ test, shouldSuccess, babelParserOptions }) => { - console.log(chalk.red(`✘ ${test.file}`)); - if (shouldSuccess) { - console.log( - chalk.yellow( - " Correctly parsed successfully, but" + - " was disallowed by the whitelist" - ) - ); - } else { - console.log( - chalk.yellow( - " Correctly failed parsing, but" + - " was disallowed by the whitelist" - ) - ); - } - console.log( - chalk.yellow( - ` Active plugins: ${JSON.stringify(babelParserOptions.plugins)}` - ) - ); - } - ); -} - -console.log("-- SUMMARY --"); -console.log( - chalk.green("✔ " + summary.allowed.success.length + " tests passed") -); -console.log( - chalk.green( - "✔ " + - summary.allowed.failure.length + - " tests failed but were allowed in the whitelist" - ) -); -console.log( - chalk.red("✘ " + summary.disallowed.failure.length + " tests failed") -); -console.log( - chalk.red( - "✘ " + - summary.disallowed.success.length + - " tests passed but were disallowed in the whitelist" - ) -); -console.log( - chalk.red( - "✘ " + - summary.unrecognized.length + - " tests specified in the whitelist were not found" - ) -); - -// Some padding to separate the output from the message `make` -// adds at the end of failing scripts -console.log(); - -if (shouldUpdateWhitelist) { - update_whitelist(summary); - console.log("\nWhitelist updated"); -} else { - process.exit(summary.passed ? 0 : 1); -} diff --git a/scripts/tests/flow/flow_tests_whitelist.txt b/scripts/tests/flow/whitelist.txt similarity index 67% rename from scripts/tests/flow/flow_tests_whitelist.txt rename to scripts/tests/flow/whitelist.txt index f3704b19d867..8e24ebf0a28d 100644 --- a/scripts/tests/flow/flow_tests_whitelist.txt +++ b/scripts/tests/flow/whitelist.txt @@ -1,13 +1,4 @@ -# This file lists tests that are known to produce incorrect results when parsed -# with the babel parser: -# -# - Tests that are expected to parse successfully but for which the babel parser reports -# a syntax error -# - Tests that contain invalid syntax but for which the babel parser reports no syntax -# error -# -# Entries should be removed incrementally as the babel parser is improved. - +ES6/modules/migrated_0020.js JSX_invalid/migrated_0000.js arrow_function_invalid/migrated_0002.js async_await/migrated_0007.js @@ -15,18 +6,18 @@ async_await/migrated_0020.js async_await/migrated_0024.js async_await/migrated_0027.js async_generators/migrated_0007.js +class_method_kinds/polymorphic_getter.js class_properties/migrated_0021.js class_properties/migrated_0026.js decorators/migrated_0003.js -private_class_properties/multiple.js +export_import_reserved_words/migrated_0003.js +export_statements/export_trailing_comma.js +nullish_coalescing/precedence_and.js +nullish_coalescing/precedence_or.js private_class_properties/getter_and_field.js private_class_properties/getter_duplicate.js +private_class_properties/multiple.js +private_class_properties/multiple.js private_class_properties/setter_and_field.js private_class_properties/setter_duplicate.js types/member/reserved_words.js -class_method_kinds/polymorphic_getter.js -ES6/modules/migrated_0020.js -export_import_reserved_words/migrated_0003.js -export_statements/export_trailing_comma.js -nullish_coalescing/precedence_and.js -nullish_coalescing/precedence_or.js diff --git a/scripts/tests/test262/index.js b/scripts/tests/test262/index.js new file mode 100644 index 000000000000..0fd8eb229c42 --- /dev/null +++ b/scripts/tests/test262/index.js @@ -0,0 +1,193 @@ +const path = require("path"); +const TestStream = require("test262-stream"); +const TestRunner = require("../utils/parser-test-runner"); + +const ignoredFeatures = [ + "Array.prototype.flat", + "Array.prototype.flatMap", + "Array.prototype.values", + "ArrayBuffer", + "async-functions", + "async-iteration", + "arrow-function", + "Atomics", + "caller", + "class", + "computed-property-names", + "const", + "cross-realm", + "DataView", + "DataView.prototype.getFloat32", + "DataView.prototype.getFloat64", + "DataView.prototype.getInt8", + "DataView.prototype.getInt16", + "DataView.prototype.getInt32", + "DataView.prototype.getUint16", + "DataView.prototype.getUint32", + "DataView.prototype.setUint8", + "default-parameters", + "destructuring-assignment", + "destructuring-binding", + "FinalizationGroup", + "Float32Array", + "Float64Array", + "for-of", + "generators", + "globalThis", + "hashbang", + "host-gc-required", + "Int8Array", + "Int32Array", + "Intl.DateTimeFormat-datetimestyle", + "Intl.DateTimeFormat-dayPeriod", + "Intl.DateTimeFormat-fractionalSecondDigits", + "Intl.DateTimeFormat-formatRange", + "Intl.ListFormat", + "Intl.Locale", + "Intl.NumberFormat-unified", + "Intl.RelativeTimeFormat", + "Intl.Segmenter", + "IsHTMLDDA", + "json-superset", + "let", + "Map", + "new.target", + "Object.fromEntries", + "Object.is", + "object-rest", + "object-spread", + "optional-catch-binding", + "Promise.allSettled", + "Promise.prototype.finally", + "Proxy", + "proxy-missing-checks", + "Reflect", + "Reflect.construct", + "Reflect.set", + "Reflect.setPrototypeOf", + "regexp-dotall", + "regexp-lookbehind", + "regexp-named-groups", + "regexp-unicode-property-escapes", + "rest-parameters", + "SharedArrayBuffer", + "Set", + "String.fromCodePoint", + "String.prototype.endsWith", + "String.prototype.includes", + "String.prototype.matchAll", + "String.prototype.trimEnd", + "String.prototype.trimStart", + "string-trimming", + "super", + "Symbol", + "Symbol.asyncIterator", + "Symbol.hasInstance", + "Symbol.isConcatSpreadable", + "Symbol.iterator", + "Symbol.match", + "Symbol.matchAll", + "Symbol.prototype.description", + "Symbol.replace", + "Symbol.search", + "Symbol.split", + "Symbol.species", + "Symbol.toPrimitive", + "Symbol.toStringTag", + "Symbol.unscopables", + "tail-call-optimization", + "template", + "TypedArray", + "u180e", + "Uint8Array", + "Uint8ClampedArray", + "Uint16Array", + "WeakMap", + "WeakSet", + "WeakRef", + "well-formed-json-stringify", +]; + +const ignoredTests = ["built-ins/RegExp/", "language/literals/regexp/"]; + +const featuresToPlugins = { + BigInt: "bigInt", + "class-fields-private": "classPrivateProperties", + "class-fields-public": "classProperties", + "class-methods-private": "classPrivateMethods", + "class-static-fields-public": "classProperties", + "class-static-fields-private": "classPrivateProperties", + "class-static-methods-private": "classPrivateMethods", + "dynamic-import": "dynamicImport", + "export-star-as-namespace-from-module": "exportNamespaceFrom", + "import.meta": "importMeta", + "numeric-separator-literal": "numericSeparator", + "optional-chaining": "optionalChaining", + "top-level-await": "topLevelAwait", +}; + +const unmappedFeatures = new Set(); + +function* getPlugins(features) { + if (!features) return; + + for (const f of features) { + if (featuresToPlugins[f]) { + yield featuresToPlugins[f]; + } else if (!ignoredFeatures.includes(f)) { + unmappedFeatures.add(f); + } + } +} + +const runner = new TestRunner({ + testDir: path.join(__dirname, "../../../build/test262"), + whitelist: path.join(__dirname, "whitelist.txt"), + logInterval: 500, + shouldUpdate: process.argv.includes("--update-whitelist"), + + async *getTests() { + const stream = new TestStream(this.testDir, { + omitRuntime: true, + }); + + for await (const test of stream) { + // strip test/ + const fileName = test.file.substr(5); + + if (ignoredTests.some(start => fileName.startsWith(start))) continue; + + yield { + contents: test.contents, + fileName, + id: `${fileName}(${test.scenario})`, + sourceType: test.attrs.flags.module ? "module" : "script", + plugins: Array.from(getPlugins(test.attrs.features)), + expectedError: + !!test.attrs.negative && + (test.attrs.negative.phase === "parse" || + test.attrs.negative.phase === "early"), + }; + } + }, +}); + +runner + .run() + .then(() => { + if (unmappedFeatures.size) { + console.log(""); + console.log( + "The following Features are not currently mapped or ignored:" + ); + console.log( + Array.from(unmappedFeatures) + .join("\n") + .replace(/^/gm, " ") + ); + } + }) + .catch(err => { + console.error(err); + process.exitCode = 1; + }); diff --git a/scripts/tests/test262/run_babel_parser_test262.js b/scripts/tests/test262/run_babel_parser_test262.js deleted file mode 100644 index acd0c5122606..000000000000 --- a/scripts/tests/test262/run_babel_parser_test262.js +++ /dev/null @@ -1,126 +0,0 @@ -"use strict"; - -const path = require("path"); -const chalk = require("chalk"); -const utils = require("./run_babel_parser_test262_utils"); - -const testDir = path.join(__dirname, "../../../build/test262"); -const whitelistFile = path.join(__dirname, "test262_whitelist.txt"); -const shouldUpdate = process.argv.indexOf("--update-whitelist") > -1; - -Promise.all([utils.getTests(testDir), utils.getWhitelist(whitelistFile)]) - .then(function([tests, whitelist]) { - const total = tests.length; - const reportInc = Math.floor(total / 20); - - console.log(`Now running ${total} tests...`); - - const results = tests.map(function(test, idx) { - if (idx % reportInc === 0) { - console.log(`> ${Math.round((100 * idx) / total)}% complete`); - } - - return utils.runTest(test); - }); - - return utils.interpret(results, whitelist); - }) - .then(function(summary) { - const goodnews = [ - summary.allowed.success.length + " valid programs parsed without error", - summary.allowed.failure.length + - " invalid programs produced a parsing error", - summary.allowed.falsePositive.length + - " invalid programs did not produce a parsing error" + - " (and allowed by the whitelist file)", - summary.allowed.falseNegative.length + - " valid programs produced a parsing error" + - " (and allowed by the whitelist file)", - ]; - const badnews = []; - const badnewsDetails = []; - - void [ - { - tests: summary.disallowed.success, - label: - "valid programs parsed without error" + - " (in violation of the whitelist file)", - }, - { - tests: summary.disallowed.failure, - label: - "invalid programs produced a parsing error" + - " (in violation of the whitelist file)", - }, - { - tests: summary.disallowed.falsePositive, - label: - "invalid programs did not produce a parsing error" + - " (without a corresponding entry in the whitelist file)", - }, - { - tests: summary.disallowed.falseNegative, - label: - "valid programs produced a parsing error" + - " (without a corresponding entry in the whitelist file)", - }, - { - tests: summary.unrecognized, - label: "non-existent programs specified in the whitelist file", - }, - ].forEach(function({ tests, label }) { - if (!tests.length) { - return; - } - - const desc = tests.length + " " + label; - - badnews.push(desc); - badnewsDetails.push(desc + ":"); - badnewsDetails.push( - ...tests.map(function(test) { - return test.id || test; - }) - ); - }); - - console.log("Testing complete."); - console.log("Summary:"); - console.log(chalk.green(goodnews.join("\n").replace(/^/gm, " ✔ "))); - - if (!summary.passed) { - console.log(""); - console.log(chalk.red(badnews.join("\n").replace(/^/gm, " ✘ "))); - console.log(""); - console.log("Details:"); - console.log(badnewsDetails.join("\n").replace(/^/gm, " ")); - } - - if (shouldUpdate) { - return utils.updateWhitelist(whitelistFile, summary).then(function() { - console.log(""); - console.log("Whitelist file updated."); - }); - } else { - process.exitCode = summary.passed ? 0 : 1; - } - - const unmappedFeatures = utils.getUnmappedFeatures(); - - if (unmappedFeatures.size) { - console.log(""); - console.log( - "The following Features are not currently mapped or ignored:" - ); - console.log( - Array.from(unmappedFeatures) - .join("\n") - .replace(/^/gm, " ") - ); - } - }) - .catch(function(err) { - console.error(err); - process.exitCode = 1; - }); diff --git a/scripts/tests/test262/run_babel_parser_test262_utils.js b/scripts/tests/test262/run_babel_parser_test262_utils.js deleted file mode 100644 index 2c35d0ed5933..000000000000 --- a/scripts/tests/test262/run_babel_parser_test262_utils.js +++ /dev/null @@ -1,297 +0,0 @@ -"use strict"; - -const fs = require("graceful-fs"); -const promisify = require("util").promisify; -const TestStream = require("test262-stream"); -const pfs = { - readFile: promisify(fs.readFile), - writeFile: promisify(fs.writeFile), - readdir: promisify(fs.readdir), - stat: promisify(fs.stat), -}; - -const parse = require("../../../packages/babel-parser").parse; - -const ignoredFeatures = [ - "Array.prototype.flat", - "Array.prototype.flatMap", - "Array.prototype.values", - "ArrayBuffer", - "async-functions", - "async-iteration", - "arrow-function", - "Atomics", - "caller", - "class", - "computed-property-names", - "const", - "cross-realm", - "DataView", - "DataView.prototype.getFloat32", - "DataView.prototype.getFloat64", - "DataView.prototype.getInt8", - "DataView.prototype.getInt16", - "DataView.prototype.getInt32", - "DataView.prototype.getUint16", - "DataView.prototype.getUint32", - "DataView.prototype.setUint8", - "default-parameters", - "destructuring-assignment", - "destructuring-binding", - "Float32Array", - "Float64Array", - "for-of", - "generators", - "globalThis", - "hashbang", - "Int8Array", - "Int32Array", - "Intl.ListFormat", - "Intl.Locale", - "Intl.NumberFormat-unified", - "Intl.RelativeTimeFormat", - "Intl.Segmenter", - "IsHTMLDDA", - "json-superset", - "let", - "Map", - "new.target", - "Object.fromEntries", - "Object.is", - "object-rest", - "object-spread", - "optional-catch-binding", - "Promise.prototype.finally", - "Proxy", - "Reflect", - "Reflect.construct", - "Reflect.set", - "Reflect.setPrototypeOf", - "regexp-dotall", - "regexp-lookbehind", - "regexp-named-groups", - "regexp-unicode-property-escapes", - "SharedArrayBuffer", - "Set", - "String.fromCodePoint", - "String.prototype.endsWith", - "String.prototype.includes", - "String.prototype.matchAll", - "String.prototype.trimEnd", - "String.prototype.trimStart", - "string-trimming", - "super", - "Symbol", - "Symbol.asyncIterator", - "Symbol.hasInstance", - "Symbol.isConcatSpreadable", - "Symbol.iterator", - "Symbol.match", - "Symbol.matchAll", - "Symbol.prototype.description", - "Symbol.replace", - "Symbol.search", - "Symbol.split", - "Symbol.species", - "Symbol.toPrimitive", - "Symbol.toStringTag", - "Symbol.unscopables", - "tail-call-optimization", - "template", - "TypedArray", - "u180e", - "Uint8Array", - "Uint8ClampedArray", - "Uint16Array", - "WeakMap", - "WeakSet", - "well-formed-json-stringify", -]; - -const ignoredTests = ["built-ins/RegExp/", "language/literals/regexp/"]; - -const featuresToPlugins = { - BigInt: "bigInt", - "class-fields-private": "classPrivateProperties", - "class-fields-public": "classProperties", - "class-methods-private": "classPrivateMethods", - "class-static-fields-public": "classProperties", - "class-static-fields-private": "classPrivateProperties", - "class-static-methods-private": "classPrivateMethods", - "dynamic-import": "dynamicImport", - "export-star-as-namespace-from-module": "exportNamespaceFrom", - "import.meta": "importMeta", - "numeric-separator-literal": "numericSeparator", - "optional-chaining": "optionalChaining", - "top-level-await": "topLevelAwait", -}; - -function getPlugins(features) { - return ( - features && - features - .map(f => { - if (!featuresToPlugins[f] && !ignoredFeatures.includes(f)) { - unmappedFeatures.add(f); - } - return featuresToPlugins[f]; - }) - .filter(Boolean) - ); -} - -const unmappedFeatures = new Set(); - -exports.getUnmappedFeatures = function() { - return unmappedFeatures; -}; - -exports.getTests = function(testDir) { - const stream = new TestStream(testDir, { - omitRuntime: true, - }); - const tests = []; - - stream.on("data", test => { - // strip test/ - const fileName = test.file.substr(5); - - if (ignoredTests.some(start => fileName.startsWith(start))) return; - - tests.push({ - contents: test.contents, - fileName, - id: `${fileName}(${test.scenario})`, - sourceType: test.attrs.flags.module ? "module" : "script", - plugins: getPlugins(test.attrs.features), - expectedError: - !!test.attrs.negative && - (test.attrs.negative.phase === "parse" || - test.attrs.negative.phase === "early"), - }); - }); - - return new Promise((resolve, reject) => { - stream.on("end", () => resolve(tests)); - stream.on("error", reject); - }); -}; - -exports.runTest = function(test) { - try { - parse(test.contents, { - sourceType: test.sourceType, - plugins: test.plugins, - }); - test.actualError = false; - } catch (err) { - test.actualError = true; - } - - test.result = test.expectedError !== test.actualError ? "fail" : "pass"; - - return test; -}; - -exports.getWhitelist = function(filename) { - return pfs.readFile(filename, "utf-8").then(function(contents) { - return contents - .split("\n") - .map(function(line) { - return line.replace(/#.*$/, "").trim(); - }) - .filter(function(line) { - return line.length > 0; - }) - .reduce(function(table, filename) { - table[filename] = true; - return table; - }, Object.create(null)); - }); -}; - -exports.updateWhitelist = function(filename, summary) { - return pfs.readFile(filename, "utf-8").then(function(contents) { - const toRemove = summary.disallowed.success - .concat(summary.disallowed.failure) - .map(function(test) { - return test.id; - }) - .concat(summary.unrecognized); - const toAdd = summary.disallowed.falsePositive - .concat(summary.disallowed.falseNegative) - .map(function(test) { - return test.id; - }); - const newContents = contents - .split("\n") - .map(function(line) { - const testId = line.replace(/#.*$/, "").trim(); - - if (toRemove.indexOf(testId) > -1) { - return null; - } - - return line; - }) - .filter(function(line) { - return line !== null && line !== ""; - }) - .concat(toAdd) - .sort() - .join("\n"); - - return pfs.writeFile(filename, newContents, "utf-8"); - }); -}; - -exports.interpret = function(results, whitelist) { - const summary = { - passed: true, - allowed: { - success: [], - failure: [], - falsePositive: [], - falseNegative: [], - }, - disallowed: { - success: [], - failure: [], - falsePositive: [], - falseNegative: [], - }, - unrecognized: null, - }; - - results.forEach(function(result) { - let classification, isAllowed; - const inWhitelist = result.id in whitelist; - delete whitelist[result.id]; - - if (!result.expectedError) { - if (!result.actualError) { - classification = "success"; - isAllowed = !inWhitelist; - } else { - classification = "falseNegative"; - isAllowed = inWhitelist; - } - } else { - if (!result.actualError) { - classification = "falsePositive"; - isAllowed = inWhitelist; - } else { - classification = "failure"; - isAllowed = !inWhitelist; - } - } - - summary.passed &= isAllowed; - summary[isAllowed ? "allowed" : "disallowed"][classification].push(result); - }); - - summary.unrecognized = Object.keys(whitelist); - summary.passed = !!summary.passed && summary.unrecognized.length === 0; - - return summary; -}; diff --git a/scripts/tests/test262/test262_whitelist.txt b/scripts/tests/test262/whitelist.txt similarity index 99% rename from scripts/tests/test262/test262_whitelist.txt rename to scripts/tests/test262/whitelist.txt index 1e21dcd228dc..512bee9a2ca8 100644 --- a/scripts/tests/test262/test262_whitelist.txt +++ b/scripts/tests/test262/whitelist.txt @@ -191,4 +191,4 @@ language/statements/class/elements/syntax/early-errors/invalid-names/method-outt language/statements/class/elements/syntax/early-errors/invalid-names/method-outter-member-expression-bad-reference.js(default) language/statements/class/elements/syntax/early-errors/invalid-names/method-outter-member-expression-bad-reference.js(strict mode) language/statements/class/elements/syntax/early-errors/invalid-names/method-outter-member-expression-this.js(default) -language/statements/class/elements/syntax/early-errors/invalid-names/method-outter-member-expression-this.js(strict mode) \ No newline at end of file +language/statements/class/elements/syntax/early-errors/invalid-names/method-outter-member-expression-this.js(strict mode) diff --git a/scripts/tests/utils/parser-test-runner.js b/scripts/tests/utils/parser-test-runner.js new file mode 100644 index 000000000000..764ef593d4f5 --- /dev/null +++ b/scripts/tests/utils/parser-test-runner.js @@ -0,0 +1,237 @@ +"use strict"; + +const fs = require("fs").promises; +const chalk = require("chalk"); +const { parse: parser } = require("../../../packages/babel-parser"); + +const dot = chalk.gray("."); + +class TestRunner { + constructor({ + testDir, + whitelist, + logInterval = 1, + shouldUpdate, + getTests, + parse = this.parse, + }) { + this.testDir = testDir; + this.whitelist = whitelist; + this.logInterval = logInterval; + this.shouldUpdate = shouldUpdate; + this.getTests = getTests; + this.parse = parse; + } + + async run() { + const whitelistP = this.getWhitelist(); + + console.log(`Now running tests...`); + + const results = []; + for await (const result of this.runTests()) { + results.push(result); + if (results.length % this.logInterval === 0) process.stdout.write(dot); + } + process.stdout.write("\n"); + + const summary = this.interpret(results, await whitelistP); + + await this.output(summary); + } + + async *runTests() { + for await (const test of this.getTests()) { + yield this.runTest(test); + } + } + + runTest(test) { + try { + this.parse(test, parser); + test.actualError = false; + } catch (err) { + test.actualError = true; + } + + test.result = test.expectedError !== test.actualError ? "fail" : "pass"; + + return test; + } + + parse(test, parser) { + parser(test.contents, { + sourceType: test.sourceType, + plugins: test.plugins, + }); + } + + async getWhitelist() { + const contents = await fs.readFile(this.whitelist, "utf-8"); + const table = new Set(); + + for (const line of contents.split("\n")) { + const filename = line.replace(/#.*$/, "").trim(); + if (filename) table.add(filename); + } + + return table; + } + + async updateWhitelist(summary) { + const contents = await fs.readFile(this.whitelist, "utf-8"); + + const toRemove = summary.disallowed.success + .concat(summary.disallowed.failure) + .map(test => test.id) + .concat(summary.unrecognized); + + const updated = summary.disallowed.falsePositive + .concat(summary.disallowed.falseNegative) + .map(test => test.id); + + for (const line of contents.split("\n")) { + const testId = line.replace(/#.*$/, "").trim(); + if (!toRemove.includes(testId) && line) { + updated.push(line); + } + } + + updated.sort(); + + await fs.writeFile(this.whitelist, updated.join("\n") + "\n", "utf8"); + } + + interpret(results, whitelist) { + const summary = { + passed: true, + allowed: { + success: [], + failure: [], + falsePositive: [], + falseNegative: [], + }, + disallowed: { + success: [], + failure: [], + falsePositive: [], + falseNegative: [], + }, + unrecognized: null, + count: results.length, + }; + + results.forEach(function(result) { + let classification, isAllowed; + const inWhitelist = whitelist.has(result.id); + whitelist.delete(result.id); + + if (!result.expectedError) { + if (!result.actualError) { + classification = "success"; + isAllowed = !inWhitelist; + } else { + classification = "falseNegative"; + isAllowed = inWhitelist; + } + } else { + if (!result.actualError) { + classification = "falsePositive"; + isAllowed = inWhitelist; + } else { + classification = "failure"; + isAllowed = !inWhitelist; + } + } + + summary.passed &= isAllowed; + summary[isAllowed ? "allowed" : "disallowed"][classification].push( + result + ); + }); + + summary.unrecognized = Array.from(whitelist); + summary.passed = !!summary.passed && summary.unrecognized.length === 0; + + return summary; + } + + async output(summary) { + const goodnews = [ + summary.allowed.success.length + " valid programs parsed without error", + summary.allowed.failure.length + + " invalid programs produced a parsing error", + summary.allowed.falsePositive.length + + " invalid programs did not produce a parsing error" + + " (and allowed by the whitelist file)", + summary.allowed.falseNegative.length + + " valid programs produced a parsing error" + + " (and allowed by the whitelist file)", + ]; + const badnews = []; + const badnewsDetails = []; + + void [ + { + tests: summary.disallowed.success, + label: + "valid programs parsed without error" + + " (in violation of the whitelist file)", + }, + { + tests: summary.disallowed.failure, + label: + "invalid programs produced a parsing error" + + " (in violation of the whitelist file)", + }, + { + tests: summary.disallowed.falsePositive, + label: + "invalid programs did not produce a parsing error" + + " (without a corresponding entry in the whitelist file)", + }, + { + tests: summary.disallowed.falseNegative, + label: + "valid programs produced a parsing error" + + " (without a corresponding entry in the whitelist file)", + }, + { + tests: summary.unrecognized, + label: "non-existent programs specified in the whitelist file", + }, + ].forEach(function({ tests, label }) { + if (!tests.length) { + return; + } + + const desc = tests.length + " " + label; + + badnews.push(desc); + badnewsDetails.push(desc + ":"); + badnewsDetails.push(...tests.map(test => ` ${test.id || test}`)); + }); + + console.log(`Testing complete (${summary.count} tests).`); + console.log("Summary:"); + console.log(chalk.green(goodnews.join("\n").replace(/^/gm, " ✔ "))); + + if (!summary.passed) { + console.log(""); + console.log(chalk.red(badnews.join("\n").replace(/^/gm, " ✘ "))); + console.log(""); + console.log("Details:"); + console.log(badnewsDetails.join("\n").replace(/^/gm, " ")); + } + + if (this.shouldUpdate) { + await this.updateWhitelist(summary); + console.log(""); + console.log("Whitelist file updated."); + } else { + process.exitCode = summary.passed ? 0 : 1; + } + } +} + +module.exports = exports = TestRunner; diff --git a/yarn.lock b/yarn.lock index f8c8092acdce..99c6211d1b03 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7438,6 +7438,11 @@ merge2@^1.2.3: resolved "https://registry.yarnpkg.com/merge2/-/merge2-1.3.0.tgz#5b366ee83b2f1582c48f87e47cf1a9352103ca81" integrity sha512-2j4DAdlBOkiSZIsaXk4mTE3sRS02yBHAtfy127xRV3bQUFqXkjHCHLW6Scv7DwNRbIWNHH8zpnz9zMaKXIdvYw== +mergeiterator@^1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/mergeiterator/-/mergeiterator-1.2.5.tgz#40f0a27888f2eb85d80fa4c29632cf400a0b8c7b" + integrity sha512-KVZWGnRkEgmVewnpVIEUm/KuQ2kHx4T0zYQlPbGypBk4kM5sJ1+7bPDR2N4yg3EoS1CiGKk2GRb5PkAsMEoh3w== + micromatch@^2.1.5: version "2.3.11" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-2.3.11.tgz#86677c97d1720b363431d04d0d15293bd38c1565"