From 29c1d3d993f9e345bf42ea9528464643f5dfa592 Mon Sep 17 00:00:00 2001 From: Kenny F Date: Sat, 27 Jul 2019 10:15:51 +0800 Subject: [PATCH] fix(lexer): improved identifier scanning performance --- package-lock.json | 2 +- src/errors.ts | 6 +- src/lexer/charClassifier.ts | 31 +++++----- src/lexer/identifier.ts | 41 ++++++------ src/parser.ts | 106 +++++++++++++++----------------- src/token.ts | 10 ++- test/lexer/charClassifier.ts | 1 - test/lexer/identifiers.ts | 5 ++ test/parser/declarations/let.ts | 2 +- 9 files changed, 100 insertions(+), 104 deletions(-) diff --git a/package-lock.json b/package-lock.json index cdc2e6ab..aca026a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "meriyah", - "version": "1.4.2", + "version": "1.4.4", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/src/errors.ts b/src/errors.ts index 532400bb..5368365e 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -167,7 +167,8 @@ export const enum Errors { InvalidDotProperty, UnclosedSpreadElement, CatchWithoutTry, - FinallyWithoutTry + FinallyWithoutTry, + UnCorrespondingFragmentTag } /*@internal*/ @@ -347,7 +348,8 @@ export const errorMessages: { [Errors.InvalidDotProperty]: 'Dot property must be an identifier', [Errors.UnclosedSpreadElement]: 'Encountered invalid input after spread/rest argument', [Errors.CatchWithoutTry]: 'Catch without try', - [Errors.FinallyWithoutTry]: 'Finally without try' + [Errors.FinallyWithoutTry]: 'Finally without try', + [Errors.UnCorrespondingFragmentTag]: 'Expected corresponding closing tag for JSX fragment' }; export class ParseError extends SyntaxError { diff --git a/src/lexer/charClassifier.ts b/src/lexer/charClassifier.ts index a8d0faec..76d63db2 100644 --- a/src/lexer/charClassifier.ts +++ b/src/lexer/charClassifier.ts @@ -5,21 +5,20 @@ export const enum CharFlags { None = 0, IdentifierStart = 1 << 0, IdentifierPart = 1 << 1, - KeywordCandidate = 1 << 6, - LineTerminator = 1 << 9, // ECMA-262 11.3 Line Terminators - Decimal = 1 << 10, - Octal = 1 << 11, - Hex = 1 << 12, - Binary = 1 << 13, - Exponent = 1 << 15, - BackSlash = 1 << 17, - ImplicitOctalDigits = 1 << 18, - CarriageReturn = 1 << 19, - LineFeed = 1 << 20, - Underscore = 1 << 21, - StringLiteral = 1 << 22, - JSXToken = 1 << 23, - Hyphen = 1 << 24, + KeywordCandidate = 1 << 2, + LineTerminator = 1 << 3, // ECMA-262 11.3 Line Terminators + Decimal = 1 << 4, + Octal = 1 << 5, + Hex = 1 << 6, + Binary = 1 << 7, + Exponent = 1 << 8, + ImplicitOctalDigits = 1 << 9, + CarriageReturn = 1 << 10, + LineFeed = 1 << 11, + Underscore = 1 << 12, + StringLiteral = 1 << 13, + JSXToken = 1 << 14, + Hyphen = 1 << 15, } /** @@ -118,7 +117,7 @@ export const CharTypes = [ CharFlags.IdentifierStart | CharFlags.IdentifierPart /* 0x59 Y */, CharFlags.IdentifierStart | CharFlags.IdentifierPart /* 0x5A Z */, CharFlags.None /* 0x5B */, - CharFlags.BackSlash | CharFlags.IdentifierStart /* 0x5C */, + CharFlags.IdentifierStart /* 0x5C */, CharFlags.None /* 0x5D */, CharFlags.None /* 0x5E */, CharFlags.IdentifierStart | CharFlags.IdentifierPart | CharFlags.Underscore/* 0x5F _ */, diff --git a/src/lexer/identifier.ts b/src/lexer/identifier.ts index 34bb4ce7..acd8616c 100644 --- a/src/lexer/identifier.ts +++ b/src/lexer/identifier.ts @@ -14,11 +14,9 @@ import { report, reportMessageAt, Errors } from '../errors'; export function scanIdentifier(parser: ParserState, context: Context, isValidAsKeyword: 0 | 1): Token { while ((CharTypes[nextCP(parser)] & CharFlags.IdentifierPart) !== 0) {} parser.tokenValue = parser.source.slice(parser.tokenPos, parser.index); - if ((CharTypes[parser.nextCP] & CharFlags.BackSlash) === 0 && parser.nextCP < 0x7e) { - return descKeywordTable[parser.tokenValue] || Token.Identifier; - } - // Slow path that has to deal with multi unit encoding - return scanIdentifierSlowCase(parser, context, 0, isValidAsKeyword); + return parser.nextCP !== Chars.Backslash && parser.nextCP < 0x7e + ? descKeywordTable[parser.tokenValue] || Token.Identifier + : scanIdentifierSlowCase(parser, context, 0, isValidAsKeyword); } /** @@ -28,7 +26,7 @@ export function scanIdentifier(parser: ParserState, context: Context, isValidAsK * @param context Context masks */ export function scanUnicodeIdentifier(parser: ParserState, context: Context): Token { - const cookedChar = scanIdentifierUnicodeEscape(parser) as number; + const cookedChar = scanIdentifierUnicodeEscape(parser); if (!isIdentifierPart(cookedChar)) report(parser, Errors.InvalidUnicodeEscapeSequence); parser.tokenValue = fromCodePoint(cookedChar); return scanIdentifierSlowCase(parser, context, /* hasEscape */ 1, CharTypes[cookedChar] & CharFlags.KeywordCandidate); @@ -39,8 +37,8 @@ export function scanUnicodeIdentifier(parser: ParserState, context: Context): To * * @param parser Parser object * @param context Context masks - * @param hasEscape - * @param canBeKeyword + * @param hasEscape True if contains a unicode sequence + * @param isValidAsKeyword */ export function scanIdentifierSlowCase( parser: ParserState, @@ -49,11 +47,12 @@ export function scanIdentifierSlowCase( isValidAsKeyword: number ): Token { let start = parser.index; + while (parser.index < parser.end) { if (parser.nextCP === Chars.Backslash) { parser.tokenValue += parser.source.slice(start, parser.index); hasEscape = 1; - const code = scanIdentifierUnicodeEscape(parser) as number; + const code = scanIdentifierUnicodeEscape(parser); if (!isIdentifierPart(code)) report(parser, Errors.InvalidUnicodeEscapeSequence); isValidAsKeyword = isValidAsKeyword && CharTypes[code] & CharFlags.KeywordCandidate; parser.tokenValue += fromCodePoint(code); @@ -72,18 +71,18 @@ export function scanIdentifierSlowCase( const length = parser.tokenValue.length; if (isValidAsKeyword && (length >= 2 && length <= 11)) { - const t: Token | undefined = descKeywordTable[parser.tokenValue]; + const token: Token | undefined = descKeywordTable[parser.tokenValue]; - return t === void 0 + return token === void 0 ? Token.Identifier - : t === Token.YieldKeyword || !hasEscape - ? t - : context & Context.Strict && (t === Token.LetKeyword || t === Token.StaticKeyword) + : token === Token.YieldKeyword || !hasEscape + ? token + : context & Context.Strict && (token === Token.LetKeyword || token === Token.StaticKeyword) ? Token.EscapedFutureReserved - : (t & Token.FutureReserved) === Token.FutureReserved + : (token & Token.FutureReserved) === Token.FutureReserved ? context & Context.Strict ? Token.EscapedFutureReserved - : t + : token : Token.EscapedReserved; } return Token.Identifier; @@ -104,14 +103,14 @@ export function scanPrivateName(parser: ParserState): Token { * * @param parser Parser object */ -export function scanIdentifierUnicodeEscape(parser: ParserState): number | void { +export function scanIdentifierUnicodeEscape(parser: ParserState): number { // Check for Unicode escape of the form '\uXXXX' // and return code point value if valid Unicode escape is found. - if (parser.index + 5 < parser.end && parser.source.charCodeAt(parser.index + 1) === Chars.LowerU) { - parser.nextCP = parser.source.charCodeAt((parser.index += 2)); - return scanUnicodeEscapeValue(parser); + if (parser.source.charCodeAt(parser.index + 1) !== Chars.LowerU) { + report(parser, Errors.InvalidUnicodeEscapeSequence); } - report(parser, Errors.InvalidUnicodeEscapeSequence); + parser.nextCP = parser.source.charCodeAt((parser.index += 2)); + return scanUnicodeEscapeValue(parser); } /** diff --git a/src/parser.ts b/src/parser.ts index d43d0d87..4e102d66 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -198,7 +198,7 @@ export interface Options { jsx?: boolean; // Allow edge cases that deviate from the spec specDeviation?: boolean; - // Allowes comment extraction. Accepts either a function or array + // Allowes comment extraction. Accepts either a a callback function or an array onComment?: OnComment; } @@ -1902,8 +1902,6 @@ export function parseLetIdentOrVarDeclarationStatement( if (context & Context.Strict) report(parser, Errors.UnexpectedLetStrictReserved); - if (parser.token === Token.LetKeyword) report(parser, Errors.InvalidLetBoundName); - /** LabelledStatement[Yield, Await, Return]: * * ExpressionStatement | LabelledStatement :: @@ -2404,7 +2402,7 @@ function parseImportDeclaration( nextToken(parser, context); - let source: ESTree.Literal; + let source: ESTree.Literal | null = null; // This is `import("...")` call or `import.meta` meta property case. // @@ -3955,7 +3953,7 @@ export function parsePrimaryExpressionExtended( return parseImportCallExpression(parser, context, inNewExpression, inGroup, start, line, column); case Token.LessThan: if (context & Context.OptionsJSX) - return parseJSXRootElementOrFragment(parser, context, /*isJSXChild*/ 1, start, line, column); + return parseJSXRootElementOrFragment(parser, context, /*inJSXChild*/ 1, start, line, column); default: if ( context & Context.Strict @@ -6451,7 +6449,7 @@ export function parseParenthesizedExpression( expr = parsePrimaryExpressionExtended(parser, context, kind, 0, 1, 0, 1, tokenPos, linePos, colPos); - if ((parser.token & Token.IsCommaOrRightParen) === Token.IsCommaOrRightParen) { + if (parser.token === Token.RightParen || parser.token === Token.Comma) { if (parser.assignable & AssignmentKind.NotAssignable) { destructible |= DestructuringKind.CannotDestruct; isComplex = 1; @@ -6474,7 +6472,7 @@ export function parseParenthesizedExpression( expr = parseMemberOrUpdateExpression(parser, context, expr, 1, tokenPos, linePos, colPos); - if ((parser.token & Token.IsCommaOrRightParen) !== Token.IsCommaOrRightParen) { + if (parser.token !== Token.RightParen && parser.token !== Token.Comma) { expr = parseAssignmentExpression(parser, context, 1, tokenPos, linePos, colPos, expr); } } @@ -6490,14 +6488,14 @@ export function parseParenthesizedExpression( parser.assignable = AssignmentKind.NotAssignable; - if ((parser.token & Token.IsCommaOrRightParen) !== Token.IsCommaOrRightParen) { + if (parser.token !== Token.RightParen && parser.token !== Token.Comma) { if (destructible & DestructuringKind.MustDestruct) report(parser, Errors.InvalidPatternTail); expr = parseMemberOrUpdateExpression(parser, context, expr, 0, tokenPos, linePos, colPos); destructible |= DestructuringKind.CannotDestruct; - if ((parser.token & Token.IsCommaOrRightParen) !== Token.IsCommaOrRightParen) { + if (parser.token !== Token.RightParen && parser.token !== Token.Comma) { expr = parseAssignmentExpression(parser, context, 0, tokenPos, linePos, colPos, expr); } } @@ -6520,7 +6518,7 @@ export function parseParenthesizedExpression( isComplex = 1; - if (isSequence && (parser.token & Token.IsCommaOrRightParen) === Token.IsCommaOrRightParen) { + if (isSequence && (parser.token === Token.RightParen || parser.token === Token.Comma)) { expressions.push(expr); } destructible |= DestructuringKind.MustDestruct; @@ -6530,7 +6528,7 @@ export function parseParenthesizedExpression( expr = parseExpression(parser, context, 1, 0, 1, tokenPos, linePos, colPos); - if (isSequence && (parser.token & Token.IsCommaOrRightParen) === Token.IsCommaOrRightParen) { + if (isSequence && (parser.token === Token.RightParen || parser.token === Token.Comma)) { expressions.push(expr); } @@ -6561,7 +6559,7 @@ export function parseParenthesizedExpression( return expr; } - if (isSequence && (parser.token & Token.IsCommaOrRightParen) === Token.IsCommaOrRightParen) { + if (isSequence && (parser.token === Token.RightParen || parser.token === Token.Comma)) { expressions.push(expr); } @@ -7270,7 +7268,7 @@ export function parseAsyncArrowOrCallExpression( } expr = parsePrimaryExpressionExtended(parser, context, kind, 0, 1, 0, 1, tokenPos, linePos, colPos); - if ((parser.token & Token.IsCommaOrRightParen) === Token.IsCommaOrRightParen) { + if (parser.token === Token.RightParen || parser.token === Token.Comma) { if (parser.assignable & AssignmentKind.NotAssignable) { destructible |= DestructuringKind.CannotDestruct; isComplex = 1; @@ -7291,7 +7289,7 @@ export function parseAsyncArrowOrCallExpression( expr = parseMemberOrUpdateExpression(parser, context, expr, 1, tokenPos, linePos, colPos); - if ((parser.token & Token.IsCommaOrRightParen) !== Token.IsCommaOrRightParen) { + if (parser.token !== Token.RightParen && parser.token !== Token.Comma) { expr = parseAssignmentExpression(parser, context, 1, tokenPos, linePos, colPos, expr); } } @@ -7307,14 +7305,14 @@ export function parseAsyncArrowOrCallExpression( parser.assignable = AssignmentKind.NotAssignable; - if ((parser.token & Token.IsCommaOrRightParen) !== Token.IsCommaOrRightParen) { + if (parser.token !== Token.RightParen && parser.token !== Token.Comma) { if (destructible & DestructuringKind.MustDestruct) report(parser, Errors.InvalidPatternTail); expr = parseMemberOrUpdateExpression(parser, context, expr, 0, tokenPos, linePos, colPos); destructible |= DestructuringKind.CannotDestruct; - if ((parser.token & Token.IsCommaOrRightParen) !== Token.IsCommaOrRightParen) { + if (parser.token !== Token.RightParen && parser.token !== Token.Comma) { expr = parseAssignmentExpression(parser, context, 0, parser.tokenPos, parser.linePos, parser.colPos, expr); } } @@ -8248,7 +8246,7 @@ function parseAndClassifyIdentifier( * * @param parser Parser object * @param context Context masks - * @param isJSXChild + * @param inJSXChild * @param start * @param line * @param column @@ -8257,7 +8255,7 @@ function parseAndClassifyIdentifier( function parseJSXRootElementOrFragment( parser: ParserState, context: Context, - isJSXChild: 0 | 1, + inJSXChild: 0 | 1, start: number, line: number, column: number @@ -8273,7 +8271,7 @@ function parseJSXRootElementOrFragment( closingFragment: parseJSXClosingFragment( parser, context, - isJSXChild, + inJSXChild, parser.tokenPos, parser.linePos, parser.colPos @@ -8283,10 +8281,11 @@ function parseJSXRootElementOrFragment( let closingElement: ESTree.JSXClosingElement | null = null; let children: ESTree.JSXChild[] = []; + const openingElement: ESTree.JSXOpeningElement = parseJSXOpeningFragmentOrSelfCloseElement( parser, context, - isJSXChild, + inJSXChild, start, line, column @@ -8297,7 +8296,7 @@ function parseJSXRootElementOrFragment( closingElement = parseJSXClosingElement( parser, context, - isJSXChild, + inJSXChild, parser.tokenPos, parser.linePos, parser.colPos @@ -8341,7 +8340,7 @@ export function parseOpeningFragment( * * @param parser Parser object * @param context Context masks - * @param isJSXChild + * @param inJSXChild * @param start * @param line * @param column @@ -8349,14 +8348,14 @@ export function parseOpeningFragment( function parseJSXClosingElement( parser: ParserState, context: Context, - isJSXChild: 0 | 1, + inJSXChild: 0 | 1, start: number, line: number, column: number ): ESTree.JSXClosingElement { consume(parser, context, Token.JSXClose); const name = parseJSXElementName(parser, context, parser.tokenPos, parser.linePos, parser.colPos); - if (isJSXChild) { + if (inJSXChild) { consume(parser, context, Token.GreaterThan); } else { parser.token = scanJSXToken(parser); @@ -8373,7 +8372,7 @@ function parseJSXClosingElement( * * @param parser Parser object * @param context Context masks - * @param isJSXChild + * @param inJSXChild * @param start * @param line * @param column @@ -8381,16 +8380,18 @@ function parseJSXClosingElement( export function parseJSXClosingFragment( parser: ParserState, context: Context, - isJSXChild: 0 | 1, + inJSXChild: 0 | 1, start: number, line: number, column: number ): ESTree.JSXClosingFragment { consume(parser, context, Token.JSXClose); + if ((parser.token & Token.IsIdentifier) === Token.IsIdentifier) { - report(parser, Errors.Unexpected); // Expected_corresponding_closing_tag_for_JSX_fragment + report(parser, Errors.UnCorrespondingFragmentTag); } - if (isJSXChild) { + + if (inJSXChild) { consume(parser, context, Token.GreaterThan); } else { consume(parser, context, Token.GreaterThan); @@ -8429,17 +8430,12 @@ export function parseJSXChildren(parser: ParserState, context: Context): ESTree. * @param column */ function parseJSXChild(parser: ParserState, context: Context, start: number, line: number, column: number): any { - switch (parser.token) { - case Token.JSXText: - case Token.Identifier: - return parseJSXText(parser, context, start, line, column); - case Token.LeftBrace: - return parseJSXExpressionContainer(parser, context, /*isJSXChild*/ 0, /* isAttr */ 0, start, line, column); - case Token.LessThan: - return parseJSXRootElementOrFragment(parser, context, /*isJSXChild*/ 0, start, line, column); - default: - report(parser, Errors.Unexpected); - } + if (parser.token === Token.JSXText) return parseJSXText(parser, context, start, line, column); + if (parser.token === Token.LeftBrace) + return parseJSXExpressionContainer(parser, context, /*inJSXChild*/ 0, /* isAttr */ 0, start, line, column); + if (parser.token === Token.LessThan) + return parseJSXRootElementOrFragment(parser, context, /*inJSXChild*/ 0, start, line, column); + report(parser, Errors.Unexpected); } /** @@ -8458,11 +8454,10 @@ export function parseJSXText( line: number, column: number ): ESTree.JSXText { - const value = parser.tokenValue; scanJSXToken(parser); return finishNode(parser, context, start, line, column, { type: 'JSXText', - value + value: parser.tokenValue as string }); } @@ -8471,7 +8466,7 @@ export function parseJSXText( * * @param parser Parser object * @param context Context masks - * @param isJSXChild + * @param inJSXChild * @param start * @param line * @param column @@ -8479,7 +8474,7 @@ export function parseJSXText( function parseJSXOpeningFragmentOrSelfCloseElement( parser: ParserState, context: Context, - isJSXChild: 0 | 1, + inJSXChild: 0 | 1, start: number, line: number, column: number @@ -8494,7 +8489,7 @@ function parseJSXOpeningFragmentOrSelfCloseElement( scanJSXToken(parser); } else { consume(parser, context, Token.Divide); - if (isJSXChild) { + if (inJSXChild) { consume(parser, context, Token.GreaterThan); } else { scanJSXToken(parser); @@ -8527,17 +8522,17 @@ function parseJSXElementName( ): ESTree.JSXIdentifier | ESTree.JSXMemberExpression { scanJSXIdentifier(parser); - let name: any = parseJSXIdentifier(parser, context, start, line, column); + let key: ESTree.JSXIdentifier | ESTree.JSXMemberExpression = parseJSXIdentifier(parser, context, start, line, column); // Namespace - if (parser.token === Token.Colon) return parseJSXNamespacedName(parser, context, name, start, line, column); + if (parser.token === Token.Colon) return parseJSXNamespacedName(parser, context, key, start, line, column); // Member expression while (consumeOpt(parser, context, Token.Period)) { scanJSXIdentifier(parser); - name = parseJSXMemberExpression(parser, context, name, start, line, column); + key = parseJSXMemberExpression(parser, context, key, start, line, column); } - return name; + return key; } /** @@ -8562,7 +8557,7 @@ export function parseJSXMemberExpression( type: 'JSXMemberExpression', object, property - } as any); + }); } /** @@ -8579,8 +8574,7 @@ export function parseJSXAttributes( context: Context ): (ESTree.JSXAttribute | ESTree.JSXSpreadAttribute)[] { const attributes: (ESTree.JSXAttribute | ESTree.JSXSpreadAttribute)[] = []; - while (parser.index < parser.end) { - if (parser.token === Token.Divide || parser.token === Token.GreaterThan) break; + while (parser.token !== Token.Divide && parser.token !== Token.GreaterThan) { attributes.push(parseJsxAttribute(parser, context, parser.tokenPos, parser.linePos, parser.colPos)); } return attributes; @@ -8602,7 +8596,7 @@ export function parseJSXSpreadAttribute( line: number, column: number ): ESTree.JSXSpreadAttribute { - nextToken(parser, context); + nextToken(parser, context); // skips: '{' consume(parser, context, Token.Ellipsis); const expression = parseExpression(parser, context, 1, 0, 0, parser.tokenPos, parser.linePos, parser.colPos); consume(parser, context, Token.RightBrace); @@ -8646,7 +8640,7 @@ function parseJsxAttribute( value = parseLiteral(parser, context, tokenPos, linePos, colPos); break; case Token.LessThan: - value = parseJSXRootElementOrFragment(parser, context, /*isJSXChild*/ 1, tokenPos, linePos, colPos); + value = parseJSXRootElementOrFragment(parser, context, /*inJSXChild*/ 1, tokenPos, linePos, colPos); break; case Token.LeftBrace: value = parseJSXExpressionContainer(parser, context, 1, 1, tokenPos, linePos, colPos); @@ -8695,7 +8689,7 @@ function parseJSXNamespacedName( * * @param parser Parser object * @param context Context masks - * @param isJSXChild + * @param inJSXChild * @param start * @param line * @param column @@ -8703,7 +8697,7 @@ function parseJSXNamespacedName( function parseJSXExpressionContainer( parser: ParserState, context: Context, - isJSXChild: 0 | 1, + inJSXChild: 0 | 1, isAttr: 0 | 1, start: number, line: number, @@ -8722,7 +8716,7 @@ function parseJSXExpressionContainer( } else { expression = parseExpression(parser, context, 1, 0, 0, tokenPos, linePos, colPos); } - if (isJSXChild) { + if (inJSXChild) { consume(parser, context, Token.RightBrace); } else { scanJSXToken(parser); diff --git a/src/token.ts b/src/token.ts index 9a8c93be..4d3fcf18 100644 --- a/src/token.ts +++ b/src/token.ts @@ -17,7 +17,6 @@ export const enum Token { IsLogical = 1 << 19, IsAutoSemicolon = 1 << 20, IsPatternStart = 1 << 21, // Start of pattern, '[' or '{' - IsAssignOp = 1 << 22, IsBinaryOp = 1 << 23 | IsExpressionStart, IsUnaryOp = 1 << 24 | IsExpressionStart, @@ -26,8 +25,7 @@ export const enum Token { IsStringOrNumber = 1 << 27, VarDecl = 1 << 28, IsEvalOrArguments = 1 << 29 | IsExpressionStart | IsIdentifier, - IsCommaOrRightParen = 1 << 30, - IsClassField = 1 << 31, + IsClassField = 1 << 30, /* Node types */ EOF = 0 | IsAutoSemicolon, @@ -52,9 +50,9 @@ export const enum Token { Period = 13 | IsMemberOrCallExpression, // . Ellipsis = 14, // ... RightBrace = 15 | IsAutoSemicolon | IsClassField, // } - RightParen = 16 | IsCommaOrRightParen, // ) + RightParen = 16, // ) Semicolon = 17 | IsAutoSemicolon | IsClassField, // ; - Comma = 18 | IsCommaOrRightParen | IsClassField, // , + Comma = 18 | IsClassField, // , LeftBracket = 19 | IsExpressionStart | IsPatternStart | IsMemberOrCallExpression, // [ RightBracket = 20, // ] Colon = 21, // : @@ -174,7 +172,7 @@ export const enum Token { EscapedFutureReserved = 119 | IsIdentifier, ReservedIfStrict = 120 | IsIdentifier, - PrivateName = 121 | IsIdentifier, + PrivateName = 121, BigIntLiteral = 122, WhiteSpace = 124, Illegal = 129, diff --git a/test/lexer/charClassifier.ts b/test/lexer/charClassifier.ts index 11ab2b58..ad5cee10 100644 --- a/test/lexer/charClassifier.ts +++ b/test/lexer/charClassifier.ts @@ -18,7 +18,6 @@ describe('Lexer - charClassifier', () => { [CharFlags.IdentifierStart | CharFlags.IdentifierPart, 77], [CharFlags.IdentifierStart | CharFlags.IdentifierPart, 78], [CharFlags.IdentifierStart | CharFlags.IdentifierPart, 103], - [CharFlags.BackSlash | CharFlags.IdentifierStart, 92], [CharFlags.IdentifierStart | CharFlags.IdentifierPart, 122] ]; diff --git a/test/lexer/identifiers.ts b/test/lexer/identifiers.ts index 6dc98111..ceb8ce19 100644 --- a/test/lexer/identifiers.ts +++ b/test/lexer/identifiers.ts @@ -79,6 +79,7 @@ describe('Lexer - Identifiers', () => { [Context.None, Token.Identifier, 'ab\\u{00072}', 'abr'], [Context.None, Token.Identifier, 'ab\\u{072}', 'abr'], [Context.None, Token.Identifier, '\\u{4fff}', '俿'], + [Context.None, Token.Identifier, '\\u{222}', 'Ȣ'], [Context.None, Token.Identifier, '\\u{1EE00}', '{Ȁ'], [Context.None, Token.Identifier, 'a\\u{0000000000000000000071}c', 'aqc'], @@ -219,6 +220,7 @@ describe('Lexer - Identifiers', () => { fail('fails on \\u007', '\\u007', Context.OptionsNext); fail('fails on \\u007Xvwxyz', '\\u007Xvwxyz', Context.OptionsNext); fail('fails on abc\\u{}', 'abc\\u{}', Context.OptionsNext); + fail('fails on abc\\u{}', 'abc\\u{}', Context.OptionsWebCompat); fail('fails on abc\\u}', 'abc\\u}', Context.OptionsNext); fail('fails on abc\\u{', 'abc\\u{', Context.OptionsNext); fail('fails on \\u{70bc', '\\u{70bc', Context.OptionsNext); @@ -227,6 +229,9 @@ describe('Lexer - Identifiers', () => { fail('fails on \\u{10401', '\\u{10401', Context.None); fail('fails on \\u104', '\\u104', Context.None); fail('fails on \\u{!', '\\u{!', Context.None); + fail('fails on \\u{}', '\\u{}', Context.None); + fail('fails on \\u}', '\\u}', Context.None); + fail('fails on \\}', '\\}', Context.None); fail('fails on \\u', '\\u', Context.None); fail('fails on \\u{4fff', '\\u{4fff', Context.None); fail('fails on \\u{4ff', '\\u{4ff', Context.None); diff --git a/test/parser/declarations/let.ts b/test/parser/declarations/let.ts index 07b852c1..28d592b4 100644 --- a/test/parser/declarations/let.ts +++ b/test/parser/declarations/let.ts @@ -486,7 +486,7 @@ describe('Declarations - Let', () => { ['let [...[ x ] = []] = [];', Context.None], ['let [...[ x ] = []] = [];', Context.None], ['let [...{ x } = []] = [];', Context.None], - ['let\\nlet', Context.None], + ['let\nlet', Context.None], ['do let [x] = 0; while (false);', Context.None], ['if (x) let [x] = y; else x;', Context.None], ['do let [] while (a);', Context.None],