Update: rewrite TokenStore (fixes #7810) (#7936)

* Update: rewrite TokenStore (fixes #7810)

- Refactors TokenStore with ES2015 classes.
- Adds object-style options to every method of TokenStore.
  E.g. `{includeComments: true, skip: 0, filter: null}`
- Adds `getFirstTokenBetween` and `getLastTokenBetween` methods.
- Adds `getTokenOrCommentAfter` and `getTokenOrCommentBefore` methods
  to TokenStore for backward compatibility.
- Removes the TokenStore instance for the mix of tokens and comments.

* Chore: refactor with new API

* Docs: update working-with-rules.md

* Fix: improve tests

* Chore: simplify `isNot*` functions

* Chore: fix jsdoc comments.

* Fix: `isComment` returns true for shebang

* Chore: remove unnecessary `isOfToken` function

* Chore: refactor TokenStore; remove `movePrev` methods

* Chore: rename variables: start, end, imap

* Fix: this.predicate() → predicate()

* Chore: typeof → in

* Chore: use lodash.sortedIndexBy

* Chore: update jsdoc comments

* Docs: add a note for skip & filter

* Chore: fix typo

* Chore: fix "Block" to `.startsWith("Block")`

To change `Block` to `BlockComment` in future.

* Docs: add compatibility notes to JSDoc

* Chore: modularize TokenStore

* Chore: fix plural in test descriptions

* Fix: add assertions for options.skip and options.count

* Chore: it/ic → tokenIndex/commentIndex

docs/developer-guide/working-with-rules.md

@@ -261,23 +261,37 @@ module.exports = {
 
 Once you have an instance of `SourceCode`, you can use the methods on it to work with the code:
 
+* `getText(node)` - returns the source code for the given node. Omit `node` to get the whole source.
 * `getAllComments()` - returns an array of all comments in the source.
 * `getComments(node)` - returns the leading and trailing comments arrays for the given node.
-* `getFirstToken(node)` - returns the first token representing the given node.
-* `getFirstTokens(node, count)` - returns the first `count` tokens representing the given node.
 * `getJSDocComment(node)` - returns the JSDoc comment for a given node or `null` if there is none.
-* `getLastToken(node)` - returns the last token representing the given node.
-* `getLastTokens(node, count)` - returns the last `count` tokens representing the given node.
-* `getNodeByRangeIndex(index)` - returns the deepest node in the AST containing the given source index.
 * `isSpaceBetweenTokens(first, second)` - returns true if there is a whitespace character between the two tokens.
-* `getText(node)` - returns the source code for the given node. Omit `node` to get the whole source.
-* `getTokenAfter(nodeOrToken)` - returns the first token after the given node or token.
-* `getTokenBefore(nodeOrToken)` - returns the first token before the given node or token.
-* `getTokenByRangeStart(index)` - returns the token whose range starts at the given index in the source.
+* `getFirstToken(node, skipOptions)` - returns the first token representing the given node.
+* `getFirstTokens(node, countOptions)` - returns the first `count` tokens representing the given node.
+* `getLastToken(node, skipOptions)` - returns the last token representing the given node.
+* `getLastTokens(node, countOptions)` - returns the last `count` tokens representing the given node.
+* `getTokenAfter(nodeOrToken, skipOptions)` - returns the first token after the given node or token.
+* `getTokensAfter(nodeOrToken, countOptions)` - returns `count` tokens after the given node or token.
+* `getTokenBefore(nodeOrToken, skipOptions)` - returns the first token before the given node or token.
+* `getTokensBefore(nodeOrToken, countOptions)` - returns `count` tokens before the given node or token.
+* `getFirstTokenBetween(nodeOrToken1, nodeOrToken2, skipOptions)` - returns the first token between two nodes or tokens.
+* `getFirstTokensBetween(nodeOrToken1, nodeOrToken2, countOptions)` - returns the first `count` tokens between two nodes or tokens.
+* `getLastTokenBetween(nodeOrToken1, nodeOrToken2, skipOptions)` - returns the last token between two nodes or tokens.
+* `getLastTokensBetween(nodeOrToken1, nodeOrToken2, countOptions)` - returns the last `count` tokens between two nodes or tokens.
 * `getTokens(node)` - returns all tokens for the given node.
-* `getTokensAfter(nodeOrToken, count)` - returns `count` tokens after the given node or token.
-* `getTokensBefore(nodeOrToken, count)` - returns `count` tokens before the given node or token.
-* `getTokensBetween(node1, node2)` - returns the tokens between two nodes.
+* `getTokensBetween(nodeOrToken1, nodeOrToken2)` - returns all tokens between two nodes.
+* `getTokenByRangeStart(index)` - returns the token whose range starts at the given index in the source.
+* `getNodeByRangeIndex(index)` - returns the deepest node in the AST containing the given source index.
+
+> `skipOptions` is an object which has 3 properties; `skip`, `includeComments`, and `filter`. Default is `{skip: 0, includeComments: false, filter: null}`.
+> - The `skip` is a positive integer, the number of skipping tokens. If `filter` option is given at the same time, it doesn't count filtered tokens as skipped.
+> - The `includeComments` is a boolean value, the flag to include comment tokens into the result.
+> - The `filter` is a function which gets a token as the first argument, if the function returns `false` then the result excludes the token.
+>
+> `countOptions` is an object which has 3 properties; `count`, `includeComments`, and `filter`. Default is `{count: 0, includeComments: false, filter: null}`.
+> - The `count` is a positive integer, the maximum number of returning tokens.
+> - The `includeComments` is a boolean value, the flag to include comment tokens into the result.
+> - The `filter` is a function which gets a token as the first argument, if the function returns `false` then the result excludes the token.
 
 There are also some properties you can access:
 

lib/ast-utils.js

@@ -213,6 +213,15 @@ function isMethodWhichHasThisArg(node) {
     return false;
 }
 
+/**
+ * Creates the negate function of the given function.
+ * @param {Function} f - The function to negate.
+ * @returns {Function} Negated function.
+ */
+function negate(f) {
+    return token => !f(token);
+}
+
 /**
  * Checks whether or not a node has a `@this` tag in its comments.
  * @param {ASTNode} node - A node to check.
@@ -250,20 +259,123 @@ function isParenthesised(sourceCode, node) {
 }
 
 /**
- * Gets the `=>` token of the given arrow function node.
+ * Checks if the given token is an arrow token or not.
  *
- * @param {ASTNode} node - The arrow function node to get.
- * @param {SourceCode} sourceCode - The source code object to get tokens.
- * @returns {Token} `=>` token.
+ * @param {Token} token - The token to check.
+ * @returns {boolean} `true` if the token is an arrow token.
  */
-function getArrowToken(node, sourceCode) {
-    let token = sourceCode.getTokenBefore(node.body);
+function isArrowToken(token) {
+    return token.value === "=>" && token.type === "Punctuator";
+}
 
-    while (token.value !== "=>") {
-        token = sourceCode.getTokenBefore(token);
-    }
+/**
+ * Checks if the given token is a comma token or not.
+ *
+ * @param {Token} token - The token to check.
+ * @returns {boolean} `true` if the token is a comma token.
+ */
+function isCommaToken(token) {
+    return token.value === "," && token.type === "Punctuator";
+}
+
+/**
+ * Checks if the given token is a semicolon token or not.
+ *
+ * @param {Token} token - The token to check.
+ * @returns {boolean} `true` if the token is a semicolon token.
+ */
+function isSemicolonToken(token) {
+    return token.value === ";" && token.type === "Punctuator";
+}
 
-    return token;
+/**
+ * Checks if the given token is a colon token or not.
+ *
+ * @param {Token} token - The token to check.
+ * @returns {boolean} `true` if the token is a colon token.
+ */
+function isColonToken(token) {
+    return token.value === ":" && token.type === "Punctuator";
+}
+
+/**
+ * Checks if the given token is an opening parenthesis token or not.
+ *
+ * @param {Token} token - The token to check.
+ * @returns {boolean} `true` if the token is an opening parenthesis token.
+ */
+function isOpeningParenToken(token) {
+    return token.value === "(" && token.type === "Punctuator";
+}
+
+/**
+ * Checks if the given token is a closing parenthesis token or not.
+ *
+ * @param {Token} token - The token to check.
+ * @returns {boolean} `true` if the token is a closing parenthesis token.
+ */
+function isClosingParenToken(token) {
+    return token.value === ")" && token.type === "Punctuator";
+}
+
+/**
+ * Checks if the given token is an opening square bracket token or not.
+ *
+ * @param {Token} token - The token to check.
+ * @returns {boolean} `true` if the token is an opening square bracket token.
+ */
+function isOpeningBracketToken(token) {
+    return token.value === "[" && token.type === "Punctuator";
+}
+
+/**
+ * Checks if the given token is a closing square bracket token or not.
+ *
+ * @param {Token} token - The token to check.
+ * @returns {boolean} `true` if the token is a closing square bracket token.
+ */
+function isClosingBracketToken(token) {
+    return token.value === "]" && token.type === "Punctuator";
+}
+
+/**
+ * Checks if the given token is an opening brace token or not.
+ *
+ * @param {Token} token - The token to check.
+ * @returns {boolean} `true` if the token is an opening brace token.
+ */
+function isOpeningBraceToken(token) {
+    return token.value === "{" && token.type === "Punctuator";
+}
+
+/**
+ * Checks if the given token is a closing brace token or not.
+ *
+ * @param {Token} token - The token to check.
+ * @returns {boolean} `true` if the token is a closing brace token.
+ */
+function isClosingBraceToken(token) {
+    return token.value === "}" && token.type === "Punctuator";
+}
+
+/**
+ * Checks if the given token is a comment token or not.
+ *
+ * @param {Token} token - The token to check.
+ * @returns {boolean} `true` if the token is a comment token.
+ */
+function isCommentToken(token) {
+    return token.type === "Line" || token.type === "Block" || token.type === "Shebang";
+}
+
+/**
+ * Checks if the given token is a keyword token or not.
+ *
+ * @param {Token} token - The token to check.
+ * @returns {boolean} `true` if the token is a keyword token.
+ */
+function isKeywordToken(token) {
+    return token.type === "Keyword";
 }
 
 /**
@@ -274,13 +386,9 @@ function getArrowToken(node, sourceCode) {
  * @returns {Token} `(` token.
  */
 function getOpeningParenOfParams(node, sourceCode) {
-    let token = node.id ? sourceCode.getTokenAfter(node.id) : sourceCode.getFirstToken(node);
-
-    while (token.value !== "(") {
-        token = sourceCode.getTokenAfter(token);
-    }
-
-    return token;
+    return node.id
+        ? sourceCode.getTokenAfter(node.id, isOpeningParenToken)
+        : sourceCode.getFirstToken(node, isOpeningParenToken);
 }
 
 const lineIndexCache = new WeakMap();
@@ -345,6 +453,28 @@ module.exports = {
     isArrayFromMethod,
     isParenthesised,
 
+    isArrowToken,
+    isClosingBraceToken,
+    isClosingBracketToken,
+    isClosingParenToken,
+    isColonToken,
+    isCommaToken,
+    isCommentToken,
+    isKeywordToken,
+    isNotClosingBraceToken: negate(isClosingBraceToken),
+    isNotClosingBracketToken: negate(isClosingBracketToken),
+    isNotClosingParenToken: negate(isClosingParenToken),
+    isNotColonToken: negate(isColonToken),
+    isNotCommaToken: negate(isCommaToken),
+    isNotOpeningBraceToken: negate(isOpeningBraceToken),
+    isNotOpeningBracketToken: negate(isOpeningBracketToken),
+    isNotOpeningParenToken: negate(isOpeningParenToken),
+    isNotSemicolonToken: negate(isSemicolonToken),
+    isOpeningBraceToken,
+    isOpeningBracketToken,
+    isOpeningParenToken,
+    isSemicolonToken,
+
     /**
      * Checks whether or not a given node is a string literal.
      * @param {ASTNode} node - A node to check.
@@ -1044,7 +1174,7 @@ module.exports = {
         let end = null;
 
         if (node.type === "ArrowFunctionExpression") {
-            const arrowToken = getArrowToken(node, sourceCode);
+            const arrowToken = sourceCode.getTokenBefore(node.body, isArrowToken);
 
             start = arrowToken.loc.start;
             end = arrowToken.loc.end;

lib/rules/arrow-body-style.js

@@ -4,6 +4,12 @@
  */
 "use strict";
 
+//------------------------------------------------------------------------------
+// Requirements
+//------------------------------------------------------------------------------
+
+const astUtils = require("../ast-utils");
+
 //------------------------------------------------------------------------------
 // Rule Definition
 //------------------------------------------------------------------------------
@@ -136,10 +142,7 @@ module.exports = {
                         loc: arrowBody.loc.start,
                         message: "Expected block statement surrounding arrow body.",
                         fix(fixer) {
-                            const lastTokenBeforeBody = sourceCode.getTokensBetween(sourceCode.getFirstToken(node), arrowBody)
-                                .reverse()
-                                .find(token => token.value !== "(");
-
+                            const lastTokenBeforeBody = sourceCode.getLastTokenBetween(sourceCode.getFirstToken(node), arrowBody, astUtils.isNotOpeningParenToken);
                             const firstBodyToken = sourceCode.getTokenAfter(lastTokenBeforeBody);
 
                             return fixer.replaceTextRange(

lib/rules/arrow-spacing.js

@@ -4,6 +4,12 @@
  */
 "use strict";
 
+//------------------------------------------------------------------------------
+// Requirements
+//------------------------------------------------------------------------------
+
+const astUtils = require("../ast-utils");
+
 //------------------------------------------------------------------------------
 // Rule Definition
 //------------------------------------------------------------------------------
@@ -51,12 +57,7 @@ module.exports = {
          * @returns {Object} Tokens of arrow and before/after arrow.
          */
         function getTokens(node) {
-            let arrow = sourceCode.getTokenBefore(node.body);
-
-            // skip '(' tokens.
-            while (arrow.value !== "=>") {
-                arrow = sourceCode.getTokenBefore(arrow);
-            }
+            const arrow = sourceCode.getTokenBefore(node.body, astUtils.isArrowToken);
 
             return {
                 before: sourceCode.getTokenBefore(arrow),

lib/rules/consistent-return.js

@@ -104,7 +104,7 @@ module.exports = {
             } else if (node.type === "ArrowFunctionExpression") {
 
                 // `=>` token
-                loc = context.getSourceCode().getTokenBefore(node.body).loc.start;
+                loc = context.getSourceCode().getTokenBefore(node.body, astUtils.isArrowToken).loc.start;
                 type = "function";
             } else if (
                 node.parent.type === "MethodDefinition" ||

lib/rules/curly.js

@@ -94,19 +94,23 @@ module.exports = {
             return first.loc.start.line === last.loc.end.line;
         }
 
+        /**
+         * Checks if the given token is an `else` token or not.
+         *
+         * @param {Token} token - The token to check.
+         * @returns {boolean} `true` if the token is an `else` token.
+         */
+        function isElseKeywordToken(token) {
+            return token.value === "else" && token.type === "Keyword";
+        }
+
         /**
          * Gets the `else` keyword token of a given `IfStatement` node.
          * @param {ASTNode} node - A `IfStatement` node to get.
          * @returns {Token} The `else` keyword token.
          */
         function getElseKeyword(node) {
-            let token = sourceCode.getTokenAfter(node.consequent);
-
-            while (token.type !== "Keyword" || token.value !== "else") {
-                token = sourceCode.getTokenAfter(token);
-            }
-
-            return token;
+            return node.alternate && sourceCode.getFirstTokenBetween(node.consequent, node.alternate, isElseKeywordToken);
         }
 
         /**

lib/rules/eqeqeq.js

@@ -134,7 +134,11 @@ module.exports = {
 
                     // If the comparison is a `typeof` comparison or both sides are literals with the same type, then it's safe to fix.
                     if (isTypeOfBinary(node) || areLiteralsAndSameType(node)) {
-                        const operatorToken = sourceCode.getTokensBetween(node.left, node.right).find(token => token.value === node.operator);
+                        const operatorToken = sourceCode.getFirstTokenBetween(
+                            node.left,
+                            node.right,
+                            token => token.value === node.operator
+                        );
 
                         return fixer.replaceText(operatorToken, expectedOperator);
                     }

lib/rules/func-call-spacing.js

@@ -5,6 +5,12 @@
 
 "use strict";
 
+//------------------------------------------------------------------------------
+// Requirements
+//------------------------------------------------------------------------------
+
+const astUtils = require("../ast-utils");
+
 //------------------------------------------------------------------------------
 // Rule Definition
 //------------------------------------------------------------------------------
@@ -67,19 +73,10 @@ module.exports = {
          * @private
          */
         function checkSpacing(node) {
+            const lastToken = sourceCode.getLastToken(node);
             const lastCalleeToken = sourceCode.getLastToken(node.callee);
-            let prevToken = lastCalleeToken;
-            let parenToken = sourceCode.getTokenAfter(lastCalleeToken);
-
-            // advances to an open parenthesis.
-            while (
-                parenToken &&
-                parenToken.range[1] < node.range[1] &&
-                parenToken.value !== "("
-            ) {
-                prevToken = parenToken;
-                parenToken = sourceCode.getTokenAfter(parenToken);
-            }
+            const parenToken = sourceCode.getFirstTokenBetween(lastCalleeToken, lastToken, astUtils.isOpeningParenToken);
+            const prevToken = parenToken && sourceCode.getTokenBefore(parenToken);
 
             // Parens in NewExpression are optional
             if (!(parenToken && parenToken.range[1] < node.range[1])) {