Fix: Handle new lines inside block string correctly (#1637)

* test(lexer): test if lexer advance line after block string

* fix(lexer): advance line, update lineStart inside block string

fix #1636

* test(lexer): more multiline block string cases

src/language/__tests__/lexer-test.js

@@ -21,6 +21,12 @@ function lexOne(str) {
   return lexer.advance();
 }
 
+function lexSecond(str) {
+  const lexer = createLexer(new Source(str));
+  lexer.advance();
+  return lexer.advance();
+}
+
 function expectSyntaxError(text, message, location) {
   expect(() => lexOne(text))
     .to.throw('Syntax Error: ' + message)
@@ -359,6 +365,44 @@ describe('Lexer', () => {
     });
   });
 
+  it('advance line after lexing multiline block string', () => {
+    expect(
+      lexSecond(`"""
+
+        spans
+          multiple
+            lines
+
+        \n """ second_token`),
+    ).to.contain({
+      kind: TokenKind.NAME,
+      start: 71,
+      end: 83,
+      line: 8,
+      column: 6,
+      value: 'second_token',
+    });
+
+    expect(
+      lexSecond(
+        [
+          '""" \n',
+          'spans \r\n',
+          'multiple \n\r',
+          'lines \n\n',
+          '"""\n second_token',
+        ].join(''),
+      ),
+    ).to.contain({
+      kind: TokenKind.NAME,
+      start: 37,
+      end: 49,
+      line: 8,
+      column: 2,
+      value: 'second_token',
+    });
+  });
+
   it('lex reports useful block string errors', () => {
     expectSyntaxError('"""', 'Unterminated string.', { line: 1, column: 4 });
 

src/language/lexer.js

@@ -334,7 +334,7 @@ function readToken(lexer: Lexer<*>, prev: Token): Token {
         charCodeAt.call(body, pos + 1) === 34 &&
         charCodeAt.call(body, pos + 2) === 34
       ) {
-        return readBlockString(source, pos, line, col, prev);
+        return readBlockString(source, pos, line, col, prev, lexer);
       }
       return readString(source, pos, line, col, prev);
   }
@@ -625,7 +625,7 @@ function readString(source, start, line, col, prev): Token {
  *
  * """("?"?(\\"""|\\(?!=""")|[^"\\]))*"""
  */
-function readBlockString(source, start, line, col, prev): Token {
+function readBlockString(source, start, line, col, prev, lexer): Token {
   const body = source.body;
   let position = start + 3;
   let chunkStart = position;
@@ -668,8 +668,22 @@ function readBlockString(source, start, line, col, prev): Token {
       );
     }
 
-    // Escape Triple-Quote (\""")
-    if (
+    if (code === 10) {
+      // new line
+      ++position;
+      ++lexer.line;
+      lexer.lineStart = position;
+    } else if (code === 13) {
+      // carriage return
+      if (charCodeAt.call(body, position + 1) === 10) {
+        position += 2;
+      } else {
+        ++position;
+      }
+      ++lexer.line;
+      lexer.lineStart = position;
+    } else if (
+      // Escape Triple-Quote (\""")
       code === 92 &&
       charCodeAt.call(body, position + 1) === 34 &&
       charCodeAt.call(body, position + 2) === 34 &&