Properly set location for string tokens ending in a newline (#4344)

This is an upstream port of https://github.com/decaffeinate/coffeescript/pull/9

The existing logic for computing the end location of a string was to take the
end of the string contents, then add the delimiter length to last_column. For
example, `"""abc"""` would have an end position three characters after the `c`.
However, if a string ended in a newline, then the end location for the string
contents would be one line above the end location for the string, so the proper
fix is to move the end location to the next line, not just to shift it to the
right.

This avoids a bug where the location data would sometimes reference a
non-existent location (one past the end of its line). It fixes the AST location
data, although as far as I know, it never has caused correctness issues in the
CoffeeScript output.

lib/coffee-script/lexer.js

@@ -685,7 +685,12 @@
       }
       firstToken = tokens[0], lastToken = tokens[tokens.length - 1];
       firstToken[2].first_column -= delimiter.length;
-      lastToken[2].last_column += delimiter.length;
+      if (lastToken[1].substr(-1) === '\n') {
+        lastToken[2].last_line += 1;
+        lastToken[2].last_column = delimiter.length - 1;
+      } else {
+        lastToken[2].last_column += delimiter.length;
+      }
       if (lastToken[1].length === 0) {
         lastToken[2].last_column -= 1;
       }

src/lexer.coffee

@@ -573,7 +573,11 @@ exports.Lexer = class Lexer
 
     [firstToken, ..., lastToken] = tokens
     firstToken[2].first_column -= delimiter.length
-    lastToken[2].last_column += delimiter.length
+    if lastToken[1].substr(-1) is '\n'
+      lastToken[2].last_line += 1
+      lastToken[2].last_column = delimiter.length - 1
+    else
+      lastToken[2].last_column += delimiter.length
     lastToken[2].last_column -= 1 if lastToken[1].length is 0
 
     {tokens, index: offsetInChunk + delimiter.length}

test/location.coffee

@@ -528,6 +528,42 @@ test "Verify real CALL_END tokens have the right position", ->
   eq callEnd[2].first_column, startIndex + 2
   eq callEnd[2].last_column, startIndex + 2
 
+test "Verify normal heredocs have the right position", ->
+  source = '''
+    """
+    a"""
+  '''
+  [stringToken] = CoffeeScript.tokens source
+  eq stringToken[2].first_line, 0
+  eq stringToken[2].first_column, 0
+  eq stringToken[2].last_line, 1
+  eq stringToken[2].last_column, 3
+
+test "Verify heredocs ending with a newline have the right position", ->
+  source = '''
+    """
+    a
+    """
+  '''
+  [stringToken] = CoffeeScript.tokens source
+  eq stringToken[2].first_line, 0
+  eq stringToken[2].first_column, 0
+  eq stringToken[2].last_line, 2
+  eq stringToken[2].last_column, 2
+
+test "Verify indented heredocs have the right position", ->
+  source = '''
+    ->
+      """
+        a
+      """
+  '''
+  [arrow, indent, stringToken] = CoffeeScript.tokens source
+  eq stringToken[2].first_line, 1
+  eq stringToken[2].first_column, 2
+  eq stringToken[2].last_line, 3
+  eq stringToken[2].last_column, 4
+
 test "Verify all tokens get a location", ->
   doesNotThrow ->
     tokens = CoffeeScript.tokens testScript