Properly set location for string tokens ending in a newline (#4344)
This is an upstream port of https://github.com/decaffeinate/coffeescript/pull/9 The existing logic for computing the end location of a string was to take the end of the string contents, then add the delimiter length to last_column. For example, `"""abc"""` would have an end position three characters after the `c`. However, if a string ended in a newline, then the end location for the string contents would be one line above the end location for the string, so the proper fix is to move the end location to the next line, not just to shift it to the right. This avoids a bug where the location data would sometimes reference a non-existent location (one past the end of its line). It fixes the AST location data, although as far as I know, it never has caused correctness issues in the CoffeeScript output.
This commit is contained in:
parent
3c42b400a1
commit
6087c2c8fc
|
@ -685,7 +685,12 @@
|
||||||
}
|
}
|
||||||
firstToken = tokens[0], lastToken = tokens[tokens.length - 1];
|
firstToken = tokens[0], lastToken = tokens[tokens.length - 1];
|
||||||
firstToken[2].first_column -= delimiter.length;
|
firstToken[2].first_column -= delimiter.length;
|
||||||
lastToken[2].last_column += delimiter.length;
|
if (lastToken[1].substr(-1) === '\n') {
|
||||||
|
lastToken[2].last_line += 1;
|
||||||
|
lastToken[2].last_column = delimiter.length - 1;
|
||||||
|
} else {
|
||||||
|
lastToken[2].last_column += delimiter.length;
|
||||||
|
}
|
||||||
if (lastToken[1].length === 0) {
|
if (lastToken[1].length === 0) {
|
||||||
lastToken[2].last_column -= 1;
|
lastToken[2].last_column -= 1;
|
||||||
}
|
}
|
||||||
|
|
|
@ -573,7 +573,11 @@ exports.Lexer = class Lexer
|
||||||
|
|
||||||
[firstToken, ..., lastToken] = tokens
|
[firstToken, ..., lastToken] = tokens
|
||||||
firstToken[2].first_column -= delimiter.length
|
firstToken[2].first_column -= delimiter.length
|
||||||
lastToken[2].last_column += delimiter.length
|
if lastToken[1].substr(-1) is '\n'
|
||||||
|
lastToken[2].last_line += 1
|
||||||
|
lastToken[2].last_column = delimiter.length - 1
|
||||||
|
else
|
||||||
|
lastToken[2].last_column += delimiter.length
|
||||||
lastToken[2].last_column -= 1 if lastToken[1].length is 0
|
lastToken[2].last_column -= 1 if lastToken[1].length is 0
|
||||||
|
|
||||||
{tokens, index: offsetInChunk + delimiter.length}
|
{tokens, index: offsetInChunk + delimiter.length}
|
||||||
|
|
|
@ -528,6 +528,42 @@ test "Verify real CALL_END tokens have the right position", ->
|
||||||
eq callEnd[2].first_column, startIndex + 2
|
eq callEnd[2].first_column, startIndex + 2
|
||||||
eq callEnd[2].last_column, startIndex + 2
|
eq callEnd[2].last_column, startIndex + 2
|
||||||
|
|
||||||
|
test "Verify normal heredocs have the right position", ->
|
||||||
|
source = '''
|
||||||
|
"""
|
||||||
|
a"""
|
||||||
|
'''
|
||||||
|
[stringToken] = CoffeeScript.tokens source
|
||||||
|
eq stringToken[2].first_line, 0
|
||||||
|
eq stringToken[2].first_column, 0
|
||||||
|
eq stringToken[2].last_line, 1
|
||||||
|
eq stringToken[2].last_column, 3
|
||||||
|
|
||||||
|
test "Verify heredocs ending with a newline have the right position", ->
|
||||||
|
source = '''
|
||||||
|
"""
|
||||||
|
a
|
||||||
|
"""
|
||||||
|
'''
|
||||||
|
[stringToken] = CoffeeScript.tokens source
|
||||||
|
eq stringToken[2].first_line, 0
|
||||||
|
eq stringToken[2].first_column, 0
|
||||||
|
eq stringToken[2].last_line, 2
|
||||||
|
eq stringToken[2].last_column, 2
|
||||||
|
|
||||||
|
test "Verify indented heredocs have the right position", ->
|
||||||
|
source = '''
|
||||||
|
->
|
||||||
|
"""
|
||||||
|
a
|
||||||
|
"""
|
||||||
|
'''
|
||||||
|
[arrow, indent, stringToken] = CoffeeScript.tokens source
|
||||||
|
eq stringToken[2].first_line, 1
|
||||||
|
eq stringToken[2].first_column, 2
|
||||||
|
eq stringToken[2].last_line, 3
|
||||||
|
eq stringToken[2].last_column, 4
|
||||||
|
|
||||||
test "Verify all tokens get a location", ->
|
test "Verify all tokens get a location", ->
|
||||||
doesNotThrow ->
|
doesNotThrow ->
|
||||||
tokens = CoffeeScript.tokens testScript
|
tokens = CoffeeScript.tokens testScript
|
||||||
|
|
Loading…
Reference in New Issue