mirror of
https://github.com/jashkenas/coffeescript.git
synced 2022-11-09 12:23:24 -05:00
ce971b766f
This commit adds another post-processing step after normal lexing that sets the locationData on all OUTDENT tokens to be at the last character of the previous token. This does feel like a little bit of a hack. Ideally the location data would be set correctly in the first place and not in a post-processing step, but I tried that and some temporary intermediate tokens were causing problems, so I decided to set the location data once those intermediate tokens were removed. Also, having this as a separate processing step makes it more robust and isolated. This fixes the problem in https://github.com/decaffeinate/decaffeinate/issues/371 . In that issue, the CoffeeScript tokens had three OUTDENT tokens in a row, and the last two overlapped with the `]`. Since at least one of those OUTDENT tokens was considered part of the function body, the function expression had an ending position just after the end of the `]`. OUTDENT tokens are sort of a weird case in the lexer anyway, since they often don't correspond to an actual location in the source code. It seems like the code in `lexer.coffee` makes an attempt at finding a good place for them, but in some cases, it has a bad result. This seems hard to avoid in the general case. For example, in this code: ```coffee [-> a] ``` There must be an OUTDENT between the `a` and the `]`, but CoffeeScript tokens have an inclusive start and end, so they must always be at least one character wide (I think). In this case, the lexer was choosing the `]` as the location, and the parser ended up generating correct location data, I believe because it ignores the outermost INDENT and OUTDENT tokens. However, with multiple OUTDENT tokens in a row, the parser ends up producing location data that is wrong. It seems to me like there isn't a solid answer to "what location do OUTDENT tokens have", since it hasn't mattered much, but for this commit, I'm defining it: they always have the location of the last character of the previous token. This should hopefully be fairly safe because tokens are still in the same order relative to each other. Also, it's worth noting that this makes the start location for OUTDENT tokens awkward. However, OUTDENT tokens are always used to mark the end of something, so their `last_line` and `last_column` values are always what matter when determining AST node bounds, so it is most important for those to be correct.
494 lines
13 KiB
CoffeeScript
494 lines
13 KiB
CoffeeScript
testScript = '''
|
|
if true
|
|
x = 6
|
|
console.log "A console #{x + 7} log"
|
|
|
|
foo = "bar"
|
|
z = /// ^ (a#{foo}) ///
|
|
|
|
x = () ->
|
|
try
|
|
console.log "foo"
|
|
catch err
|
|
# Rewriter will generate explicit indentation here.
|
|
|
|
return null
|
|
'''
|
|
|
|
test "Verify location of generated tokens", ->
|
|
tokens = CoffeeScript.tokens "a = 79"
|
|
|
|
eq tokens.length, 4
|
|
[aToken, equalsToken, numberToken] = tokens
|
|
|
|
eq aToken[2].first_line, 0
|
|
eq aToken[2].first_column, 0
|
|
eq aToken[2].last_line, 0
|
|
eq aToken[2].last_column, 0
|
|
|
|
eq equalsToken[2].first_line, 0
|
|
eq equalsToken[2].first_column, 2
|
|
eq equalsToken[2].last_line, 0
|
|
eq equalsToken[2].last_column, 2
|
|
|
|
eq numberToken[2].first_line, 0
|
|
eq numberToken[2].first_column, 4
|
|
eq numberToken[2].last_line, 0
|
|
eq numberToken[2].last_column, 5
|
|
|
|
test "Verify location of generated tokens (with indented first line)", ->
|
|
tokens = CoffeeScript.tokens " a = 83"
|
|
|
|
eq tokens.length, 4
|
|
[aToken, equalsToken, numberToken] = tokens
|
|
|
|
eq aToken[2].first_line, 0
|
|
eq aToken[2].first_column, 2
|
|
eq aToken[2].last_line, 0
|
|
eq aToken[2].last_column, 2
|
|
|
|
eq equalsToken[2].first_line, 0
|
|
eq equalsToken[2].first_column, 4
|
|
eq equalsToken[2].last_line, 0
|
|
eq equalsToken[2].last_column, 4
|
|
|
|
eq numberToken[2].first_line, 0
|
|
eq numberToken[2].first_column, 6
|
|
eq numberToken[2].last_line, 0
|
|
eq numberToken[2].last_column, 7
|
|
|
|
getMatchingTokens = (str, wantedTokens...) ->
|
|
tokens = CoffeeScript.tokens str
|
|
matchingTokens = []
|
|
i = 0
|
|
for token in tokens
|
|
if token[1].replace(/^'|'$/g, '"') is wantedTokens[i]
|
|
i++
|
|
matchingTokens.push token
|
|
eq wantedTokens.length, matchingTokens.length
|
|
matchingTokens
|
|
|
|
test 'Verify locations in string interpolation (in "string")', ->
|
|
[a, b, c] = getMatchingTokens '"a#{b}c"', '"a"', 'b', '"c"'
|
|
|
|
eq a[2].first_line, 0
|
|
eq a[2].first_column, 0
|
|
eq a[2].last_line, 0
|
|
eq a[2].last_column, 1
|
|
|
|
eq b[2].first_line, 0
|
|
eq b[2].first_column, 4
|
|
eq b[2].last_line, 0
|
|
eq b[2].last_column, 4
|
|
|
|
eq c[2].first_line, 0
|
|
eq c[2].first_column, 6
|
|
eq c[2].last_line, 0
|
|
eq c[2].last_column, 7
|
|
|
|
test 'Verify locations in string interpolation (in "string", multiple interpolation)', ->
|
|
[a, b, c] = getMatchingTokens '"#{a}b#{c}"', 'a', '"b"', 'c'
|
|
|
|
eq a[2].first_line, 0
|
|
eq a[2].first_column, 3
|
|
eq a[2].last_line, 0
|
|
eq a[2].last_column, 3
|
|
|
|
eq b[2].first_line, 0
|
|
eq b[2].first_column, 5
|
|
eq b[2].last_line, 0
|
|
eq b[2].last_column, 5
|
|
|
|
eq c[2].first_line, 0
|
|
eq c[2].first_column, 8
|
|
eq c[2].last_line, 0
|
|
eq c[2].last_column, 8
|
|
|
|
test 'Verify locations in string interpolation (in "string", multiple interpolation and line breaks)', ->
|
|
[a, b, c] = getMatchingTokens '"#{a}\nb\n#{c}"', 'a', '" b "', 'c'
|
|
|
|
eq a[2].first_line, 0
|
|
eq a[2].first_column, 3
|
|
eq a[2].last_line, 0
|
|
eq a[2].last_column, 3
|
|
|
|
eq b[2].first_line, 0
|
|
eq b[2].first_column, 5
|
|
eq b[2].last_line, 1
|
|
eq b[2].last_column, 1
|
|
|
|
eq c[2].first_line, 2
|
|
eq c[2].first_column, 2
|
|
eq c[2].last_line, 2
|
|
eq c[2].last_column, 2
|
|
|
|
test 'Verify locations in string interpolation (in "string", multiple interpolation and starting with line breaks)', ->
|
|
[a, b, c] = getMatchingTokens '"\n#{a}\nb\n#{c}"', 'a', '" b "', 'c'
|
|
|
|
eq a[2].first_line, 1
|
|
eq a[2].first_column, 2
|
|
eq a[2].last_line, 1
|
|
eq a[2].last_column, 2
|
|
|
|
eq b[2].first_line, 1
|
|
eq b[2].first_column, 4
|
|
eq b[2].last_line, 2
|
|
eq b[2].last_column, 1
|
|
|
|
eq c[2].first_line, 3
|
|
eq c[2].first_column, 2
|
|
eq c[2].last_line, 3
|
|
eq c[2].last_column, 2
|
|
|
|
test 'Verify locations in string interpolation (in "string", multiple interpolation and starting with line breaks)', ->
|
|
[a, b, c] = getMatchingTokens '"\n\n#{a}\n\nb\n\n#{c}"', 'a', '" b "', 'c'
|
|
|
|
eq a[2].first_line, 2
|
|
eq a[2].first_column, 2
|
|
eq a[2].last_line, 2
|
|
eq a[2].last_column, 2
|
|
|
|
eq b[2].first_line, 2
|
|
eq b[2].first_column, 4
|
|
eq b[2].last_line, 5
|
|
eq b[2].last_column, 0
|
|
|
|
eq c[2].first_line, 6
|
|
eq c[2].first_column, 2
|
|
eq c[2].last_line, 6
|
|
eq c[2].last_column, 2
|
|
|
|
test 'Verify locations in string interpolation (in "string", multiple interpolation and starting with line breaks)', ->
|
|
[a, b, c] = getMatchingTokens '"\n\n\n#{a}\n\n\nb\n\n\n#{c}"', 'a', '" b "', 'c'
|
|
|
|
eq a[2].first_line, 3
|
|
eq a[2].first_column, 2
|
|
eq a[2].last_line, 3
|
|
eq a[2].last_column, 2
|
|
|
|
eq b[2].first_line, 3
|
|
eq b[2].first_column, 4
|
|
eq b[2].last_line, 8
|
|
eq b[2].last_column, 0
|
|
|
|
eq c[2].first_line, 9
|
|
eq c[2].first_column, 2
|
|
eq c[2].last_line, 9
|
|
eq c[2].last_column, 2
|
|
|
|
test 'Verify locations in string interpolation (in """string""", line breaks)', ->
|
|
[a, b, c] = getMatchingTokens '"""a\n#{b}\nc"""', '"a\\n"', 'b', '"\\nc"'
|
|
|
|
eq a[2].first_line, 0
|
|
eq a[2].first_column, 0
|
|
eq a[2].last_line, 0
|
|
eq a[2].last_column, 4
|
|
|
|
eq b[2].first_line, 1
|
|
eq b[2].first_column, 2
|
|
eq b[2].last_line, 1
|
|
eq b[2].last_column, 2
|
|
|
|
eq c[2].first_line, 1
|
|
eq c[2].first_column, 4
|
|
eq c[2].last_line, 2
|
|
eq c[2].last_column, 3
|
|
|
|
test 'Verify locations in string interpolation (in """string""", starting with a line break)', ->
|
|
[b, c] = getMatchingTokens '"""\n#{b}\nc"""', 'b', '"\\nc"'
|
|
|
|
eq b[2].first_line, 1
|
|
eq b[2].first_column, 2
|
|
eq b[2].last_line, 1
|
|
eq b[2].last_column, 2
|
|
|
|
eq c[2].first_line, 1
|
|
eq c[2].first_column, 4
|
|
eq c[2].last_line, 2
|
|
eq c[2].last_column, 3
|
|
|
|
test 'Verify locations in string interpolation (in """string""", starting with line breaks)', ->
|
|
[a, b, c] = getMatchingTokens '"""\n\n#{b}\nc"""', '"\\n"', 'b', '"\\nc"'
|
|
|
|
eq a[2].first_line, 0
|
|
eq a[2].first_column, 0
|
|
eq a[2].last_line, 1
|
|
eq a[2].last_column, 0
|
|
|
|
eq b[2].first_line, 2
|
|
eq b[2].first_column, 2
|
|
eq b[2].last_line, 2
|
|
eq b[2].last_column, 2
|
|
|
|
eq c[2].first_line, 2
|
|
eq c[2].first_column, 4
|
|
eq c[2].last_line, 3
|
|
eq c[2].last_column, 3
|
|
|
|
test 'Verify locations in string interpolation (in """string""", multiple interpolation)', ->
|
|
[a, b, c] = getMatchingTokens '"""#{a}\nb\n#{c}"""', 'a', '"\\nb\\n"', 'c'
|
|
|
|
eq a[2].first_line, 0
|
|
eq a[2].first_column, 5
|
|
eq a[2].last_line, 0
|
|
eq a[2].last_column, 5
|
|
|
|
eq b[2].first_line, 0
|
|
eq b[2].first_column, 7
|
|
eq b[2].last_line, 1
|
|
eq b[2].last_column, 1
|
|
|
|
eq c[2].first_line, 2
|
|
eq c[2].first_column, 2
|
|
eq c[2].last_line, 2
|
|
eq c[2].last_column, 2
|
|
|
|
test 'Verify locations in string interpolation (in """string""", multiple interpolation, and starting with line breaks)', ->
|
|
[a, b, c] = getMatchingTokens '"""\n\n#{a}\n\nb\n\n#{c}"""', 'a', '"\\n\\nb\\n\\n"', 'c'
|
|
|
|
eq a[2].first_line, 2
|
|
eq a[2].first_column, 2
|
|
eq a[2].last_line, 2
|
|
eq a[2].last_column, 2
|
|
|
|
eq b[2].first_line, 2
|
|
eq b[2].first_column, 4
|
|
eq b[2].last_line, 5
|
|
eq b[2].last_column, 0
|
|
|
|
eq c[2].first_line, 6
|
|
eq c[2].first_column, 2
|
|
eq c[2].last_line, 6
|
|
eq c[2].last_column, 2
|
|
|
|
test 'Verify locations in string interpolation (in """string""", multiple interpolation, and starting with line breaks)', ->
|
|
[a, b, c] = getMatchingTokens '"""\n\n\n#{a}\n\n\nb\n\n\n#{c}"""', 'a', '"\\n\\n\\nb\\n\\n\\n"', 'c'
|
|
|
|
eq a[2].first_line, 3
|
|
eq a[2].first_column, 2
|
|
eq a[2].last_line, 3
|
|
eq a[2].last_column, 2
|
|
|
|
eq b[2].first_line, 3
|
|
eq b[2].first_column, 4
|
|
eq b[2].last_line, 8
|
|
eq b[2].last_column, 0
|
|
|
|
eq c[2].first_line, 9
|
|
eq c[2].first_column, 2
|
|
eq c[2].last_line, 9
|
|
eq c[2].last_column, 2
|
|
|
|
test 'Verify locations in heregex interpolation (in ///regex///, multiple interpolation)', ->
|
|
[a, b, c] = getMatchingTokens '///#{a}b#{c}///', 'a', '"b"', 'c'
|
|
|
|
eq a[2].first_line, 0
|
|
eq a[2].first_column, 5
|
|
eq a[2].last_line, 0
|
|
eq a[2].last_column, 5
|
|
|
|
eq b[2].first_line, 0
|
|
eq b[2].first_column, 7
|
|
eq b[2].last_line, 0
|
|
eq b[2].last_column, 7
|
|
|
|
eq c[2].first_line, 0
|
|
eq c[2].first_column, 10
|
|
eq c[2].last_line, 0
|
|
eq c[2].last_column, 10
|
|
|
|
test 'Verify locations in heregex interpolation (in ///regex///, multiple interpolation)', ->
|
|
[a, b, c] = getMatchingTokens '///a#{b}c///', '"a"', 'b', '"c"'
|
|
|
|
eq a[2].first_line, 0
|
|
eq a[2].first_column, 0
|
|
eq a[2].last_line, 0
|
|
eq a[2].last_column, 3
|
|
|
|
eq b[2].first_line, 0
|
|
eq b[2].first_column, 6
|
|
eq b[2].last_line, 0
|
|
eq b[2].last_column, 6
|
|
|
|
eq c[2].first_line, 0
|
|
eq c[2].first_column, 8
|
|
eq c[2].last_line, 0
|
|
eq c[2].last_column, 11
|
|
|
|
test 'Verify locations in heregex interpolation (in ///regex///, multiple interpolation and line breaks)', ->
|
|
[a, b, c] = getMatchingTokens '///#{a}\nb\n#{c}///', 'a', '"b"', 'c'
|
|
|
|
eq a[2].first_line, 0
|
|
eq a[2].first_column, 5
|
|
eq a[2].last_line, 0
|
|
eq a[2].last_column, 5
|
|
|
|
eq b[2].first_line, 0
|
|
eq b[2].first_column, 7
|
|
eq b[2].last_line, 1
|
|
eq b[2].last_column, 1
|
|
|
|
eq c[2].first_line, 2
|
|
eq c[2].first_column, 2
|
|
eq c[2].last_line, 2
|
|
eq c[2].last_column, 2
|
|
|
|
test 'Verify locations in heregex interpolation (in ///regex///, multiple interpolation and line breaks)', ->
|
|
[a, b, c] = getMatchingTokens '///#{a}\n\n\nb\n\n\n#{c}///', 'a', '"b"', 'c'
|
|
|
|
eq a[2].first_line, 0
|
|
eq a[2].first_column, 5
|
|
eq a[2].last_line, 0
|
|
eq a[2].last_column, 5
|
|
|
|
eq b[2].first_line, 0
|
|
eq b[2].first_column, 7
|
|
eq b[2].last_line, 5
|
|
eq b[2].last_column, 0
|
|
|
|
eq c[2].first_line, 6
|
|
eq c[2].first_column, 2
|
|
eq c[2].last_line, 6
|
|
eq c[2].last_column, 2
|
|
|
|
test 'Verify locations in heregex interpolation (in ///regex///, multiple interpolation and line breaks)', ->
|
|
[a, b, c] = getMatchingTokens '///a\n\n\n#{b}\n\n\nc///', '"a"', 'b', '"c"'
|
|
|
|
eq a[2].first_line, 0
|
|
eq a[2].first_column, 0
|
|
eq a[2].last_line, 2
|
|
eq a[2].last_column, 0
|
|
|
|
eq b[2].first_line, 3
|
|
eq b[2].first_column, 2
|
|
eq b[2].last_line, 3
|
|
eq b[2].last_column, 2
|
|
|
|
eq c[2].first_line, 3
|
|
eq c[2].first_column, 4
|
|
eq c[2].last_line, 6
|
|
eq c[2].last_column, 3
|
|
|
|
test 'Verify locations in heregex interpolation (in ///regex///, multiple interpolation and line breaks and starting with linebreak)', ->
|
|
[a, b, c] = getMatchingTokens '///\n#{a}\nb\n#{c}///', 'a', '"b"', 'c'
|
|
|
|
eq a[2].first_line, 1
|
|
eq a[2].first_column, 2
|
|
eq a[2].last_line, 1
|
|
eq a[2].last_column, 2
|
|
|
|
eq b[2].first_line, 1
|
|
eq b[2].first_column, 4
|
|
eq b[2].last_line, 2
|
|
eq b[2].last_column, 1
|
|
|
|
eq c[2].first_line, 3
|
|
eq c[2].first_column, 2
|
|
eq c[2].last_line, 3
|
|
eq c[2].last_column, 2
|
|
|
|
test 'Verify locations in heregex interpolation (in ///regex///, multiple interpolation and line breaks and starting with linebreak)', ->
|
|
[a, b, c] = getMatchingTokens '///\n\n\n#{a}\n\n\nb\n\n\n#{c}///', 'a', '"b"', 'c'
|
|
|
|
eq a[2].first_line, 3
|
|
eq a[2].first_column, 2
|
|
eq a[2].last_line, 3
|
|
eq a[2].last_column, 2
|
|
|
|
eq b[2].first_line, 3
|
|
eq b[2].first_column, 4
|
|
eq b[2].last_line, 8
|
|
eq b[2].last_column, 0
|
|
|
|
eq c[2].first_line, 9
|
|
eq c[2].first_column, 2
|
|
eq c[2].last_line, 9
|
|
eq c[2].last_column, 2
|
|
|
|
test 'Verify locations in heregex interpolation (in ///regex///, multiple interpolation and line breaks and starting with linebreak)', ->
|
|
[a, b, c] = getMatchingTokens '///\n\n\na\n\n\n#{b}\n\n\nc///', '"a"', 'b', '"c"'
|
|
|
|
eq a[2].first_line, 0
|
|
eq a[2].first_column, 0
|
|
eq a[2].last_line, 5
|
|
eq a[2].last_column, 0
|
|
|
|
eq b[2].first_line, 6
|
|
eq b[2].first_column, 2
|
|
eq b[2].last_line, 6
|
|
eq b[2].last_column, 2
|
|
|
|
eq c[2].first_line, 6
|
|
eq c[2].first_column, 4
|
|
eq c[2].last_line, 9
|
|
eq c[2].last_column, 3
|
|
|
|
test "#3822: Simple string/regex start/end should include delimiters", ->
|
|
[stringToken] = CoffeeScript.tokens "'string'"
|
|
eq stringToken[2].first_line, 0
|
|
eq stringToken[2].first_column, 0
|
|
eq stringToken[2].last_line, 0
|
|
eq stringToken[2].last_column, 7
|
|
|
|
[regexToken] = CoffeeScript.tokens "/regex/"
|
|
eq regexToken[2].first_line, 0
|
|
eq regexToken[2].first_column, 0
|
|
eq regexToken[2].last_line, 0
|
|
eq regexToken[2].last_column, 6
|
|
|
|
test "#3621: Multiline regex and manual `Regex` call with interpolation should
|
|
result in the same tokens", ->
|
|
tokensA = CoffeeScript.tokens '(RegExp(".*#{a}[0-9]"))'
|
|
tokensB = CoffeeScript.tokens '///.*#{a}[0-9]///'
|
|
eq tokensA.length, tokensB.length
|
|
for i in [0...tokensA.length] by 1
|
|
tokenA = tokensA[i]
|
|
tokenB = tokensB[i]
|
|
eq tokenA[0], tokenB[0] unless tokenB[0] in ['REGEX_START', 'REGEX_END']
|
|
eq tokenA[1], tokenB[1]
|
|
unless tokenA[0] is 'STRING_START' or tokenB[0] is 'REGEX_START'
|
|
eq tokenA.origin?[1], tokenB.origin?[1]
|
|
eq tokenA.stringEnd, tokenB.stringEnd
|
|
|
|
test "Verify tokens have locations that are in order", ->
|
|
source = '''
|
|
a {
|
|
b: ->
|
|
return c d,
|
|
if e
|
|
f
|
|
}
|
|
g
|
|
'''
|
|
tokens = CoffeeScript.tokens source
|
|
lastToken = null
|
|
for token in tokens
|
|
if lastToken
|
|
ok token[2].first_line >= lastToken[2].last_line
|
|
if token[2].first_line == lastToken[2].last_line
|
|
ok token[2].first_column >= lastToken[2].last_column
|
|
lastToken = token
|
|
|
|
test "Verify OUTDENT tokens are located at the end of the previous token", ->
|
|
source = '''
|
|
SomeArr = [ ->
|
|
if something
|
|
lol =
|
|
count: 500
|
|
]
|
|
'''
|
|
tokens = CoffeeScript.tokens source
|
|
[..., number, curly, outdent1, outdent2, outdent3, bracket, terminator] = tokens
|
|
eq number[0], 'NUMBER'
|
|
for outdent in [outdent1, outdent2, outdent3]
|
|
eq outdent[0], 'OUTDENT'
|
|
eq outdent[2].first_line, number[2].last_line
|
|
eq outdent[2].first_column, number[2].last_column
|
|
eq outdent[2].last_line, number[2].last_line
|
|
eq outdent[2].last_column, number[2].last_column
|
|
|
|
test "Verify all tokens get a location", ->
|
|
doesNotThrow ->
|
|
tokens = CoffeeScript.tokens testScript
|
|
for token in tokens
|
|
ok !!token[2]
|