kotovalexarian-likes-github
/
jashkenas--coffeescript
mirror of https://github.com/jashkenas/coffeescript.git
1
0
Fork 0
Code Releases Activity

Handle implicit object as end of outer implicit object property value (#5296) 

* not continuing implicit object if after UNFINISHED

* following property not working

* handle following object properties

* tests

* indebt -> continuationLineAdditionalIndent
This commit is contained in:
Julian Rosse 2020-01-29 16:20:40 -08:00 committed by GitHub
parent 92ad04b9b1
commit 9cef39d21a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 98 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
lib/coffeescript/lexer.js
View File

@ -14,7 +14,7 @@
indexOf = [].indexOf,
slice = [].slice;
({Rewriter, INVERSES} = require('./rewriter'));
({Rewriter, INVERSES, UNFINISHED} = require('./rewriter'));
// Import the helpers we need.
({count, starts, compact, repeat, invertLiterate, merge, attachCommentsToNode, locationDataToString, throwSyntaxError, replaceUnicodeCodePointEscapes, flatten, parseNumber} = require('./helpers'));
@ -47,7 +47,7 @@
this.literate = opts.literate; // Are we lexing literate CoffeeScript?
this.indent = 0; // The current indentation level.
this.baseIndent = 0; // The overall minimum indentation level.
this.indebt = 0; // The over-indentation at the current level.
this.continuationLineAdditionalIndent = 0; // The over-indentation at the current level.
this.outdebt = 0; // The under-outdentation at the current level.
this.indents = []; // The stack of all current indentation levels.
this.indentLiteral = ''; // The indentation.
@ -715,7 +715,7 @@
// Keeps track of the level of indentation, because a single outdent token
// can close multiple indents, so we need to know how far in we happen to be.
lineToken({chunk = this.chunk, offset = 0} = {}) {
var backslash, diff, indent, match, minLiteralLength, newIndentLiteral, noNewlines, prev, ref, size;
var backslash, diff, endsContinuationLineIndentation, indent, match, minLiteralLength, newIndentLiteral, noNewlines, prev, ref, size;
if (!(match = MULTI_DENT.exec(chunk))) {
return 0;
}
@ -747,7 +747,7 @@
});
return indent.length;
}
if (size - this.indebt === this.indent) {
if (size - this.continuationLineAdditionalIndent === this.indent) {
if (noNewlines) {
this.suppressNewlines();
} else {
@ -758,7 +758,10 @@
if (size > this.indent) {
if (noNewlines) {
if (!backslash) {
this.indebt = size - this.indent;
this.continuationLineAdditionalIndent = size - this.indent;
}
if (this.continuationLineAdditionalIndent) {
prev.continuationLineIndent = this.indent + this.continuationLineAdditionalIndent;
}
this.suppressNewlines();
return indent.length;
@ -777,7 +780,7 @@
this.ends.push({
tag: 'OUTDENT'
});
this.outdebt = this.indebt = 0;
this.outdebt = this.continuationLineAdditionalIndent = 0;
this.indent = size;
this.indentLiteral = newIndentLiteral;
} else if (size < this.baseIndent) {
@ -785,13 +788,15 @@
offset: offset + indent.length
});
} else {
this.indebt = 0;
endsContinuationLineIndentation = this.continuationLineAdditionalIndent > 0;
this.continuationLineAdditionalIndent = 0;
this.outdentToken({
moveOut: this.indent - size,
noNewlines,
outdentLength: indent.length,
offset,
indentSize: size
indentSize: size,
endsContinuationLineIndentation
});
}
return indent.length;
@ -799,8 +804,8 @@
// Record an outdent token or multiple tokens, if we happen to be moving back
// inwards past several recorded indents. Sets new @indent value.
outdentToken({moveOut, noNewlines, outdentLength = 0, offset = 0, indentSize}) {
var decreasedIndent, dent, lastIndent, ref;
outdentToken({moveOut, noNewlines, outdentLength = 0, offset = 0, indentSize, endsContinuationLineIndentation}) {
var decreasedIndent, dent, lastIndent, ref, terminatorToken;
decreasedIndent = this.indent - moveOut;
while (moveOut > 0) {
lastIndent = this.indents[this.indents.length - 1];
@ -830,10 +835,15 @@
}
this.suppressSemicolons();
if (!(this.tag() === 'TERMINATOR' || noNewlines)) {
this.token('TERMINATOR', '\n', {
terminatorToken = this.token('TERMINATOR', '\n', {
offset: offset + outdentLength,
length: 0
});
if (endsContinuationLineIndentation) {
terminatorToken.endsContinuationLineIndentation = {
preContinuationLineIndent: this.indent
};
}
}
this.indent = decreasedIndent;
this.indentLiteral = this.indentLiteral.slice(0, decreasedIndent);
@ -1928,7 +1938,4 @@
// Additional indent in front of these is ignored.
INDENTABLE_CLOSERS = [')', '}', ']'];
// Tokens that, when appearing at the end of a line, suppress a following TERMINATOR/INDENT token
UNFINISHED = ['\\', '.', '?.', '?::', 'UNARY', 'DO', 'DO_IIFE', 'MATH', 'UNARY_MATH', '+', '-', '**', 'SHIFT', 'RELATION', 'COMPARE', '&', '^', '|', '&&', '||', 'BIN?', 'EXTENDS'];
}).call(this);

29
lib/coffeescript/rewriter.js
View File

@ -6,7 +6,7 @@
// a series of passes over the token stream, using this **Rewriter** to convert
// shorthand into the unambiguous long form, add implicit indentation and
// parentheses, and generally clean things up.
var BALANCED_PAIRS, CALL_CLOSERS, CONTROL_IN_IMPLICIT, DISCARDED, EXPRESSION_CLOSE, EXPRESSION_END, EXPRESSION_START, IMPLICIT_CALL, IMPLICIT_END, IMPLICIT_FUNC, IMPLICIT_UNSPACED_CALL, INVERSES, LINEBREAKS, Rewriter, SINGLE_CLOSERS, SINGLE_LINERS, extractAllCommentTokens, generate, k, left, len, moveComments, right, throwSyntaxError,
var BALANCED_PAIRS, CALL_CLOSERS, CONTROL_IN_IMPLICIT, DISCARDED, EXPRESSION_CLOSE, EXPRESSION_END, EXPRESSION_START, IMPLICIT_CALL, IMPLICIT_END, IMPLICIT_FUNC, IMPLICIT_UNSPACED_CALL, INVERSES, LINEBREAKS, Rewriter, SINGLE_CLOSERS, SINGLE_LINERS, UNFINISHED, extractAllCommentTokens, generate, k, left, len, moveComments, right, throwSyntaxError,
indexOf = [].indexOf,
hasProp = {}.hasOwnProperty;
@ -289,7 +289,7 @@
stack = [];
start = null;
return this.scanTokens(function(token, i, tokens) {
var endImplicitCall, endImplicitObject, forward, implicitObjectContinues, inControlFlow, inImplicit, inImplicitCall, inImplicitControl, inImplicitObject, isImplicit, isImplicitCall, isImplicitObject, k, newLine, nextTag, nextToken, offset, prevTag, prevToken, ref, ref1, ref2, s, sameLine, stackIdx, stackItem, stackTag, stackTop, startIdx, startImplicitCall, startImplicitObject, startIndex, startTag, startsLine, tag;
var endImplicitCall, endImplicitObject, forward, implicitObjectContinues, implicitObjectIndent, inControlFlow, inImplicit, inImplicitCall, inImplicitControl, inImplicitObject, isImplicit, isImplicitCall, isImplicitObject, k, newLine, nextTag, nextToken, offset, preContinuationLineIndent, preObjectToken, prevTag, prevToken, ref, ref1, ref2, ref3, s, sameLine, stackIdx, stackItem, stackTag, stackTop, startIdx, startImplicitCall, startImplicitObject, startIndex, startTag, startsLine, tag;
[tag] = token;
[prevTag] = prevToken = i > 0 ? tokens[i - 1] : [];
[nextTag] = nextToken = i < tokens.length - 1 ? tokens[i + 1] : [];
@ -343,7 +343,7 @@
tokens.splice(i, 0, generate('CALL_END', ')', ['', 'end of input', token[2]], prevToken));
return i += 1;
};
startImplicitObject = function(idx, startsLine = true) {
startImplicitObject = function(idx, {startsLine = true, continuationLineIndent} = {}) {
var val;
stack.push([
'{',
@ -351,7 +351,8 @@
{
sameLine: true,
startsLine: startsLine,
ours: true
ours: true,
continuationLineIndent: continuationLineIndent
}
]);
val = new String('{');
@ -506,11 +507,15 @@
// Are we just continuing an already declared object?
if (stackTop()) {
[stackTag, stackIdx] = stackTop();
if ((stackTag === '{' || stackTag === 'INDENT' && this.tag(stackIdx - 1) === '{') && (startsLine || this.tag(s - 1) === ',' || this.tag(s - 1) === '{')) {
if ((stackTag === '{' || stackTag === 'INDENT' && this.tag(stackIdx - 1) === '{') && (startsLine || this.tag(s - 1) === ',' || this.tag(s - 1) === '{') && (ref2 = this.tag(s - 1), indexOf.call(UNFINISHED, ref2) < 0)) {
return forward(1);
}
}
startImplicitObject(s, !!startsLine);
preObjectToken = i > 1 ? tokens[i - 2] : [];
startImplicitObject(s, {
startsLine: !!startsLine,
continuationLineIndent: preObjectToken.continuationLineIndent
});
return forward(2);
}
// End implicit calls when chaining method calls
@ -540,6 +545,13 @@
}
}
}
// End indented-continuation-line implicit objects once that indentation is over.
if (tag === 'TERMINATOR' && token.endsContinuationLineIndentation) {
({preContinuationLineIndent} = token.endsContinuationLineIndentation);
while (inImplicitObject() && ((implicitObjectIndent = stackTop()[2].continuationLineIndent) != null) && implicitObjectIndent > preContinuationLineIndent) {
endImplicitObject();
}
}
newLine = prevTag === 'OUTDENT' || prevToken.newLine;
if (indexOf.call(IMPLICIT_END, tag) >= 0 || (indexOf.call(CALL_CLOSERS, tag) >= 0 && newLine) || ((tag === '..' || tag === '...') && this.findTagsBackwards(i, ["INDEX_START"]))) {
while (inImplicit()) {
@ -576,7 +588,7 @@
// f a, b: c, d: e, f, g: h: i, j
if (tag === ',' && !this.looksObjectish(i + 1) && inImplicitObject() && !((ref2 = this.tag(i + 2)) === 'FOROF' || ref2 === 'FORIN') && (nextTag !== 'TERMINATOR' || !this.looksObjectish(i + 2))) {
if (tag === ',' && !this.looksObjectish(i + 1) && inImplicitObject() && !((ref3 = this.tag(i + 2)) === 'FOROF' || ref3 === 'FORIN') && (nextTag !== 'TERMINATOR' || !this.looksObjectish(i + 2))) {
// When nextTag is OUTDENT the comma is insignificant and
// should just be ignored so embed it in the implicit object.
@ -1118,4 +1130,7 @@
// `addDataToNode` attaches `STRING_START`’s tokens to that node.
DISCARDED = ['(', ')', '[', ']', '{', '}', ':', '.', '..', '...', ',', '=', '++', '--', '?', 'AS', 'AWAIT', 'CALL_START', 'CALL_END', 'DEFAULT', 'DO', 'DO_IIFE', 'ELSE', 'EXTENDS', 'EXPORT', 'FORIN', 'FOROF', 'FORFROM', 'IMPORT', 'INDENT', 'INDEX_SOAK', 'INTERPOLATION_START', 'INTERPOLATION_END', 'LEADING_WHEN', 'OUTDENT', 'PARAM_END', 'REGEX_START', 'REGEX_END', 'RETURN', 'STRING_END', 'THROW', 'UNARY', 'YIELD'].concat(IMPLICIT_UNSPACED_CALL.concat(IMPLICIT_END.concat(CALL_CLOSERS.concat(CONTROL_IN_IMPLICIT))));
// Tokens that, when appearing at the end of a line, suppress a following TERMINATOR/INDENT token
exports.UNFINISHED = UNFINISHED = ['\\', '.', '?.', '?::', 'UNARY', 'DO', 'DO_IIFE', 'MATH', 'UNARY_MATH', '+', '-', '**', 'SHIFT', 'RELATION', 'COMPARE', '&', '^', '|', '&&', '||', 'BIN?', 'EXTENDS'];
}).call(this);

28
src/lexer.coffee
View File

@ -9,7 +9,7 @@
# format that can be fed directly into [Jison](https://github.com/zaach/jison). These
# are read by jison in the `parser.lexer` function defined in coffeescript.coffee.
{Rewriter, INVERSES} = require './rewriter'
{Rewriter, INVERSES, UNFINISHED} = require './rewriter'
# Import the helpers we need.
{count, starts, compact, repeat, invertLiterate, merge,
@ -38,7 +38,7 @@ exports.Lexer = class Lexer
@literate = opts.literate # Are we lexing literate CoffeeScript?
@indent = 0 # The current indentation level.
@baseIndent = 0 # The overall minimum indentation level.
@indebt = 0 # The over-indentation at the current level.
@continuationLineAdditionalIndent = 0 # The over-indentation at the current level.
@outdebt = 0 # The under-outdentation at the current level.
@indents = [] # The stack of all current indentation levels.
@indentLiteral = '' # The indentation.
@ -531,13 +531,15 @@ exports.Lexer = class Lexer
@error 'indentation mismatch', offset: indent.length
return indent.length
if size - @indebt is @indent
if size - @continuationLineAdditionalIndent is @indent
if noNewlines then @suppressNewlines() else @newlineToken offset
return indent.length
if size > @indent
if noNewlines
@indebt = size - @indent unless backslash
@continuationLineAdditionalIndent = size - @indent unless backslash
if @continuationLineAdditionalIndent
prev.continuationLineIndent = @indent + @continuationLineAdditionalIndent
@suppressNewlines()
return indent.length
unless @tokens.length
@ -548,19 +550,20 @@ exports.Lexer = class Lexer
@token 'INDENT', diff, offset: offset + indent.length - size, length: size
@indents.push diff
@ends.push {tag: 'OUTDENT'}
@outdebt = @indebt = 0
@outdebt = @continuationLineAdditionalIndent = 0
@indent = size
@indentLiteral = newIndentLiteral
else if size < @baseIndent
@error 'missing indentation', offset: offset + indent.length
else
@indebt = 0
@outdentToken {moveOut: @indent - size, noNewlines, outdentLength: indent.length, offset, indentSize: size}
endsContinuationLineIndentation = @continuationLineAdditionalIndent > 0
@continuationLineAdditionalIndent = 0
@outdentToken {moveOut: @indent - size, noNewlines, outdentLength: indent.length, offset, indentSize: size, endsContinuationLineIndentation}
indent.length
# Record an outdent token or multiple tokens, if we happen to be moving back
# inwards past several recorded indents. Sets new @indent value.
outdentToken: ({moveOut, noNewlines, outdentLength = 0, offset = 0, indentSize}) ->
outdentToken: ({moveOut, noNewlines, outdentLength = 0, offset = 0, indentSize, endsContinuationLineIndentation}) ->
decreasedIndent = @indent - moveOut
while moveOut > 0
lastIndent = @indents[@indents.length - 1]
@ -582,7 +585,9 @@ exports.Lexer = class Lexer
@outdebt -= moveOut if dent
@suppressSemicolons()
@token 'TERMINATOR', '\n', offset: offset + outdentLength, length: 0 unless @tag() is 'TERMINATOR' or noNewlines
unless @tag() is 'TERMINATOR' or noNewlines
terminatorToken = @token 'TERMINATOR', '\n', offset: offset + outdentLength, length: 0
terminatorToken.endsContinuationLineIndentation = {preContinuationLineIndent: @indent} if endsContinuationLineIndentation
@indent = decreasedIndent
@indentLiteral = @indentLiteral[...decreasedIndent]
this
@ -1463,8 +1468,3 @@ LINE_BREAK = ['INDENT', 'OUTDENT', 'TERMINATOR']
# Additional indent in front of these is ignored.
INDENTABLE_CLOSERS = [')', '}', ']']
# Tokens that, when appearing at the end of a line, suppress a following TERMINATOR/INDENT token
UNFINISHED = ['\\', '.', '?.', '?::', 'UNARY', 'DO', 'DO_IIFE', 'MATH', 'UNARY_MATH', '+', '-',
'**', 'SHIFT', 'RELATION', 'COMPARE', '&', '^', '|', '&&', '||',
'BIN?', 'EXTENDS']

21
src/rewriter.coffee
View File

@ -209,8 +209,8 @@ exports.Rewriter = class Rewriter
tokens.splice i, 0, generate 'CALL_END', ')', ['', 'end of input', token[2]], prevToken
i += 1
startImplicitObject = (idx, startsLine = yes) ->
stack.push ['{', idx, sameLine: yes, startsLine: startsLine, ours: yes]
startImplicitObject = (idx, {startsLine = yes, continuationLineIndent} = {}) ->
stack.push ['{', idx, sameLine: yes, startsLine: startsLine, ours: yes, continuationLineIndent: continuationLineIndent]
val = new String '{'
val.generated = yes
tokens.splice idx, 0, generate '{', val, token, prevToken
@ -342,10 +342,12 @@ exports.Rewriter = class Rewriter
if stackTop()
[stackTag, stackIdx] = stackTop()
if (stackTag is '{' or stackTag is 'INDENT' and @tag(stackIdx - 1) is '{') and
(startsLine or @tag(s - 1) is ',' or @tag(s - 1) is '{')
(startsLine or @tag(s - 1) is ',' or @tag(s - 1) is '{') and
@tag(s - 1) not in UNFINISHED
return forward(1)
startImplicitObject(s, !!startsLine)
preObjectToken = if i > 1 then tokens[i - 2] else []
startImplicitObject(s, {startsLine: !!startsLine, continuationLineIndent: preObjectToken.continuationLineIndent})
return forward(2)
# End implicit calls when chaining method calls
@ -369,6 +371,12 @@ exports.Rewriter = class Rewriter
break unless isImplicit stackItem
stackItem[2].sameLine = no if isImplicitObject stackItem
# End indented-continuation-line implicit objects once that indentation is over.
if tag is 'TERMINATOR' and token.endsContinuationLineIndentation
{preContinuationLineIndent} = token.endsContinuationLineIndentation
while inImplicitObject() and (implicitObjectIndent = stackTop()[2].continuationLineIndent)? and implicitObjectIndent > preContinuationLineIndent
endImplicitObject()
newLine = prevTag is 'OUTDENT' or prevToken.newLine
if tag in IMPLICIT_END or
(tag in CALL_CLOSERS and newLine) or
@ -843,3 +851,8 @@ DISCARDED = ['(', ')', '[', ']', '{', '}', ':', '.', '..', '...', ',', '=', '++'
'INTERPOLATION_START', 'INTERPOLATION_END', 'LEADING_WHEN', 'OUTDENT', 'PARAM_END',
'REGEX_START', 'REGEX_END', 'RETURN', 'STRING_END', 'THROW', 'UNARY', 'YIELD'
].concat IMPLICIT_UNSPACED_CALL.concat IMPLICIT_END.concat CALL_CLOSERS.concat CONTROL_IN_IMPLICIT
# Tokens that, when appearing at the end of a line, suppress a following TERMINATOR/INDENT token
exports.UNFINISHED = UNFINISHED = ['\\', '.', '?.', '?::', 'UNARY', 'DO', 'DO_IIFE', 'MATH', 'UNARY_MATH', '+', '-',
'**', 'SHIFT', 'RELATION', 'COMPARE', '&', '^', '|', '&&', '||',
'BIN?', 'EXTENDS']

24
test/objects.coffee
View File

@ -906,3 +906,27 @@ test "#4579: Postfix for/while/until in first line of implicit object literals",
test "#5204: not parsed as static property", ->
doesNotThrowCompileError "@ [b]: 2"
test "#5292: implicit object after line continuer in implicit object property value", ->
a =
b: 0 or
c: 1
eq 1, a.b.c
# following object property
a =
b: null ?
c: 1
d: 2
eq 1, a.b.c
eq 2, a.d
# multiline nested object
a =
b: 0 or
c: 1
d: 2
e: 3
eq 1, a.b.c
eq 2, a.b.d
eq 3, a.e