From d03d288a989e020e30459521e6f5043170db72e0 Mon Sep 17 00:00:00 2001 From: satyr Date: Sat, 17 Sep 2011 08:26:04 +0900 Subject: [PATCH] fixed #1299: overhauled token pairings --- lib/coffee-script/lexer.js | 94 +++++++++++++++++++++++------------ lib/coffee-script/rewriter.js | 66 +----------------------- src/lexer.coffee | 34 +++++++++++-- src/rewriter.coffee | 67 ++----------------------- test/formatting.coffee | 10 ++++ 5 files changed, 105 insertions(+), 166 deletions(-) diff --git a/lib/coffee-script/lexer.js b/lib/coffee-script/lexer.js index d4be86c8..5c523baf 100644 --- a/lib/coffee-script/lexer.js +++ b/lib/coffee-script/lexer.js @@ -1,17 +1,17 @@ (function() { - var BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, HEREDOC, HEREDOC_ILLEGAL, HEREDOC_INDENT, HEREGEX, HEREGEX_OMIT, IDENTIFIER, INDEXABLE, JSTOKEN, JS_FORBIDDEN, JS_KEYWORDS, LINE_BREAK, LINE_CONTINUER, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NOT_REGEX, NOT_SPACED_REGEX, NUMBER, OPERATOR, REGEX, RELATION, RESERVED, Rewriter, SHIFT, SIMPLESTR, TRAILING_SPACES, UNARY, WHITESPACE, compact, count, key, last, starts, _ref; + var BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, HEREDOC, HEREDOC_ILLEGAL, HEREDOC_INDENT, HEREGEX, HEREGEX_OMIT, IDENTIFIER, INDEXABLE, INVERSES, JSTOKEN, JS_FORBIDDEN, JS_KEYWORDS, LINE_BREAK, LINE_CONTINUER, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NOT_REGEX, NOT_SPACED_REGEX, NUMBER, OPERATOR, REGEX, RELATION, RESERVED, Rewriter, SHIFT, SIMPLESTR, TRAILING_SPACES, UNARY, WHITESPACE, compact, count, key, last, starts, _ref, _ref2; var __hasProp = Object.prototype.hasOwnProperty, __indexOf = Array.prototype.indexOf || function(item) { for (var i = 0, l = this.length; i < l; i++) { if (__hasProp.call(this, i) && this[i] === item) return i; } return -1; }; - Rewriter = require('./rewriter').Rewriter; - _ref = require('./helpers'), count = _ref.count, starts = _ref.starts, compact = _ref.compact, last = _ref.last; + _ref = require('./rewriter'), Rewriter = _ref.Rewriter, INVERSES = _ref.INVERSES; + _ref2 = require('./helpers'), count = _ref2.count, starts = _ref2.starts, compact = _ref2.compact, last = _ref2.last; exports.Lexer = Lexer = (function() { function Lexer() {} Lexer.prototype.tokenize = function(code, opts) { - var i; + var i, tag; if (opts == null) opts = {}; if (WHITESPACE.test(code)) code = "\n" + code; code = code.replace(/\r/g, '').replace(TRAILING_SPACES, ''); @@ -21,28 +21,30 @@ this.indebt = 0; this.outdebt = 0; this.indents = []; + this.ends = []; this.tokens = []; i = 0; while (this.chunk = code.slice(i)) { i += this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.heredocToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.jsToken() || this.literalToken(); } this.closeIndentation(); + if (tag = this.ends.pop()) this.carp("missing " + tag); if (opts.rewrite === false) return this.tokens; return (new Rewriter).rewrite(this.tokens); }; Lexer.prototype.identifierToken = function() { - var colon, forcedIdentifier, id, input, match, prev, tag, _ref2, _ref3; + var colon, forcedIdentifier, id, input, match, prev, tag, _ref3, _ref4; if (!(match = IDENTIFIER.exec(this.chunk))) return 0; input = match[0], id = match[1], colon = match[2]; if (id === 'own' && this.tag() === 'FOR') { this.token('OWN', id); return id.length; } - forcedIdentifier = colon || (prev = last(this.tokens)) && (((_ref2 = prev[0]) === '.' || _ref2 === '?.' || _ref2 === '::') || !prev.spaced && prev[0] === '@'); + forcedIdentifier = colon || (prev = last(this.tokens)) && (((_ref3 = prev[0]) === '.' || _ref3 === '?.' || _ref3 === '::') || !prev.spaced && prev[0] === '@'); tag = 'IDENTIFIER'; if (!forcedIdentifier && (__indexOf.call(JS_KEYWORDS, id) >= 0 || __indexOf.call(COFFEE_KEYWORDS, id) >= 0)) { tag = id.toUpperCase(); - if (tag === 'WHEN' && (_ref3 = this.tag(), __indexOf.call(LINE_BREAK, _ref3) >= 0)) { + if (tag === 'WHEN' && (_ref4 = this.tag(), __indexOf.call(LINE_BREAK, _ref4) >= 0)) { tag = 'LEADING_WHEN'; } else if (tag === 'FOR') { this.seenFor = true; @@ -172,7 +174,7 @@ return script.length; }; Lexer.prototype.regexToken = function() { - var length, match, prev, regex, _ref2; + var length, match, prev, regex, _ref3; if (this.chunk.charAt(0) !== '/') return 0; if (match = HEREGEX.exec(this.chunk)) { length = this.heregexToken(match); @@ -180,7 +182,7 @@ return length; } prev = last(this.tokens); - if (prev && (_ref2 = prev[0], __indexOf.call((prev.spaced ? NOT_REGEX : NOT_SPACED_REGEX), _ref2) >= 0)) { + if (prev && (_ref3 = prev[0], __indexOf.call((prev.spaced ? NOT_REGEX : NOT_SPACED_REGEX), _ref3) >= 0)) { return 0; } if (!(match = REGEX.exec(this.chunk))) return 0; @@ -189,7 +191,7 @@ return regex.length; }; Lexer.prototype.heregexToken = function(match) { - var body, flags, heregex, re, tag, tokens, value, _i, _len, _ref2, _ref3, _ref4, _ref5; + var body, flags, heregex, re, tag, tokens, value, _i, _len, _ref3, _ref4, _ref5, _ref6; heregex = match[0], body = match[1], flags = match[2]; if (0 > body.indexOf('#{')) { re = body.replace(HEREGEX_OMIT, '').replace(/\//g, '\\/'); @@ -199,11 +201,11 @@ this.token('IDENTIFIER', 'RegExp'); this.tokens.push(['CALL_START', '(']); tokens = []; - _ref2 = this.interpolateString(body, { + _ref3 = this.interpolateString(body, { regex: true }); - for (_i = 0, _len = _ref2.length; _i < _len; _i++) { - _ref3 = _ref2[_i], tag = _ref3[0], value = _ref3[1]; + for (_i = 0, _len = _ref3.length; _i < _len; _i++) { + _ref4 = _ref3[_i], tag = _ref4[0], value = _ref4[1]; if (tag === 'TOKENS') { tokens.push.apply(tokens, value); } else { @@ -214,10 +216,10 @@ tokens.push(['+', '+']); } tokens.pop(); - if (((_ref4 = tokens[0]) != null ? _ref4[0] : void 0) !== 'STRING') { + if (((_ref5 = tokens[0]) != null ? _ref5[0] : void 0) !== 'STRING') { this.tokens.push(['STRING', '""'], ['+', '+']); } - (_ref5 = this.tokens).push.apply(_ref5, tokens); + (_ref6 = this.tokens).push.apply(_ref6, tokens); if (flags) this.tokens.push([',', ','], ['STRING', '"' + flags + '"']); this.token(')', ')'); return heregex.length; @@ -247,6 +249,7 @@ diff = size - this.indent + this.outdebt; this.token('INDENT', diff); this.indents.push(diff); + this.ends.push('OUTDENT'); this.outdebt = this.indebt = 0; } else { this.indebt = 0; @@ -255,7 +258,7 @@ this.indent = size; return indent.length; }; - Lexer.prototype.outdentToken = function(moveOut, noNewlines, close) { + Lexer.prototype.outdentToken = function(moveOut, noNewlines) { var dent, len; while (moveOut > 0) { len = this.indents.length - 1; @@ -271,6 +274,7 @@ dent = this.indents.pop() - this.outdebt; moveOut -= dent; this.outdebt = 0; + this.pair('OUTDENT'); this.token('OUTDENT', dent); } } @@ -308,7 +312,7 @@ return this; }; Lexer.prototype.literalToken = function() { - var match, prev, tag, value, _ref2, _ref3, _ref4, _ref5; + var match, prev, tag, value, _ref3, _ref4, _ref5, _ref6; if (match = OPERATOR.exec(this.chunk)) { value = match[0]; if (CODE.test(value)) this.tagParameters(); @@ -318,10 +322,10 @@ tag = value; prev = last(this.tokens); if (value === '=' && prev) { - if (!prev[1].reserved && (_ref2 = prev[1], __indexOf.call(JS_FORBIDDEN, _ref2) >= 0)) { + if (!prev[1].reserved && (_ref3 = prev[1], __indexOf.call(JS_FORBIDDEN, _ref3) >= 0)) { this.assignmentError(); } - if ((_ref3 = prev[1]) === '||' || _ref3 === '&&') { + if ((_ref4 = prev[1]) === '||' || _ref4 === '&&') { prev[0] = 'COMPOUND_ASSIGN'; prev[1] += '='; return value.length; @@ -342,10 +346,10 @@ } else if (__indexOf.call(LOGIC, value) >= 0 || value === '?' && (prev != null ? prev.spaced : void 0)) { tag = 'LOGIC'; } else if (prev && !prev.spaced) { - if (value === '(' && (_ref4 = prev[0], __indexOf.call(CALLABLE, _ref4) >= 0)) { + if (value === '(' && (_ref5 = prev[0], __indexOf.call(CALLABLE, _ref5) >= 0)) { if (prev[0] === '?') prev[0] = 'FUNC_EXIST'; tag = 'CALL_START'; - } else if (value === '[' && (_ref5 = prev[0], __indexOf.call(INDEXABLE, _ref5) >= 0)) { + } else if (value === '[' && (_ref6 = prev[0], __indexOf.call(INDEXABLE, _ref6) >= 0)) { tag = 'INDEX_START'; switch (prev[0]) { case '?': @@ -353,21 +357,32 @@ } } } + switch (value) { + case '(': + case '{': + case '[': + this.ends.push(INVERSES[value]); + break; + case ')': + case '}': + case ']': + this.pair(value); + } this.token(tag, value); return value.length; }; Lexer.prototype.sanitizeHeredoc = function(doc, options) { - var attempt, herecomment, indent, match, _ref2; + var attempt, herecomment, indent, match, _ref3; indent = options.indent, herecomment = options.herecomment; if (herecomment) { if (HEREDOC_ILLEGAL.test(doc)) { - throw new Error("block comment cannot contain \"*/\", starting on line " + (this.line + 1)); + this.carp("block comment cannot contain \"*/\", starting"); } if (doc.indexOf('\n') <= 0) return doc; } else { while (match = HEREDOC_INDENT.exec(doc)) { attempt = match[1]; - if (indent === null || (0 < (_ref2 = attempt.length) && _ref2 < indent.length)) { + if (indent === null || (0 < (_ref3 = attempt.length) && _ref3 < indent.length)) { indent = attempt; } } @@ -412,9 +427,9 @@ throw SyntaxError("Reserved word \"" + (this.value()) + "\" on line " + (this.line + 1) + " can't be assigned"); }; Lexer.prototype.balancedString = function(str, end) { - var i, letter, match, prev, stack, _ref2; + var i, letter, match, prev, stack, _ref3; stack = [end]; - for (i = 1, _ref2 = str.length; 1 <= _ref2 ? i < _ref2 : i > _ref2; 1 <= _ref2 ? i++ : i--) { + for (i = 1, _ref3 = str.length; 1 <= _ref3 ? i < _ref3 : i > _ref3; 1 <= _ref3 ? i++ : i--) { switch (letter = str.charAt(i)) { case '\\': i++; @@ -436,10 +451,10 @@ } prev = letter; } - throw new Error("missing " + (stack.pop()) + ", starting on line " + (this.line + 1)); + return this.carp("missing " + (stack.pop()) + ", starting"); }; Lexer.prototype.interpolateString = function(str, options) { - var expr, heredoc, i, inner, interpolated, len, letter, nested, pi, regex, tag, tokens, value, _len, _ref2, _ref3, _ref4; + var expr, heredoc, i, inner, interpolated, len, letter, nested, pi, regex, tag, tokens, value, _len, _ref3, _ref4, _ref5; if (options == null) options = {}; heredoc = options.heredoc, regex = options.regex; tokens = []; @@ -461,7 +476,7 @@ rewrite: false }); nested.pop(); - if (((_ref2 = nested[0]) != null ? _ref2[0] : void 0) === 'TERMINATOR') { + if (((_ref3 = nested[0]) != null ? _ref3[0] : void 0) === 'TERMINATOR') { nested.shift(); } if (len = nested.length) { @@ -481,10 +496,10 @@ if (tokens[0][0] !== 'NEOSTRING') tokens.unshift(['', '']); if (interpolated = tokens.length > 1) this.token('(', '('); for (i = 0, _len = tokens.length; i < _len; i++) { - _ref3 = tokens[i], tag = _ref3[0], value = _ref3[1]; + _ref4 = tokens[i], tag = _ref4[0], value = _ref4[1]; if (i) this.token('+', '+'); if (tag === 'TOKENS') { - (_ref4 = this.tokens).push.apply(_ref4, value); + (_ref5 = this.tokens).push.apply(_ref5, value); } else { this.token('STRING', this.makeString(value, '"', heredoc)); } @@ -492,6 +507,16 @@ if (interpolated) this.token(')', ')'); return tokens; }; + Lexer.prototype.pair = function(tag) { + var size, wanted; + if (tag !== (wanted = last(this.ends))) { + if ('OUTDENT' !== wanted) this.carp("unmatched " + tag); + this.indent -= size = last(this.indents); + this.outdentToken(size, true); + return this.pair(tag); + } + return this.ends.pop(); + }; Lexer.prototype.token = function(tag, value) { return this.tokens.push([tag, value, this.line]); }; @@ -504,8 +529,8 @@ return (tok = last(this.tokens, index)) && (val ? tok[1] = val : tok[1]); }; Lexer.prototype.unfinished = function() { - var _ref2; - return LINE_CONTINUER.test(this.chunk) || ((_ref2 = this.tag()) === '\\' || _ref2 === '.' || _ref2 === '?.' || _ref2 === 'UNARY' || _ref2 === 'MATH' || _ref2 === '+' || _ref2 === '-' || _ref2 === 'SHIFT' || _ref2 === 'RELATION' || _ref2 === 'COMPARE' || _ref2 === 'LOGIC' || _ref2 === 'COMPOUND_ASSIGN' || _ref2 === 'THROW' || _ref2 === 'EXTENDS'); + var _ref3; + return LINE_CONTINUER.test(this.chunk) || ((_ref3 = this.tag()) === '\\' || _ref3 === '.' || _ref3 === '?.' || _ref3 === 'UNARY' || _ref3 === 'MATH' || _ref3 === '+' || _ref3 === '-' || _ref3 === 'SHIFT' || _ref3 === 'RELATION' || _ref3 === 'COMPARE' || _ref3 === 'LOGIC' || _ref3 === 'COMPOUND_ASSIGN' || _ref3 === 'THROW' || _ref3 === 'EXTENDS'); }; Lexer.prototype.escapeLines = function(str, heredoc) { return str.replace(MULTILINER, heredoc ? '\\n' : ''); @@ -522,6 +547,9 @@ body = body.replace(RegExp("" + quote, "g"), '\\$&'); return quote + this.escapeLines(body, heredoc) + quote; }; + Lexer.prototype.carp = function(message) { + throw SyntaxError("" + message + " on line " + (this.line + 1)); + }; return Lexer; })(); JS_KEYWORDS = ['true', 'false', 'null', 'this', 'new', 'delete', 'typeof', 'in', 'instanceof', 'return', 'throw', 'break', 'continue', 'debugger', 'if', 'else', 'switch', 'for', 'while', 'do', 'try', 'catch', 'finally', 'class', 'extends', 'super']; diff --git a/lib/coffee-script/rewriter.js b/lib/coffee-script/rewriter.js index b2b259ea..f211b488 100644 --- a/lib/coffee-script/rewriter.js +++ b/lib/coffee-script/rewriter.js @@ -18,8 +18,6 @@ this.tagPostfixConditionals(); this.addImplicitBraces(); this.addImplicitParentheses(); - this.ensureBalance(BALANCED_PAIRS); - this.rewriteClosingParens(); return this.tokens; }; Rewriter.prototype.scanTokens = function(block) { @@ -146,7 +144,7 @@ noCall = false; action = function(token, i) { var idx; - idx = token[0] === 'OUTDENT' ? i + 1 : i; + idx = token[0] === 'OUTDENT' ? i : i; return this.tokens.splice(idx, 0, ['CALL_END', ')', token[2]]); }; return this.scanTokens(function(token, i, tokens) { @@ -235,66 +233,6 @@ return 1; }); }; - Rewriter.prototype.ensureBalance = function(pairs) { - var close, level, levels, open, openLine, tag, token, _i, _j, _len, _len2, _ref, _ref2; - levels = {}; - openLine = {}; - _ref = this.tokens; - for (_i = 0, _len = _ref.length; _i < _len; _i++) { - token = _ref[_i]; - tag = token[0]; - for (_j = 0, _len2 = pairs.length; _j < _len2; _j++) { - _ref2 = pairs[_j], open = _ref2[0], close = _ref2[1]; - levels[open] |= 0; - if (tag === open) { - if (levels[open]++ === 0) openLine[open] = token[2]; - } else if (tag === close && --levels[open] < 0) { - throw Error("too many " + token[1] + " on line " + (token[2] + 1)); - } - } - } - for (open in levels) { - level = levels[open]; - if (level > 0) { - throw Error("unclosed " + open + " on line " + (openLine[open] + 1)); - } - } - return this; - }; - Rewriter.prototype.rewriteClosingParens = function() { - var debt, key, stack; - stack = []; - debt = {}; - for (key in INVERSES) { - debt[key] = 0; - } - return this.scanTokens(function(token, i, tokens) { - var inv, match, mtag, oppos, tag, val, _ref; - if (_ref = (tag = token[0]), __indexOf.call(EXPRESSION_START, _ref) >= 0) { - stack.push(token); - return 1; - } - if (__indexOf.call(EXPRESSION_END, tag) < 0) return 1; - if (debt[inv = INVERSES[tag]] > 0) { - debt[inv] -= 1; - tokens.splice(i, 1); - return 0; - } - match = stack.pop(); - mtag = match[0]; - oppos = INVERSES[mtag]; - if (tag === oppos) return 1; - debt[mtag] += 1; - val = [oppos, mtag === 'INDENT' ? match[1] : oppos]; - if (this.tag(i + 2) === mtag) { - tokens.splice(i + 3, 0, val); - stack.push(match); - } else { - tokens.splice(i, 0, val); - } - return 1; - }); - }; Rewriter.prototype.indentation = function(token) { return [['INDENT', 2, token[2]], ['OUTDENT', 2, token[2]]]; }; @@ -305,7 +243,7 @@ return Rewriter; })(); BALANCED_PAIRS = [['(', ')'], ['[', ']'], ['{', '}'], ['INDENT', 'OUTDENT'], ['CALL_START', 'CALL_END'], ['PARAM_START', 'PARAM_END'], ['INDEX_START', 'INDEX_END']]; - INVERSES = {}; + exports.INVERSES = INVERSES = {}; EXPRESSION_START = []; EXPRESSION_END = []; for (_i = 0, _len = BALANCED_PAIRS.length; _i < _len; _i++) { diff --git a/src/lexer.coffee b/src/lexer.coffee index 7a079126..3380f96a 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -7,7 +7,7 @@ # # Which is a format that can be fed directly into [Jison](http://github.com/zaach/jison). -{Rewriter} = require './rewriter' +{Rewriter, INVERSES} = require './rewriter' # Import the helpers we need. {count, starts, compact, last} = require './helpers' @@ -41,6 +41,7 @@ exports.Lexer = class Lexer @indebt = 0 # The over-indentation at the current level. @outdebt = 0 # The under-outdentation at the current level. @indents = [] # The stack of all current indentation levels. + @ends = [] # The stack for pairing up tokens. @tokens = [] # Stream of parsed tokens in the form `['TYPE', value, line]`. # At every position, run through this list of attempted matches, @@ -60,6 +61,7 @@ exports.Lexer = class Lexer @literalToken() @closeIndentation() + @carp "missing #{tag}" if tag = @ends.pop() return @tokens if opts.rewrite is off (new Rewriter).rewrite @tokens @@ -253,6 +255,7 @@ exports.Lexer = class Lexer diff = size - @indent + @outdebt @token 'INDENT', diff @indents.push diff + @ends .push 'OUTDENT' @outdebt = @indebt = 0 else @indebt = 0 @@ -262,7 +265,7 @@ exports.Lexer = class Lexer # Record an outdent token or multiple tokens, if we happen to be moving back # inwards past several recorded indents. - outdentToken: (moveOut, noNewlines, close) -> + outdentToken: (moveOut, noNewlines) -> while moveOut > 0 len = @indents.length - 1 if @indents[len] is undefined @@ -277,6 +280,7 @@ exports.Lexer = class Lexer dent = @indents.pop() - @outdebt moveOut -= dent @outdebt = 0 + @pair 'OUTDENT' @token 'OUTDENT', dent @outdebt -= moveOut if dent @tokens.pop() while @value() is ';' @@ -338,6 +342,9 @@ exports.Lexer = class Lexer tag = 'INDEX_START' switch prev[0] when '?' then prev[0] = 'INDEX_SOAK' + switch value + when '(', '{', '[' then @ends.push INVERSES[value] + when ')', '}', ']' then @pair value @token tag, value value.length @@ -350,7 +357,7 @@ exports.Lexer = class Lexer {indent, herecomment} = options if herecomment if HEREDOC_ILLEGAL.test doc - throw new Error "block comment cannot contain \"*/\", starting on line #{@line + 1}" + @carp "block comment cannot contain \"*/\", starting" return doc if doc.indexOf('\n') <= 0 else while match = HEREDOC_INDENT.exec doc @@ -421,8 +428,7 @@ exports.Lexer = class Lexer else if end is '"' and prev is '#' and letter is '{' stack.push end = '}' prev = letter - throw new Error "missing #{ stack.pop() }, starting on line #{ @line + 1 }" - + @carp "missing #{ stack.pop() }, starting" # Expand variables and expressions inside double-quoted strings using # Ruby-like notation for substitution of arbitrary expressions. @@ -471,6 +477,21 @@ exports.Lexer = class Lexer @token ')', ')' if interpolated tokens + # Pairs up a closing token, ensuring that all listed pairs of tokens are + # correctly balanced throughout the course of the token stream. + pair: (tag) -> + unless tag is wanted = last @ends + @carp "unmatched #{tag}" unless 'OUTDENT' is wanted + # Auto-close INDENT to support syntax like this: + # + # el.click((event) -> + # el.hide()) + # + @indent -= size = last @indents + @outdentToken size, true + return @pair tag + @ends.pop() + # Helpers # ------- @@ -504,6 +525,9 @@ exports.Lexer = class Lexer body = body.replace /// #{quote} ///g, '\\$&' quote + @escapeLines(body, heredoc) + quote + # Throws a syntax error from current `@line`. + carp: (message) -> throw SyntaxError "#{message} on line #{ @line + 1}" + # Constants # --------- diff --git a/src/rewriter.coffee b/src/rewriter.coffee index 7cf82588..5fec8eb8 100644 --- a/src/rewriter.coffee +++ b/src/rewriter.coffee @@ -3,7 +3,7 @@ # the resulting parse table. Instead of making the parser handle it all, we take # a series of passes over the token stream, using this **Rewriter** to convert # shorthand into the unambiguous long form, add implicit indentation and -# parentheses, balance incorrect nestings, and generally clean things up. +# parentheses, and generally clean things up. # The **Rewriter** class is used by the [Lexer](lexer.html), directly against # its internal array of tokens. @@ -26,8 +26,6 @@ class exports.Rewriter @tagPostfixConditionals() @addImplicitBraces() @addImplicitParentheses() - @ensureBalance BALANCED_PAIRS - @rewriteClosingParens() @tokens # Rewrite the token stream, looking one token ahead and behind. @@ -134,7 +132,7 @@ class exports.Rewriter addImplicitParentheses: -> noCall = no action = (token, i) -> - idx = if token[0] is 'OUTDENT' then i + 1 else i + idx = if token[0] is 'OUTDENT' then i else i @tokens.splice idx, 0, ['CALL_END', ')', token[2]] @scanTokens (token, i, tokens) -> tag = token[0] @@ -211,65 +209,6 @@ class exports.Rewriter original[0] = 'POST_' + original[0] if token[0] isnt 'INDENT' 1 - # Ensure that all listed pairs of tokens are correctly balanced throughout - # the course of the token stream. - ensureBalance: (pairs) -> - levels = {} - openLine = {} - for token in @tokens - [tag] = token - for [open, close] in pairs - levels[open] |= 0 - if tag is open - openLine[open] = token[2] if levels[open]++ is 0 - else if tag is close and --levels[open] < 0 - throw Error "too many #{token[1]} on line #{token[2] + 1}" - for open, level of levels when level > 0 - throw Error "unclosed #{ open } on line #{openLine[open] + 1}" - this - - # We'd like to support syntax like this: - # - # el.click((event) -> - # el.hide()) - # - # In order to accomplish this, move outdents that follow closing parens - # inwards, safely. The steps to accomplish this are: - # - # 1. Check that all paired tokens are balanced and in order. - # 2. Rewrite the stream with a stack: if you see an `EXPRESSION_START`, add it - # to the stack. If you see an `EXPRESSION_END`, pop the stack and replace - # it with the inverse of what we've just popped. - # 3. Keep track of "debt" for tokens that we manufacture, to make sure we end - # up balanced in the end. - # 4. Be careful not to alter array or parentheses delimiters with overzealous - # rewriting. - rewriteClosingParens: -> - stack = [] - debt = {} - debt[key] = 0 for key of INVERSES - @scanTokens (token, i, tokens) -> - if (tag = token[0]) in EXPRESSION_START - stack.push token - return 1 - return 1 unless tag in EXPRESSION_END - if debt[inv = INVERSES[tag]] > 0 - debt[inv] -= 1 - tokens.splice i, 1 - return 0 - match = stack.pop() - mtag = match[0] - oppos = INVERSES[mtag] - return 1 if tag is oppos - debt[mtag] += 1 - val = [oppos, if mtag is 'INDENT' then match[1] else oppos] - if @tag(i + 2) is mtag - tokens.splice i + 3, 0, val - stack.push match - else - tokens.splice i, 0, val - 1 - # Generate the indentation tokens, based on another token on the same line. indentation: (token) -> [['INDENT', 2, token[2]], ['OUTDENT', 2, token[2]]] @@ -293,7 +232,7 @@ BALANCED_PAIRS = [ # The inverse mappings of `BALANCED_PAIRS` we're trying to fix up, so we can # look things up from either end. -INVERSES = {} +exports.INVERSES = INVERSES = {} # The tokens that signal the start/end of a balanced pair. EXPRESSION_START = [] diff --git a/test/formatting.coffee b/test/formatting.coffee index e5c2cdee..dcbd01e8 100644 --- a/test/formatting.coffee +++ b/test/formatting.coffee @@ -134,3 +134,13 @@ test "#1195 Ignore trailing semicolons (before newlines or as the last char in a lastChar = '-> lastChar;' doesNotThrow -> CoffeeScript.compile lastChar, bare: true + +test "#1299: Disallow token misnesting", -> + try + CoffeeScript.compile ''' + [{ + ]} + ''' + ok no + catch e + eq 'unmatched ] on line 2', e.message