From f0463e9981f83dca2042f595f40a52cfaaf2c69a Mon Sep 17 00:00:00 2001 From: xixixao Date: Wed, 22 Jan 2014 02:44:50 +0000 Subject: [PATCH] Improve error messages for generated tokens --- lib/coffee-script/coffee-script.js | 9 ++++--- lib/coffee-script/helpers.js | 15 +++++++++++ lib/coffee-script/lexer.js | 18 ++++++++----- lib/coffee-script/rewriter.js | 9 ++++--- src/coffee-script.coffee | 14 ++++++++-- src/helpers.coffee | 8 ++++++ src/lexer.coffee | 15 +++++++---- src/rewriter.coffee | 7 ++--- test/error_messages.coffee | 41 +++++++++++++++++++++++++++--- 9 files changed, 111 insertions(+), 25 deletions(-) diff --git a/lib/coffee-script/coffee-script.js b/lib/coffee-script/coffee-script.js index 8f128d75..1b290b1c 100644 --- a/lib/coffee-script/coffee-script.js +++ b/lib/coffee-script/coffee-script.js @@ -214,6 +214,7 @@ token = this.tokens[this.pos++]; if (token) { tag = token[0], this.yytext = token[1], this.yylloc = token[2]; + this.errorToken = token.origin || token; this.yylineno = this.yylloc.first_line; } else { tag = ''; @@ -232,10 +233,12 @@ parser.yy = require('./nodes'); parser.yy.parseError = function(message, _arg) { - var token; + var errorLoc, errorText, errorToken, ignored, token, tokens, _ref; token = _arg.token; - message = "unexpected " + (token === 1 ? 'end of input' : token); - return helpers.throwSyntaxError(message, parser.lexer.yylloc); + _ref = parser.lexer, errorToken = _ref.errorToken, tokens = _ref.tokens; + ignored = errorToken[0], errorText = errorToken[1], errorLoc = errorToken[2]; + errorText = errorToken === tokens[tokens.length - 1] ? 'end of input' : helpers.nameWhitespaceCharacter(errorText); + return helpers.throwSyntaxError("unexpected " + errorText, errorLoc); }; formatSourcePosition = function(frame, getSourceMapping) { diff --git a/lib/coffee-script/helpers.js b/lib/coffee-script/helpers.js index 0954a793..de47f156 100644 --- a/lib/coffee-script/helpers.js +++ b/lib/coffee-script/helpers.js @@ -234,4 +234,19 @@ return "" + filename + ":" + (first_line + 1) + ":" + (first_column + 1) + ": error: " + this.message + "\n" + codeLine + "\n" + marker; }; + exports.nameWhitespaceCharacter = function(string) { + switch (string) { + case ' ': + return 'space'; + case '\n': + return 'newline'; + case '\r': + return 'carriage return'; + case '\t': + return 'tab'; + default: + return string; + } + }; + }).call(this); diff --git a/lib/coffee-script/lexer.js b/lib/coffee-script/lexer.js index 41de5487..1a99d8c9 100644 --- a/lib/coffee-script/lexer.js +++ b/lib/coffee-script/lexer.js @@ -588,14 +588,14 @@ }; Lexer.prototype.interpolateString = function(str, options) { - var column, expr, heredoc, i, inner, interpolated, len, letter, lexedLength, line, locationToken, nested, offsetInChunk, pi, plusToken, popped, regex, rparen, strOffset, tag, token, tokens, value, _i, _len, _ref2, _ref3, _ref4; + var column, errorToken, expr, heredoc, i, inner, interpolated, len, letter, lexedLength, line, locationToken, nested, offsetInChunk, pi, plusToken, popped, regex, rparen, strOffset, tag, token, tokens, value, _i, _len, _ref2, _ref3, _ref4; if (options == null) { options = {}; } heredoc = options.heredoc, regex = options.regex, offsetInChunk = options.offsetInChunk, strOffset = options.strOffset, lexedLength = options.lexedLength; - offsetInChunk = offsetInChunk || 0; - strOffset = strOffset || 0; - lexedLength = lexedLength || str.length; + offsetInChunk || (offsetInChunk = 0); + strOffset || (strOffset = 0); + lexedLength || (lexedLength = str.length); tokens = []; pi = 0; i = -1; @@ -610,6 +610,9 @@ if (pi < i) { tokens.push(this.makeToken('NEOSTRING', str.slice(pi, i), strOffset + pi)); } + if (!errorToken) { + errorToken = this.makeToken('', 'string interpolation', offsetInChunk + i + 1, 2); + } inner = expr.slice(1, -1); if (inner.length) { _ref2 = this.getLineAndColumnFromChunk(strOffset + i + 1), line = _ref2[0], column = _ref2[1]; @@ -646,7 +649,7 @@ tokens.unshift(this.makeToken('NEOSTRING', '', offsetInChunk)); } if (interpolated = tokens.length > 1) { - this.token('(', '(', offsetInChunk, 0); + this.token('(', '(', offsetInChunk, 0, errorToken); } for (i = _i = 0, _len = tokens.length; _i < _len; i = ++_i) { token = tokens[i]; @@ -731,9 +734,12 @@ return token; }; - Lexer.prototype.token = function(tag, value, offsetInChunk, length) { + Lexer.prototype.token = function(tag, value, offsetInChunk, length, origin) { var token; token = this.makeToken(tag, value, offsetInChunk, length); + if (origin) { + token.origin = origin; + } this.tokens.push(token); return token; }; diff --git a/lib/coffee-script/rewriter.js b/lib/coffee-script/rewriter.js index 0fedb120..bcf4cb79 100644 --- a/lib/coffee-script/rewriter.js +++ b/lib/coffee-script/rewriter.js @@ -4,10 +4,13 @@ __indexOf = [].indexOf || function(item) { for (var i = 0, l = this.length; i < l; i++) { if (i in this && this[i] === item) return i; } return -1; }, __slice = [].slice; - generate = function(tag, value) { + generate = function(tag, value, origin) { var tok; tok = [tag, value]; tok.generated = true; + if (origin) { + tok.origin = origin; + } return tok; }; @@ -212,7 +215,7 @@ ours: true } ]); - tokens.splice(idx, 0, generate('{', generate(new String('{')))); + tokens.splice(idx, 0, generate('{', generate(new String('{')), token)); if (j == null) { return i += 1; } @@ -220,7 +223,7 @@ endImplicitObject = function(j) { j = j != null ? j : i; stack.pop(); - tokens.splice(j, 0, generate('}', '}')); + tokens.splice(j, 0, generate('}', '}', token)); return i += 1; }; if (inImplicitCall() && (tag === 'IF' || tag === 'TRY' || tag === 'FINALLY' || tag === 'CATCH' || tag === 'CLASS' || tag === 'SWITCH')) { diff --git a/src/coffee-script.coffee b/src/coffee-script.coffee index 69f15386..7d1b1c23 100644 --- a/src/coffee-script.coffee +++ b/src/coffee-script.coffee @@ -183,6 +183,7 @@ parser.lexer = token = @tokens[@pos++] if token [tag, @yytext, @yylloc] = token + @errorToken = token.origin or token @yylineno = @yylloc.first_line else tag = '' @@ -198,12 +199,21 @@ parser.yy = require './nodes' # Override Jison's default error handling function. parser.yy.parseError = (message, {token}) -> # Disregard Jison's message, it contains redundant line numer information. - message = "unexpected #{if token is 1 then 'end of input' else token}" + # Disregard the token, we take its value directly from the lexer in case + # the error is caused by a generated token which might refer to its origin. + {errorToken, tokens} = parser.lexer + [ignored, errorText, errorLoc] = errorToken + + errorText = if errorToken is tokens[tokens.length - 1] + 'end of input' + else + helpers.nameWhitespaceCharacter errorText + # The second argument has a `loc` property, which should have the location # data for this token. Unfortunately, Jison seems to send an outdated `loc` # (from the previous token), so we take the location information directly # from the lexer. - helpers.throwSyntaxError message, parser.lexer.yylloc + helpers.throwSyntaxError "unexpected #{errorText}", errorLoc # Based on http://v8.googlecode.com/svn/branches/bleeding_edge/src/messages.js # Modified to handle sourceMap diff --git a/src/helpers.coffee b/src/helpers.coffee index 60b2259d..98792cd0 100644 --- a/src/helpers.coffee +++ b/src/helpers.coffee @@ -188,3 +188,11 @@ syntaxErrorToString = -> #{codeLine} #{marker} """ + +exports.nameWhitespaceCharacter = (string) -> + switch string + when ' ' then 'space' + when '\n' then 'newline' + when '\r' then 'carriage return' + when '\t' then 'tab' + else string diff --git a/src/lexer.coffee b/src/lexer.coffee index 985c4042..3356d875 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -520,9 +520,9 @@ exports.Lexer = class Lexer # current chunk. interpolateString: (str, options = {}) -> {heredoc, regex, offsetInChunk, strOffset, lexedLength} = options - offsetInChunk = offsetInChunk || 0 - strOffset = strOffset || 0 - lexedLength = lexedLength || str.length + offsetInChunk ||= 0 + strOffset ||= 0 + lexedLength ||= str.length # Parse the string. tokens = [] @@ -537,6 +537,8 @@ exports.Lexer = class Lexer continue # NEOSTRING is a fake token. This will be converted to a string below. tokens.push @makeToken('NEOSTRING', str[pi...i], strOffset + pi) if pi < i + unless errorToken + errorToken = @makeToken '', 'string interpolation', offsetInChunk + i + 1, 2 inner = expr[1...-1] if inner.length [line, column] = @getLineAndColumnFromChunk(strOffset + i + 1) @@ -562,7 +564,9 @@ exports.Lexer = class Lexer # If the first token is not a string, add a fake empty string to the beginning. tokens.unshift @makeToken('NEOSTRING', '', offsetInChunk) unless tokens[0][0] is 'NEOSTRING' - @token '(', '(', offsetInChunk, 0 if interpolated = tokens.length > 1 + if interpolated = tokens.length > 1 + @token '(', '(', offsetInChunk, 0, errorToken + # Push all the tokens for token, i in tokens [tag, value] = token @@ -656,8 +660,9 @@ exports.Lexer = class Lexer # not specified, the length of `value` will be used. # # Returns the new token. - token: (tag, value, offsetInChunk, length) -> + token: (tag, value, offsetInChunk, length, origin) -> token = @makeToken tag, value, offsetInChunk, length + token.origin = origin if origin @tokens.push token token diff --git a/src/rewriter.coffee b/src/rewriter.coffee index 9001372f..68ea23e7 100644 --- a/src/rewriter.coffee +++ b/src/rewriter.coffee @@ -6,9 +6,10 @@ # parentheses, and generally clean things up. # Create a generated token: one that exists due to a use of implicit syntax. -generate = (tag, value) -> +generate = (tag, value, origin) -> tok = [tag, value] tok.generated = yes + tok.origin = origin if origin tok # The **Rewriter** class is used by the [Lexer](lexer.html), directly against @@ -167,13 +168,13 @@ class exports.Rewriter startImplicitObject = (j, startsLine = yes) -> idx = j ? i stack.push ['{', idx, sameLine: yes, startsLine: startsLine, ours: yes] - tokens.splice idx, 0, generate '{', generate(new String('{')) + tokens.splice idx, 0, generate '{', generate(new String('{')), token i += 1 if not j? endImplicitObject = (j) -> j = j ? i stack.pop() - tokens.splice j, 0, generate '}', '}' + tokens.splice j, 0, generate '}', '}', token i += 1 # Don't end an implicit call on next indent if any of these are in an argument diff --git a/test/error_messages.coffee b/test/error_messages.coffee index 199d1f48..e6d084b4 100644 --- a/test/error_messages.coffee +++ b/test/error_messages.coffee @@ -26,7 +26,7 @@ test "parser error formating", -> foo in bar or in baz ''', ''' - [stdin]:1:15: error: unexpected RELATION + [stdin]:1:15: error: unexpected in foo in bar or in baz ^^ ''' @@ -58,9 +58,44 @@ test "#2849: compilation error in a require()d file", -> require './test/syntax-error' ''', """ - #{path.join __dirname, 'syntax-error.coffee'}:1:15: error: unexpected RELATION + #{path.join __dirname, 'syntax-error.coffee'}:1:15: error: unexpected in foo in bar or in baz ^^ """ finally - fs.unlink 'test/syntax-error.coffee' \ No newline at end of file + fs.unlink 'test/syntax-error.coffee' + +test "#1096: unexpected generated tokens", -> + # Unexpected interpolation + assertErrorFormat '{"#{key}": val}', ''' + [stdin]:1:3: error: unexpected string interpolation + {"#{key}": val} + ^^ + ''' + # Implicit ends + assertErrorFormat 'a:, b', ''' + [stdin]:1:3: error: unexpected , + a:, b + ^ + ''' + # Explicit ends + assertErrorFormat '(a:)', ''' + [stdin]:1:4: error: unexpected ) + (a:) + ^ + ''' + # Unexpected end of file + assertErrorFormat 'a:', ''' + [stdin]:1:3: error: unexpected end of input + a: + ^ + ''' + # Unexpected implicit object + assertErrorFormat ''' + for i in [1]: + 1 + ''', ''' + [stdin]:1:13: error: unexpected : + for i in [1]: + ^ + '''