From e89d7a77568401ca008e251c24e1484924078fd9 Mon Sep 17 00:00:00 2001 From: satyr Date: Mon, 11 Oct 2010 19:10:30 +0900 Subject: [PATCH] lexer: fixed a regression where assignmentError wasn't fired --- lib/lexer.js | 57 ++++++++++++++++++------------------- src/lexer.coffee | 37 +++++++++++++----------- test/test_assignment.coffee | 6 +++- 3 files changed, 53 insertions(+), 47 deletions(-) diff --git a/lib/lexer.js b/lib/lexer.js index ea5c151e..54052e54 100644 --- a/lib/lexer.js +++ b/lib/lexer.js @@ -29,17 +29,17 @@ return (new Rewriter).rewrite(this.tokens); }; Lexer.prototype.identifierToken = function() { - var forcedIdentifier, id, match, tag; + var _ref2, colon, forcedIdentifier, id, input, match, tag; if (!(match = IDENTIFIER.exec(this.chunk))) { return false; } - id = match[0]; - this.i += id.length; + _ref2 = match, input = _ref2[0], id = _ref2[1], colon = _ref2[2]; + this.i += input.length; if (id === 'all' && this.tag() === 'FOR') { this.token('ALL', id); return true; } - forcedIdentifier = this.tagAccessor() || ASSIGNED.test(this.chunk); + forcedIdentifier = colon || this.tagAccessor(); tag = 'IDENTIFIER'; if (include(JS_KEYWORDS, id) || !forcedIdentifier && include(COFFEE_KEYWORDS, id)) { tag = id.toUpperCase(); @@ -85,6 +85,9 @@ } } this.token(tag, id); + if (colon) { + this.token(':', ':'); + } return true; }; Lexer.prototype.numberToken = function() { @@ -241,7 +244,7 @@ prev = last(this.tokens, 1); size = indent.length - 1 - indent.lastIndexOf('\n'); nextCharacter = NEXT_CHARACTER.exec(this.chunk)[1]; - noNewlines = (('.' === nextCharacter || ',' === nextCharacter)) || this.unfinished(); + noNewlines = ('.' === nextCharacter || ',' === nextCharacter) || this.unfinished(); if (size - this.indebt === this.indent) { if (noNewlines) { return this.suppressNewlines(); @@ -332,9 +335,9 @@ if (!prev[1].reserved && include(JS_FORBIDDEN, prev[1])) { this.assignmentError(); } - if (('or' === (_ref2 = prev[1]) || 'and' === _ref2)) { + if (('||' === (_ref2 = prev[1]) || '&&' === _ref2)) { prev[0] = 'COMPOUND_ASSIGN'; - prev[1] = COFFEE_ALIASES[prev[1]] + '='; + prev[1] += '='; return true; } } @@ -376,25 +379,23 @@ return true; }; Lexer.prototype.tagAccessor = function() { - var accessor, prev; + var prev; if (!(prev = last(this.tokens)) || prev.spaced) { return false; } - accessor = (function() { - if (prev[1] === '::') { - return this.tag(0, 'PROTOTYPE_ACCESS'); - } else if (prev[1] === '.' && this.value(1) !== '.') { - if (this.tag(1) === '?') { - this.tag(0, 'SOAK_ACCESS'); - return this.tokens.splice(-2, 1); - } else { - return this.tag(0, 'PROPERTY_ACCESS'); - } + if (prev[1] === '::') { + this.tag(0, 'PROTOTYPE_ACCESS'); + } else if (prev[1] === '.' && this.value(1) !== '.') { + if (this.tag(1) === '?') { + this.tag(0, 'SOAK_ACCESS'); + this.tokens.splice(-2, 1); } else { - return prev[0] === '@'; + this.tag(0, 'PROPERTY_ACCESS'); } - }).call(this); - return accessor ? 'accessor' : false; + } else { + return prev[0] === '@'; + } + return true; }; Lexer.prototype.sanitizeHeredoc = function(doc, options) { var _ref2, _ref3, attempt, herecomment, indent, match; @@ -424,10 +425,7 @@ return; } i = this.tokens.length; - while (true) { - if (!(tok = this.tokens[--i])) { - return; - } + while (tok = this.tokens[--i]) { switch (tok[0]) { case 'IDENTIFIER': tok[0] = 'PARAM'; @@ -437,7 +435,8 @@ break; case '(': case 'CALL_START': - return (tok[0] = 'PARAM_START'); + tok[0] = 'PARAM_START'; + return true; } } return true; @@ -446,10 +445,10 @@ return this.outdentToken(this.indent); }; Lexer.prototype.identifierError = function(word) { - throw new Error("SyntaxError: Reserved word \"" + word + "\" on line " + (this.line + 1)); + throw SyntaxError("Reserved word \"" + word + "\" on line " + (this.line + 1)); }; Lexer.prototype.assignmentError = function() { - throw new Error("SyntaxError: Reserved word \"" + (this.value()) + "\" on line " + (this.line + 1) + " can't be assigned"); + throw SyntaxError("Reserved word \"" + (this.value()) + "\" on line " + (this.line + 1) + " can't be assigned"); }; Lexer.prototype.balancedString = function(str, delimited, options) { var _i, _len, _ref2, close, i, levels, open, pair, slen; @@ -598,7 +597,7 @@ COFFEE_ALIASES['==='] = '=='; RESERVED = ['case', 'default', 'do', 'function', 'var', 'void', 'with', 'const', 'let', 'enum', 'export', 'import', 'native', '__hasProp', '__extends', '__slice']; JS_FORBIDDEN = JS_KEYWORDS.concat(RESERVED); - IDENTIFIER = /^[a-zA-Z_$][\w$]*/; + IDENTIFIER = /^([$A-Za-z_][$\w]*)([^\n\S]*:(?!:))?/; NUMBER = /^0x[\da-f]+|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i; HEREDOC = /^("""|''')([\s\S]*?)(?:\n[ \t]*)?\1/; OPERATOR = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)/; diff --git a/src/lexer.coffee b/src/lexer.coffee index cc7bbeb8..8d80941d 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -73,12 +73,12 @@ exports.Lexer = class Lexer # though `is` means `===` otherwise. identifierToken: -> return false unless match = IDENTIFIER.exec @chunk - id = match[0] - @i += id.length + [input, id, colon] = match + @i += input.length if id is 'all' and @tag() is 'FOR' @token 'ALL', id return true - forcedIdentifier = @tagAccessor() or ASSIGNED.test @chunk + forcedIdentifier = colon or @tagAccessor() tag = 'IDENTIFIER' if include(JS_KEYWORDS, id) or not forcedIdentifier and include(COFFEE_KEYWORDS, id) @@ -103,7 +103,7 @@ exports.Lexer = class Lexer tag = 'IDENTIFIER' id = new String id id.reserved = yes - else if include(RESERVED, id) + else if include RESERVED, id @identifierError id unless forcedIdentifier tag = id = COFFEE_ALIASES[id] if COFFEE_ALIASES.hasOwnProperty id @@ -115,6 +115,7 @@ exports.Lexer = class Lexer id = tag.toLowerCase() tag = 'BOOL' @token tag, id + @token ':', ':' if colon true # Matches numbers, including decimals, hex, and exponential notation. @@ -236,7 +237,7 @@ exports.Lexer = class Lexer prev = last @tokens, 1 size = indent.length - 1 - indent.lastIndexOf '\n' nextCharacter = NEXT_CHARACTER.exec(@chunk)[1] - noNewlines = (nextCharacter in ['.', ',']) or @unfinished() + noNewlines = nextCharacter in ['.', ','] or @unfinished() if size - @indebt is @indent return @suppressNewlines() if noNewlines return @newlineToken indent @@ -313,9 +314,9 @@ exports.Lexer = class Lexer prev = last @tokens if value is '=' @assignmentError() if not prev[1].reserved and include JS_FORBIDDEN, prev[1] - if prev[1] in ['or', 'and'] + if prev[1] in ['||', '&&'] prev[0] = 'COMPOUND_ASSIGN' - prev[1] = COFFEE_ALIASES[prev[1]] + '=' + prev[1] += '=' return true if ';' is value then tag = 'TERMINATOR' else if include LOGIC , value then tag = 'LOGIC' @@ -345,17 +346,17 @@ exports.Lexer = class Lexer # is the previous token. tagAccessor: -> return false if not (prev = last @tokens) or prev.spaced - accessor = if prev[1] is '::' + if prev[1] is '::' @tag 0, 'PROTOTYPE_ACCESS' else if prev[1] is '.' and @value(1) isnt '.' if @tag(1) is '?' - @tag(0, 'SOAK_ACCESS') + @tag 0, 'SOAK_ACCESS' @tokens.splice(-2, 1) else @tag 0, 'PROPERTY_ACCESS' else - prev[0] is '@' - if accessor then 'accessor' else false + return prev[0] is '@' + true # Sanitize a heredoc or herecomment by # erasing all external indentation on the left-hand side. @@ -376,12 +377,11 @@ exports.Lexer = class Lexer tagParameters: -> return if @tag() isnt ')' i = @tokens.length - loop - return unless tok = @tokens[--i] + while tok = @tokens[--i] switch tok[0] when 'IDENTIFIER' then tok[0] = 'PARAM' when ')' then tok[0] = 'PARAM_END' - when '(', 'CALL_START' then return tok[0] = 'PARAM_START' + when '(', 'CALL_START' then tok[0] = 'PARAM_START'; return true true # Close up all remaining open blocks at the end of the file. @@ -391,12 +391,12 @@ exports.Lexer = class Lexer # The error for when you try to use a forbidden word in JavaScript as # an identifier. identifierError: (word) -> - throw new Error "SyntaxError: Reserved word \"#{word}\" on line #{@line + 1}" + throw SyntaxError "Reserved word \"#{word}\" on line #{@line + 1}" # The error for when you try to assign to a reserved word in JavaScript, # like "function" or "default". assignmentError: -> - throw new Error "SyntaxError: Reserved word \"#{@value()}\" on line #{@line + 1} can't be assigned" + throw SyntaxError "Reserved word \"#{@value()}\" on line #{@line + 1} can't be assigned" # Matches a balanced group such as a single or double-quoted string. Pass in # a series of delimiters, all of which must be nested correctly within the @@ -547,7 +547,10 @@ RESERVED = [ JS_FORBIDDEN = JS_KEYWORDS.concat RESERVED # Token matching regexes. -IDENTIFIER = /^[a-zA-Z_$][\w$]*/ +IDENTIFIER = /// ^ + ( [$A-Za-z_][$\w]* ) + ( [^\n\S]* : (?!:) )? # Is this a property name? +/// NUMBER = /^0x[\da-f]+|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i HEREDOC = /^("""|''')([\s\S]*?)(?:\n[ \t]*)?\1/ OPERATOR = /// ^ (?: -[-=>]? | \+[+=]? | [*&|/%=<>^:!?]+ ) /// diff --git a/test/test_assignment.coffee b/test/test_assignment.coffee index a9632ea7..00d3e10d 100644 --- a/test/test_assignment.coffee +++ b/test/test_assignment.coffee @@ -26,4 +26,8 @@ tester = -> @example = -> 'example function' this -ok tester().example() is 'example function' \ No newline at end of file +ok tester().example() is 'example function' + + +try throw CoffeeScript.tokens 'in = 1' +catch e then eq e.message, 'Reserved word "in" on line 1 can\'t be assigned'