diff --git a/lib/lexer.js b/lib/lexer.js index 027d3e33..d9affe42 100644 --- a/lib/lexer.js +++ b/lib/lexer.js @@ -1,5 +1,5 @@ (function() { - var ASSIGNED, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, CONVERSIONS, HEREDOC, HEREDOC_INDENT, HEREGEX, HEREGEX_OMIT, IDENTIFIER, JSTOKEN, JS_FORBIDDEN, JS_KEYWORDS, LEADING_SPACES, LINE_BREAK, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NEXT_CHARACTER, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX, RESERVED, Rewriter, SHIFT, SIMPLESTR, TRAILING_SPACES, UNARY, WHITESPACE, _ref, compact, count, include, last, starts; + var ASSIGNED, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, CONVERSIONS, HEREDOC, HEREDOC_INDENT, HEREGEX, HEREGEX_OMIT, IDENTIFIER, JSTOKEN, JS_FORBIDDEN, JS_KEYWORDS, LEADING_SPACES, LINE_BREAK, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NEGATABLE, NEXT_CHARACTER, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX, RESERVED, Rewriter, SHIFT, SIMPLESTR, TRAILING_SPACES, UNARY, WHITESPACE, _ref, compact, count, include, last, starts; Rewriter = require('./rewriter').Rewriter; _ref = require('./helpers'), include = _ref.include, count = _ref.count, starts = _ref.starts, compact = _ref.compact, last = _ref.last; exports.Lexer = (function() { @@ -309,9 +309,9 @@ return true; }; Lexer.prototype.literalToken = function() { - var _ref2, match, prev, space, spaced, tag, val, value; - if (match = this.chunk.match(OPERATOR)) { - _ref2 = match, value = _ref2[0], space = _ref2[1]; + var match, prev, ptag, pval, tag, value; + if (match = OPERATOR.exec(this.chunk)) { + value = match[0]; if (CODE.test(value)) { this.tagParameters(); } @@ -319,19 +319,19 @@ value = this.chunk.charAt(0); } this.i += value.length; - prev = last(this.tokens); - spaced = ((prev != null) ? prev.spaced : undefined); tag = value; if (value === '=') { - if (include(JS_FORBIDDEN, val = this.value())) { + if (include(JS_FORBIDDEN, pval = this.value())) { this.assignmentError(); } - if (('or' === val || 'and' === val)) { - this.tokens.splice(-1, 1, ['COMPOUND_ASSIGN', CONVERSIONS[val] + '=', prev[2]]); + if (('or' === pval || 'and' === pval)) { + prev = last(this.tokens); + prev[0] = 'COMPOUND_ASSIGN'; + prev[1] = CONVERSIONS[pval] + '='; return true; } } - if (value === ';') { + if (';' === value) { tag = 'TERMINATOR'; } else if (include(LOGIC, value)) { tag = 'LOGIC'; @@ -345,20 +345,20 @@ tag = 'UNARY'; } else if (include(SHIFT, value)) { tag = 'SHIFT'; - } else if (include(CALLABLE, this.tag()) && !spaced) { + } else if ((prev = last(this.tokens)) && !prev.spaced && include(CALLABLE, ptag = prev[0])) { if (value === '(') { - if (prev[0] === '?') { + if (ptag === '?') { prev[0] = 'FUNC_EXIST'; } tag = 'CALL_START'; } else if (value === '[') { tag = 'INDEX_START'; - switch (this.tag()) { + switch (ptag) { case '?': - this.tag(0, 'INDEX_SOAK'); + prev[0] = 'INDEX_SOAK'; break; case '::': - this.tag(0, 'INDEX_PROTO'); + prev[0] = 'INDEX_PROTO'; break; } } @@ -547,19 +547,13 @@ Lexer.prototype.token = function(tag, value) { return this.tokens.push([tag, value, this.line]); }; - Lexer.prototype.tag = function(index, newTag) { + Lexer.prototype.tag = function(index, tag) { var tok; - if (!(tok = last(this.tokens, index))) { - return null; - } - return (tok[0] = (newTag != null) ? newTag : tok[0]); + return (tok = last(this.tokens, index)) && ((tag != null) ? (tok[0] = tag) : tok[0]); }; Lexer.prototype.value = function(index, val) { var tok; - if (!(tok = last(this.tokens, index))) { - return null; - } - return (tok[1] = (val != null) ? val : tok[1]); + return (tok = last(this.tokens, index)) && ((val != null) ? (tok[1] = val) : tok[1]); }; Lexer.prototype.unfinished = function() { var prev, value; @@ -585,7 +579,7 @@ IDENTIFIER = /^[a-zA-Z_$][\w$]*/; NUMBER = /^0x[\da-f]+|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i; HEREDOC = /^("""|''')([\s\S]*?)(?:\n[ \t]*)?\1/; - OPERATOR = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)(?=([ \t]*))/; + OPERATOR = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)/; WHITESPACE = /^[ \t]+/; COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/; CODE = /^[-=]>/; @@ -610,6 +604,7 @@ MATH = ['*', '/', '%']; NOT_REGEX = ['NUMBER', 'REGEX', '++', '--', 'FALSE', 'NULL', 'TRUE', ']']; CALLABLE = ['IDENTIFIER', 'SUPER', ')', ']', '}', 'STRING', '@', 'THIS', '?', '::']; + NEGATABLE = ['IN', 'OF', 'INSTANCEOF']; LINE_BREAK = ['INDENT', 'OUTDENT', 'TERMINATOR']; CONVERSIONS = { 'and': '&&', diff --git a/src/lexer.coffee b/src/lexer.coffee index ae2590ca..785f1fcf 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -292,36 +292,37 @@ exports.Lexer = class Lexer # here. `;` and newlines are both treated as a `TERMINATOR`, we distinguish # parentheses that indicate a method call from regular parentheses, and so on. literalToken: -> - if match = @chunk.match OPERATOR - [value, space] = match + if match = OPERATOR.exec @chunk + [value] = match @tagParameters() if CODE.test value else value = @chunk.charAt 0 @i += value.length - prev = last @tokens - spaced = prev?.spaced tag = value if value is '=' - @assignmentError() if include JS_FORBIDDEN, val = @value() - if val in ['or', 'and'] - @tokens.splice(-1, 1, ['COMPOUND_ASSIGN', CONVERSIONS[val] + '=', prev[2]]) + @assignmentError() if include JS_FORBIDDEN, pval = @value() + if pval in ['or', 'and'] + prev = last @tokens + prev[0] = 'COMPOUND_ASSIGN' + prev[1] = CONVERSIONS[pval] + '=' return true - if value is ';' then tag = 'TERMINATOR' - else if include(LOGIC, value) then tag = 'LOGIC' - else if include(MATH, value) then tag = 'MATH' - else if include(COMPARE, value) then tag = 'COMPARE' - else if include(COMPOUND_ASSIGN, value) then tag = 'COMPOUND_ASSIGN' - else if include(UNARY, value) then tag = 'UNARY' - else if include(SHIFT, value) then tag = 'SHIFT' - else if include(CALLABLE, @tag()) and not spaced + if ';' is value then tag = 'TERMINATOR' + else if include LOGIC , value then tag = 'LOGIC' + else if include MATH , value then tag = 'MATH' + else if include COMPARE , value then tag = 'COMPARE' + else if include COMPOUND_ASSIGN, value then tag = 'COMPOUND_ASSIGN' + else if include UNARY , value then tag = 'UNARY' + else if include SHIFT , value then tag = 'SHIFT' + else if (prev = last @tokens) and not prev.spaced and + include(CALLABLE, ptag = prev[0]) if value is '(' - prev[0] = 'FUNC_EXIST' if prev[0] is '?' + prev[0] = 'FUNC_EXIST' if ptag is '?' tag = 'CALL_START' else if value is '[' tag = 'INDEX_START' - switch @tag() - when '?' then @tag 0, 'INDEX_SOAK' - when '::' then @tag 0, 'INDEX_PROTO' + switch ptag + when '?' then prev[0] = 'INDEX_SOAK' + when '::' then prev[0] = 'INDEX_PROTO' @token tag, value true @@ -468,15 +469,11 @@ exports.Lexer = class Lexer token: (tag, value) -> @tokens.push [tag, value, @line] - # Peek at a tag in the current token stream. - tag: (index, newTag) -> - return unless tok = last @tokens, index - tok[0] = newTag ? tok[0] - - # Peek at a value in the current token stream. + # Peek at a tag/value in the current token stream. + tag : (index, tag) -> + (tok = last @tokens, index) and if tag? then tok[0] = tag else tok[0] value: (index, val) -> - return unless tok = last @tokens, index - tok[1] = val ? tok[1] + (tok = last @tokens, index) and if val? then tok[1] = val else tok[1] # Are we in the midst of an unfinished expression? unfinished: -> @@ -537,7 +534,7 @@ JS_FORBIDDEN = JS_KEYWORDS.concat RESERVED IDENTIFIER = /^[a-zA-Z_$][\w$]*/ NUMBER = /^0x[\da-f]+|^(?:\d+(\.\d+)?|\.\d+)(?:e[+-]?\d+)?/i HEREDOC = /^("""|''')([\s\S]*?)(?:\n[ \t]*)?\1/ -OPERATOR = /^(?:-[-=>]?|\+[+=]?|[*&|\/%=<>^:!?]+)(?=([ \t]*))/ +OPERATOR = /// ^ (?: -[-=>]? | \+[+=]? | [*&|/%=<>^:!?]+ ) /// WHITESPACE = /^[ \t]+/ COMMENT = /^###([^#][\s\S]*?)(?:###[ \t]*\n|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/ CODE = /^[-=]>/