diff --git a/lib/coffee-script/lexer.js b/lib/coffee-script/lexer.js index 62055335..78e9ec0d 100644 --- a/lib/coffee-script/lexer.js +++ b/lib/coffee-script/lexer.js @@ -1,6 +1,6 @@ // Generated by CoffeeScript 1.5.0-pre (function() { - var BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, HEREDOC, HEREDOC_ILLEGAL, HEREDOC_INDENT, HEREGEX, HEREGEX_OMIT, IDENTIFIER, INDEXABLE, INVERSES, JSTOKEN, JS_FORBIDDEN, JS_KEYWORDS, LINE_BREAK, LINE_CONTINUER, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NOT_REGEX, NOT_SPACED_REGEX, NUMBER, OPERATOR, REGEX, RELATION, RESERVED, Rewriter, SHIFT, SIMPLESTR, STRICT_PROSCRIBED, TRAILING_SPACES, UNARY, WHITESPACE, compact, count, key, last, starts, _ref, _ref1, + var BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, HEREDOC, HEREDOC_ILLEGAL, HEREDOC_INDENT, HEREGEX, HEREGEX_OMIT, IDENTIFIER, INDEXABLE, INVERSES, JSTOKEN, JS_FORBIDDEN, JS_KEYWORDS, LINE_BREAK, LINE_CONTINUER, LITERATE, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NOT_REGEX, NOT_SPACED_REGEX, NUMBER, OPERATOR, REGEX, RELATION, RESERVED, Rewriter, SHIFT, SIMPLESTR, STRICT_PROSCRIBED, TRAILING_SPACES, UNARY, WHITESPACE, compact, count, key, last, starts, _ref, _ref1, __indexOf = [].indexOf || function(item) { for (var i = 0, l = this.length; i < l; i++) { if (i in this && this[i] === item) return i; } return -1; }; _ref = require('./rewriter'), Rewriter = _ref.Rewriter, INVERSES = _ref.INVERSES; @@ -52,8 +52,10 @@ _results = []; for (_i = 0, _len = _ref2.length; _i < _len; _i++) { line = _ref2[_i]; - if (line.substr(0, 4) === ' ') { + if (LITERATE.test(line)) { _results.push(line.substr(4)); + } else { + _results.push('# ' + line); } } return _results; @@ -756,6 +758,8 @@ COMMENT = /^###([^#][\s\S]*?)(?:###[^\n\S]*|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/; + LITERATE = /^([ ]{4}|\t)/; + CODE = /^[-=]>/; MULTI_DENT = /^(?:\n[^\n\S]*)+/; diff --git a/src/lexer.coffee b/src/lexer.coffee index 3c5f8794..f4e451c2 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -31,8 +31,8 @@ exports.Lexer = class Lexer # # Before returning the token stream, run it through the [Rewriter](rewriter.html) # unless explicitly asked not to. - tokenize: (code, opts = {}) -> - @literate = opts.literate # Are we lexing literate CoffeeScript? + tokenize: (code, opts = {}) -> + @literate = opts.literate # Are we lexing literate CoffeeScript? code = @clean code # The stripped, cleaned original source code. @line = opts.line or 0 # The current line. @indent = 0 # The current indentation level. @@ -62,16 +62,19 @@ exports.Lexer = class Lexer @error "missing #{tag}" if tag = @ends.pop() return @tokens if opts.rewrite is off (new Rewriter).rewrite @tokens - - # Preprocess the code to remove leading and trailing whitespace, carriage - # returns, etc. If we're lexing literate CoffeeScript, strip external Markdown + + # Preprocess the code to remove leading and trailing whitespace, carriage + # returns, etc. If we're lexing literate CoffeeScript, strip external Markdown # by removing all lines that aren't indented by at least four spaces. clean: (code) -> code = "\n#{code}" if WHITESPACE.test code code = code.replace(/\r/g, '').replace TRAILING_SPACES, '' if @literate - lines = for line in code.split('\n') when line.substr(0, 4) is ' ' - line.substr(4) + lines = for line in code.split('\n') + if LITERATE.test line + line.substr(4) + else + '# ' + line code = lines.join '\n' code @@ -631,6 +634,8 @@ WHITESPACE = /^[^\n\S]+/ COMMENT = /^###([^#][\s\S]*?)(?:###[^\n\S]*|(?:###)?$)|^(?:\s*#(?!##[^#]).*)+/ +LITERATE = /^([ ]{4}|\t)/ + CODE = /^[-=]>/ MULTI_DENT = /^(?:\n[^\n\S]*)+/