diff --git a/lib/coffee-script/lexer.js b/lib/coffee-script/lexer.js index 5c97c7fb..916e9cc2 100644 --- a/lib/coffee-script/lexer.js +++ b/lib/coffee-script/lexer.js @@ -17,7 +17,6 @@ opts = {}; } this.literate = opts.literate; - code = this.clean(code); this.indent = 0; this.indebt = 0; this.outdebt = 0; @@ -26,6 +25,7 @@ this.tokens = []; this.chunkLine = opts.line || 0; this.chunkColumn = opts.column || 0; + code = this.clean(code); i = 0; while (this.chunk = code.slice(i)) { consumed = this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.heredocToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.jsToken() || this.literalToken(); @@ -47,10 +47,11 @@ if (code.charCodeAt(0) === BOM) { code = code.slice(1); } + code = code.replace(/\r/g, '').replace(TRAILING_SPACES, ''); if (WHITESPACE.test(code)) { code = "\n" + code; + this.chunkLine--; } - code = code.replace(/\r/g, '').replace(TRAILING_SPACES, ''); if (this.literate) { lines = (function() { var _i, _len, _ref2, _results; diff --git a/src/lexer.coffee b/src/lexer.coffee index 93aa762d..39b4d5e6 100644 --- a/src/lexer.coffee +++ b/src/lexer.coffee @@ -35,7 +35,6 @@ exports.Lexer = class Lexer # unless explicitly asked not to. tokenize: (code, opts = {}) -> @literate = opts.literate # Are we lexing literate CoffeeScript? - code = @clean code # The stripped, cleaned original source code. @indent = 0 # The current indentation level. @indebt = 0 # The over-indentation at the current level. @outdebt = 0 # The under-outdentation at the current level. @@ -47,6 +46,7 @@ exports.Lexer = class Lexer opts.line or 0 # The start line for the current @chunk. @chunkColumn = opts.column or 0 # The start column of the current @chunk. + code = @clean code # The stripped, cleaned original source code. # At every position, run through this list of attempted matches, # short-circuiting if any of them succeed. Their order determines precedence: @@ -80,8 +80,10 @@ exports.Lexer = class Lexer # by removing all lines that aren't indented by at least four spaces or a tab. clean: (code) -> code = code.slice(1) if code.charCodeAt(0) is BOM - code = "\n#{code}" if WHITESPACE.test code code = code.replace(/\r/g, '').replace TRAILING_SPACES, '' + if WHITESPACE.test code + code = "\n#{code}" + @chunkLine-- if @literate lines = for line in code.split('\n') if match = LITERATE.exec line diff --git a/test/location.coffee b/test/location.coffee index d15002ff..1d9d6a18 100644 --- a/test/location.coffee +++ b/test/location.coffee @@ -38,6 +38,26 @@ test "Verify location of generated tokens", -> eq numberToken[2].last_line, 0 eq numberToken[2].last_column, 5 +test "Verify location of generated tokens (with indented first line)", -> + tokens = CoffeeScript.tokens " a = 83" + + eq tokens.length, 6 + [IndentToken, aToken, equalsToken, numberToken] = tokens + + eq aToken[2].first_line, 0 + eq aToken[2].first_column, 2 + eq aToken[2].last_line, 0 + eq aToken[2].last_column, 2 + + eq equalsToken[2].first_line, 0 + eq equalsToken[2].first_column, 4 + eq equalsToken[2].last_line, 0 + eq equalsToken[2].last_column, 4 + + eq numberToken[2].first_line, 0 + eq numberToken[2].first_column, 6 + eq numberToken[2].last_line, 0 + eq numberToken[2].last_column, 7 test "Verify all tokens get a location", -> doesNotThrow ->