fixes #1195: lex out semicolons before newlines and at end of program

This commit is contained in:
Gerald Lewis 2011-09-04 12:18:22 -04:00
parent d1af5163eb
commit 76b6a1771b
2 changed files with 8 additions and 0 deletions

View File

@ -275,6 +275,9 @@
} }
} }
if (dent) this.outdebt -= moveOut; if (dent) this.outdebt -= moveOut;
while (this.value() === ';') {
this.tokens.pop();
}
if (!(this.tag() === 'TERMINATOR' || noNewlines)) { if (!(this.tag() === 'TERMINATOR' || noNewlines)) {
this.token('TERMINATOR', '\n'); this.token('TERMINATOR', '\n');
} }
@ -294,6 +297,9 @@
} }
}; };
Lexer.prototype.newlineToken = function() { Lexer.prototype.newlineToken = function() {
while (this.value() === ';') {
this.tokens.pop();
}
if (this.tag() !== 'TERMINATOR') this.token('TERMINATOR', '\n'); if (this.tag() !== 'TERMINATOR') this.token('TERMINATOR', '\n');
return this; return this;
}; };

View File

@ -279,6 +279,7 @@ exports.Lexer = class Lexer
@outdebt = 0 @outdebt = 0
@token 'OUTDENT', dent @token 'OUTDENT', dent
@outdebt -= moveOut if dent @outdebt -= moveOut if dent
@tokens.pop() while @value() is ';'
@token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR' or noNewlines @token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR' or noNewlines
this this
@ -293,6 +294,7 @@ exports.Lexer = class Lexer
# Generate a newline token. Consecutive newlines get merged together. # Generate a newline token. Consecutive newlines get merged together.
newlineToken: -> newlineToken: ->
@tokens.pop() while @value() is ';'
@token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR' @token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR'
this this