fixes #1195: lex out semicolons before newlines and at end of program

This commit is contained in:
Gerald Lewis 2011-09-04 12:18:22 -04:00
parent d1af5163eb
commit 76b6a1771b
2 changed files with 8 additions and 0 deletions

View File

@ -275,6 +275,9 @@
}
}
if (dent) this.outdebt -= moveOut;
while (this.value() === ';') {
this.tokens.pop();
}
if (!(this.tag() === 'TERMINATOR' || noNewlines)) {
this.token('TERMINATOR', '\n');
}
@ -294,6 +297,9 @@
}
};
Lexer.prototype.newlineToken = function() {
while (this.value() === ';') {
this.tokens.pop();
}
if (this.tag() !== 'TERMINATOR') this.token('TERMINATOR', '\n');
return this;
};

View File

@ -279,6 +279,7 @@ exports.Lexer = class Lexer
@outdebt = 0
@token 'OUTDENT', dent
@outdebt -= moveOut if dent
@tokens.pop() while @value() is ';'
@token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR' or noNewlines
this
@ -293,6 +294,7 @@ exports.Lexer = class Lexer
# Generate a newline token. Consecutive newlines get merged together.
newlineToken: ->
@tokens.pop() while @value() is ';'
@token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR'
this