fixes #1195: lex out semicolons before newlines and at end of program
This commit is contained in:
parent
d1af5163eb
commit
76b6a1771b
|
@ -275,6 +275,9 @@
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (dent) this.outdebt -= moveOut;
|
if (dent) this.outdebt -= moveOut;
|
||||||
|
while (this.value() === ';') {
|
||||||
|
this.tokens.pop();
|
||||||
|
}
|
||||||
if (!(this.tag() === 'TERMINATOR' || noNewlines)) {
|
if (!(this.tag() === 'TERMINATOR' || noNewlines)) {
|
||||||
this.token('TERMINATOR', '\n');
|
this.token('TERMINATOR', '\n');
|
||||||
}
|
}
|
||||||
|
@ -294,6 +297,9 @@
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
Lexer.prototype.newlineToken = function() {
|
Lexer.prototype.newlineToken = function() {
|
||||||
|
while (this.value() === ';') {
|
||||||
|
this.tokens.pop();
|
||||||
|
}
|
||||||
if (this.tag() !== 'TERMINATOR') this.token('TERMINATOR', '\n');
|
if (this.tag() !== 'TERMINATOR') this.token('TERMINATOR', '\n');
|
||||||
return this;
|
return this;
|
||||||
};
|
};
|
||||||
|
|
|
@ -279,6 +279,7 @@ exports.Lexer = class Lexer
|
||||||
@outdebt = 0
|
@outdebt = 0
|
||||||
@token 'OUTDENT', dent
|
@token 'OUTDENT', dent
|
||||||
@outdebt -= moveOut if dent
|
@outdebt -= moveOut if dent
|
||||||
|
@tokens.pop() while @value() is ';'
|
||||||
@token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR' or noNewlines
|
@token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR' or noNewlines
|
||||||
this
|
this
|
||||||
|
|
||||||
|
@ -293,6 +294,7 @@ exports.Lexer = class Lexer
|
||||||
|
|
||||||
# Generate a newline token. Consecutive newlines get merged together.
|
# Generate a newline token. Consecutive newlines get merged together.
|
||||||
newlineToken: ->
|
newlineToken: ->
|
||||||
|
@tokens.pop() while @value() is ';'
|
||||||
@token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR'
|
@token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR'
|
||||||
this
|
this
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue