safer lexing at the start of the files
This commit is contained in:
parent
207ec81821
commit
b21780b738
|
@ -194,14 +194,15 @@
|
|||
};
|
||||
// Record tokens for indentation differing from the previous line.
|
||||
lex.prototype.indent_token = function indent_token() {
|
||||
var diff, indent, next_character, no_newlines, size;
|
||||
var diff, indent, next_character, no_newlines, prev, size;
|
||||
if (!((indent = this.match(MULTI_DENT, 1)))) {
|
||||
return false;
|
||||
}
|
||||
this.line += indent.match(MULTILINER).length;
|
||||
this.i += indent.length;
|
||||
next_character = this.chunk.match(MULTI_DENT)[4];
|
||||
no_newlines = next_character === '.' || (this.value().match(NO_NEWLINE) && this.tokens[this.tokens.length - 2][0] !== '.' && !this.value().match(CODE));
|
||||
prev = this.tokens[this.tokens.length - 2];
|
||||
no_newlines = next_character === '.' || (this.value().match(NO_NEWLINE) && prev && (prev[0] !== '.') && !this.value().match(CODE));
|
||||
if (no_newlines) {
|
||||
return this.suppress_newlines(indent);
|
||||
}
|
||||
|
@ -228,9 +229,7 @@
|
|||
this.token('OUTDENT', last_indent);
|
||||
move_out -= last_indent;
|
||||
}
|
||||
if (!(this.tag() === 'TERMINATOR')) {
|
||||
this.token('TERMINATOR', "\n");
|
||||
}
|
||||
return true;
|
||||
};
|
||||
// Matches and consumes non-meaningful whitespace.
|
||||
|
|
|
@ -168,7 +168,8 @@ lex::indent_token: ->
|
|||
@line += indent.match(MULTILINER).length
|
||||
@i += indent.length
|
||||
next_character: @chunk.match(MULTI_DENT)[4]
|
||||
no_newlines: next_character is '.' or (@value().match(NO_NEWLINE) and @tokens[@tokens.length - 2][0] isnt '.' and not @value().match(CODE))
|
||||
prev: @tokens[@tokens.length - 2]
|
||||
no_newlines: next_character is '.' or (@value().match(NO_NEWLINE) and prev and (prev[0] isnt '.') and not @value().match(CODE))
|
||||
return @suppress_newlines(indent) if no_newlines
|
||||
size: indent.match(LAST_DENTS).reverse()[0].match(LAST_DENT)[1].length
|
||||
return @newline_token(indent) if size is @indent
|
||||
|
@ -188,7 +189,7 @@ lex::outdent_token: (move_out) ->
|
|||
last_indent: @indents.pop()
|
||||
@token 'OUTDENT', last_indent
|
||||
move_out -= last_indent
|
||||
@token 'TERMINATOR', "\n" unless @tag() is 'TERMINATOR'
|
||||
@token 'TERMINATOR', "\n"
|
||||
true
|
||||
|
||||
# Matches and consumes non-meaningful whitespace.
|
||||
|
|
Loading…
Reference in New Issue