kotovalexarian-likes-github/jashkenas--coffeescript
1
0
Fork 0
mirror of https://github.com/jashkenas/coffeescript.git synced 2022-11-09 12:23:24 -05:00
Code Releases Activity

fixed #1299: overhauled token pairings

Browse source
This commit is contained in:
satyr 2011-09-17 08:26:04 +09:00
parent c5dbb1c933
commit d03d288a98
5 changed files with 105 additions and 166 deletions
Download patch file Download diff file Expand all files Collapse all files

94
lib/coffee-script/lexer.js
View file

@ -1,17 +1,17 @@
(function() {
var BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, HEREDOC, HEREDOC_ILLEGAL, HEREDOC_INDENT, HEREGEX, HEREGEX_OMIT, IDENTIFIER, INDEXABLE, JSTOKEN, JS_FORBIDDEN, JS_KEYWORDS, LINE_BREAK, LINE_CONTINUER, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NOT_REGEX, NOT_SPACED_REGEX, NUMBER, OPERATOR, REGEX, RELATION, RESERVED, Rewriter, SHIFT, SIMPLESTR, TRAILING_SPACES, UNARY, WHITESPACE, compact, count, key, last, starts, _ref;
var BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARE, COMPOUND_ASSIGN, HEREDOC, HEREDOC_ILLEGAL, HEREDOC_INDENT, HEREGEX, HEREGEX_OMIT, IDENTIFIER, INDEXABLE, INVERSES, JSTOKEN, JS_FORBIDDEN, JS_KEYWORDS, LINE_BREAK, LINE_CONTINUER, LOGIC, Lexer, MATH, MULTILINER, MULTI_DENT, NOT_REGEX, NOT_SPACED_REGEX, NUMBER, OPERATOR, REGEX, RELATION, RESERVED, Rewriter, SHIFT, SIMPLESTR, TRAILING_SPACES, UNARY, WHITESPACE, compact, count, key, last, starts, _ref, _ref2;
var __hasProp = Object.prototype.hasOwnProperty, __indexOf = Array.prototype.indexOf || function(item) {
for (var i = 0, l = this.length; i < l; i++) {
if (__hasProp.call(this, i) && this[i] === item) return i;
}
return -1;
};
Rewriter = require('./rewriter').Rewriter;
_ref = require('./helpers'), count = _ref.count, starts = _ref.starts, compact = _ref.compact, last = _ref.last;
_ref = require('./rewriter'), Rewriter = _ref.Rewriter, INVERSES = _ref.INVERSES;
_ref2 = require('./helpers'), count = _ref2.count, starts = _ref2.starts, compact = _ref2.compact, last = _ref2.last;
exports.Lexer = Lexer = (function() {
function Lexer() {}
Lexer.prototype.tokenize = function(code, opts) {
var i;
var i, tag;
if (opts == null) opts = {};
if (WHITESPACE.test(code)) code = "\n" + code;
code = code.replace(/\r/g, '').replace(TRAILING_SPACES, '');
@ -21,28 +21,30 @@
this.indebt = 0;
this.outdebt = 0;
this.indents = [];
this.ends = [];
this.tokens = [];
i = 0;
while (this.chunk = code.slice(i)) {
i += this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.heredocToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.jsToken() || this.literalToken();
}
this.closeIndentation();
if (tag = this.ends.pop()) this.carp("missing " + tag);
if (opts.rewrite === false) return this.tokens;
return (new Rewriter).rewrite(this.tokens);
};
Lexer.prototype.identifierToken = function() {
var colon, forcedIdentifier, id, input, match, prev, tag, _ref2, _ref3;
var colon, forcedIdentifier, id, input, match, prev, tag, _ref3, _ref4;
if (!(match = IDENTIFIER.exec(this.chunk))) return 0;
input = match[0], id = match[1], colon = match[2];
if (id === 'own' && this.tag() === 'FOR') {
this.token('OWN', id);
return id.length;
}
forcedIdentifier = colon || (prev = last(this.tokens)) && (((_ref2 = prev[0]) === '.' || _ref2 === '?.' || _ref2 === '::') || !prev.spaced && prev[0] === '@');
forcedIdentifier = colon || (prev = last(this.tokens)) && (((_ref3 = prev[0]) === '.' || _ref3 === '?.' || _ref3 === '::') || !prev.spaced && prev[0] === '@');
tag = 'IDENTIFIER';
if (!forcedIdentifier && (__indexOf.call(JS_KEYWORDS, id) >= 0 || __indexOf.call(COFFEE_KEYWORDS, id) >= 0)) {
tag = id.toUpperCase();
if (tag === 'WHEN' && (_ref3 = this.tag(), __indexOf.call(LINE_BREAK, _ref3) >= 0)) {
if (tag === 'WHEN' && (_ref4 = this.tag(), __indexOf.call(LINE_BREAK, _ref4) >= 0)) {
tag = 'LEADING_WHEN';
} else if (tag === 'FOR') {
this.seenFor = true;
@ -172,7 +174,7 @@
return script.length;
};
Lexer.prototype.regexToken = function() {
var length, match, prev, regex, _ref2;
var length, match, prev, regex, _ref3;
if (this.chunk.charAt(0) !== '/') return 0;
if (match = HEREGEX.exec(this.chunk)) {
length = this.heregexToken(match);
@ -180,7 +182,7 @@
return length;
}
prev = last(this.tokens);
if (prev && (_ref2 = prev[0], __indexOf.call((prev.spaced ? NOT_REGEX : NOT_SPACED_REGEX), _ref2) >= 0)) {
if (prev && (_ref3 = prev[0], __indexOf.call((prev.spaced ? NOT_REGEX : NOT_SPACED_REGEX), _ref3) >= 0)) {
return 0;
}
if (!(match = REGEX.exec(this.chunk))) return 0;
@ -189,7 +191,7 @@
return regex.length;
};
Lexer.prototype.heregexToken = function(match) {
var body, flags, heregex, re, tag, tokens, value, _i, _len, _ref2, _ref3, _ref4, _ref5;
var body, flags, heregex, re, tag, tokens, value, _i, _len, _ref3, _ref4, _ref5, _ref6;
heregex = match[0], body = match[1], flags = match[2];
if (0 > body.indexOf('#{')) {
re = body.replace(HEREGEX_OMIT, '').replace(/\//g, '\\/');
@ -199,11 +201,11 @@
this.token('IDENTIFIER', 'RegExp');
this.tokens.push(['CALL_START', '(']);
tokens = [];
_ref2 = this.interpolateString(body, {
_ref3 = this.interpolateString(body, {
regex: true
});
for (_i = 0, _len = _ref2.length; _i < _len; _i++) {
_ref3 = _ref2[_i], tag = _ref3[0], value = _ref3[1];
for (_i = 0, _len = _ref3.length; _i < _len; _i++) {
_ref4 = _ref3[_i], tag = _ref4[0], value = _ref4[1];
if (tag === 'TOKENS') {
tokens.push.apply(tokens, value);
} else {
@ -214,10 +216,10 @@
tokens.push(['+', '+']);
}
tokens.pop();
if (((_ref4 = tokens[0]) != null ? _ref4[0] : void 0) !== 'STRING') {
if (((_ref5 = tokens[0]) != null ? _ref5[0] : void 0) !== 'STRING') {
this.tokens.push(['STRING', '""'], ['+', '+']);
}
(_ref5 = this.tokens).push.apply(_ref5, tokens);
(_ref6 = this.tokens).push.apply(_ref6, tokens);
if (flags) this.tokens.push([',', ','], ['STRING', '"' + flags + '"']);
this.token(')', ')');
return heregex.length;
@ -247,6 +249,7 @@
diff = size - this.indent + this.outdebt;
this.token('INDENT', diff);
this.indents.push(diff);
this.ends.push('OUTDENT');
this.outdebt = this.indebt = 0;
} else {
this.indebt = 0;
@ -255,7 +258,7 @@
this.indent = size;
return indent.length;
};
Lexer.prototype.outdentToken = function(moveOut, noNewlines, close) {
Lexer.prototype.outdentToken = function(moveOut, noNewlines) {
var dent, len;
while (moveOut > 0) {
len = this.indents.length - 1;
@ -271,6 +274,7 @@
dent = this.indents.pop() - this.outdebt;
moveOut -= dent;
this.outdebt = 0;
this.pair('OUTDENT');
this.token('OUTDENT', dent);
}
}
@ -308,7 +312,7 @@
return this;
};
Lexer.prototype.literalToken = function() {
var match, prev, tag, value, _ref2, _ref3, _ref4, _ref5;
var match, prev, tag, value, _ref3, _ref4, _ref5, _ref6;
if (match = OPERATOR.exec(this.chunk)) {
value = match[0];
if (CODE.test(value)) this.tagParameters();
@ -318,10 +322,10 @@
tag = value;
prev = last(this.tokens);
if (value === '=' && prev) {
if (!prev[1].reserved && (_ref2 = prev[1], __indexOf.call(JS_FORBIDDEN, _ref2) >= 0)) {
if (!prev[1].reserved && (_ref3 = prev[1], __indexOf.call(JS_FORBIDDEN, _ref3) >= 0)) {
this.assignmentError();
}
if ((_ref3 = prev[1]) === '||' || _ref3 === '&&') {
if ((_ref4 = prev[1]) === '||' || _ref4 === '&&') {
prev[0] = 'COMPOUND_ASSIGN';
prev[1] += '=';
return value.length;
@ -342,10 +346,10 @@
} else if (__indexOf.call(LOGIC, value) >= 0 || value === '?' && (prev != null ? prev.spaced : void 0)) {
tag = 'LOGIC';
} else if (prev && !prev.spaced) {
if (value === '(' && (_ref4 = prev[0], __indexOf.call(CALLABLE, _ref4) >= 0)) {
if (value === '(' && (_ref5 = prev[0], __indexOf.call(CALLABLE, _ref5) >= 0)) {
if (prev[0] === '?') prev[0] = 'FUNC_EXIST';
tag = 'CALL_START';
} else if (value === '[' && (_ref5 = prev[0], __indexOf.call(INDEXABLE, _ref5) >= 0)) {
} else if (value === '[' && (_ref6 = prev[0], __indexOf.call(INDEXABLE, _ref6) >= 0)) {
tag = 'INDEX_START';
switch (prev[0]) {
case '?':
@ -353,21 +357,32 @@
}
}
}
switch (value) {
case '(':
case '{':
case '[':
this.ends.push(INVERSES[value]);
break;
case ')':
case '}':
case ']':
this.pair(value);
}
this.token(tag, value);
return value.length;
};
Lexer.prototype.sanitizeHeredoc = function(doc, options) {
var attempt, herecomment, indent, match, _ref2;
var attempt, herecomment, indent, match, _ref3;
indent = options.indent, herecomment = options.herecomment;
if (herecomment) {
if (HEREDOC_ILLEGAL.test(doc)) {
throw new Error("block comment cannot contain \"*/\", starting on line " + (this.line + 1));
this.carp("block comment cannot contain \"*/\", starting");
}
if (doc.indexOf('\n') <= 0) return doc;
} else {
while (match = HEREDOC_INDENT.exec(doc)) {
attempt = match[1];
if (indent === null || (0 < (_ref2 = attempt.length) && _ref2 < indent.length)) {
if (indent === null || (0 < (_ref3 = attempt.length) && _ref3 < indent.length)) {
indent = attempt;
}
}
@ -412,9 +427,9 @@
throw SyntaxError("Reserved word \"" + (this.value()) + "\" on line " + (this.line + 1) + " can't be assigned");
};
Lexer.prototype.balancedString = function(str, end) {
var i, letter, match, prev, stack, _ref2;
var i, letter, match, prev, stack, _ref3;
stack = [end];
for (i = 1, _ref2 = str.length; 1 <= _ref2 ? i < _ref2 : i > _ref2; 1 <= _ref2 ? i++ : i--) {
for (i = 1, _ref3 = str.length; 1 <= _ref3 ? i < _ref3 : i > _ref3; 1 <= _ref3 ? i++ : i--) {
switch (letter = str.charAt(i)) {
case '\\':
i++;
@ -436,10 +451,10 @@
}
prev = letter;
}
throw new Error("missing " + (stack.pop()) + ", starting on line " + (this.line + 1));
return this.carp("missing " + (stack.pop()) + ", starting");
};
Lexer.prototype.interpolateString = function(str, options) {
var expr, heredoc, i, inner, interpolated, len, letter, nested, pi, regex, tag, tokens, value, _len, _ref2, _ref3, _ref4;
var expr, heredoc, i, inner, interpolated, len, letter, nested, pi, regex, tag, tokens, value, _len, _ref3, _ref4, _ref5;
if (options == null) options = {};
heredoc = options.heredoc, regex = options.regex;
tokens = [];
@ -461,7 +476,7 @@
rewrite: false
});
nested.pop();
if (((_ref2 = nested[0]) != null ? _ref2[0] : void 0) === 'TERMINATOR') {
if (((_ref3 = nested[0]) != null ? _ref3[0] : void 0) === 'TERMINATOR') {
nested.shift();
}
if (len = nested.length) {
@ -481,10 +496,10 @@
if (tokens[0][0] !== 'NEOSTRING') tokens.unshift(['', '']);
if (interpolated = tokens.length > 1) this.token('(', '(');
for (i = 0, _len = tokens.length; i < _len; i++) {
_ref3 = tokens[i], tag = _ref3[0], value = _ref3[1];
_ref4 = tokens[i], tag = _ref4[0], value = _ref4[1];
if (i) this.token('+', '+');
if (tag === 'TOKENS') {
(_ref4 = this.tokens).push.apply(_ref4, value);
(_ref5 = this.tokens).push.apply(_ref5, value);
} else {
this.token('STRING', this.makeString(value, '"', heredoc));
}
@ -492,6 +507,16 @@
if (interpolated) this.token(')', ')');
return tokens;
};
Lexer.prototype.pair = function(tag) {
var size, wanted;
if (tag !== (wanted = last(this.ends))) {
if ('OUTDENT' !== wanted) this.carp("unmatched " + tag);
this.indent -= size = last(this.indents);
this.outdentToken(size, true);
return this.pair(tag);
}
return this.ends.pop();
};
Lexer.prototype.token = function(tag, value) {
return this.tokens.push([tag, value, this.line]);
};
@ -504,8 +529,8 @@
return (tok = last(this.tokens, index)) && (val ? tok[1] = val : tok[1]);
};
Lexer.prototype.unfinished = function() {
var _ref2;
return LINE_CONTINUER.test(this.chunk) || ((_ref2 = this.tag()) === '\\' || _ref2 === '.' || _ref2 === '?.' || _ref2 === 'UNARY' || _ref2 === 'MATH' || _ref2 === '+' || _ref2 === '-' || _ref2 === 'SHIFT' || _ref2 === 'RELATION' || _ref2 === 'COMPARE' || _ref2 === 'LOGIC' || _ref2 === 'COMPOUND_ASSIGN' || _ref2 === 'THROW' || _ref2 === 'EXTENDS');
var _ref3;
return LINE_CONTINUER.test(this.chunk) || ((_ref3 = this.tag()) === '\\' || _ref3 === '.' || _ref3 === '?.' || _ref3 === 'UNARY' || _ref3 === 'MATH' || _ref3 === '+' || _ref3 === '-' || _ref3 === 'SHIFT' || _ref3 === 'RELATION' || _ref3 === 'COMPARE' || _ref3 === 'LOGIC' || _ref3 === 'COMPOUND_ASSIGN' || _ref3 === 'THROW' || _ref3 === 'EXTENDS');
};
Lexer.prototype.escapeLines = function(str, heredoc) {
return str.replace(MULTILINER, heredoc ? '\\n' : '');
@ -522,6 +547,9 @@
body = body.replace(RegExp("" + quote, "g"), '\\$&');
return quote + this.escapeLines(body, heredoc) + quote;
};
Lexer.prototype.carp = function(message) {
throw SyntaxError("" + message + " on line " + (this.line + 1));
};
return Lexer;
})();
JS_KEYWORDS = ['true', 'false', 'null', 'this', 'new', 'delete', 'typeof', 'in', 'instanceof', 'return', 'throw', 'break', 'continue', 'debugger', 'if', 'else', 'switch', 'for', 'while', 'do', 'try', 'catch', 'finally', 'class', 'extends', 'super'];