waypoint -- starting to implement nodes.coffee with interpolations, and fixing/shortening/combining the lexer implementation to allow identifier interpolations to be interleaved with expression interps

This commit is contained in:
Jeremy Ashkenas 2010-03-05 22:52:28 -05:00
parent 08341286a3
commit b4ea43cbd0
4 changed files with 87 additions and 114 deletions

View File

@ -1,5 +1,5 @@
(function(){
var ACCESSORS, ASSIGNMENT, BEFORE_WHEN, CALLABLE, CODE, COFFEE_KEYWORDS, COMMENT, COMMENT_CLEANER, HEREDOC, HEREDOC_INDENT, IDENTIFIER, INTERPOLATED_EXPRESSION, INTERPOLATED_IDENTIFIER, JS, JS_CLEANER, JS_FORBIDDEN, JS_KEYWORDS, KEYWORDS, LAST_DENT, LAST_DENTS, Lexer, MULTILINER, MULTI_DENT, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX, RESERVED, Rewriter, STRING, STRING_NEWLINES, WHITESPACE, compact, count, include;
var ACCESSORS, ASSIGNMENT, BEFORE_WHEN, CALLABLE, CODE, COFFEE_KEYWORDS, COMMENT, COMMENT_CLEANER, HEREDOC, HEREDOC_INDENT, IDENTIFIER, INTERPOLATION, JS, JS_CLEANER, JS_FORBIDDEN, JS_KEYWORDS, KEYWORDS, LAST_DENT, LAST_DENTS, Lexer, MULTILINER, MULTI_DENT, NOT_REGEX, NO_NEWLINE, NUMBER, OPERATOR, REGEX, RESERVED, Rewriter, STRING, STRING_NEWLINES, WHITESPACE, compact, count, include;
// The CoffeeScript Lexer. Uses a series of token-matching regexes to attempt
// matches against the beginning of the source code. When a match is found,
// a token is produced, we consume the match, and start again. Tokens are in the
@ -35,6 +35,7 @@
NUMBER = /^(\b((0(x|X)[0-9a-fA-F]+)|([0-9]+(\.[0-9]+)?(e[+\-]?[0-9]+)?)))\b/i;
STRING = /^(""|''|"([\s\S]*?)([^\\]|\\\\)"|'([\s\S]*?)([^\\]|\\\\)')/;
HEREDOC = /^("{6}|'{6}|"{3}\n?([\s\S]*?)\n?([ \t]*)"{3}|'{3}\n?([\s\S]*?)\n?([ \t]*)'{3})/;
INTERPOLATION = /(^|[\s\S]*?(?:[\\]|\\\\)?)\$([a-zA-Z_@]\w*|{[\s\S]*?(?:[^\\]|\\\\)})/;
JS = /^(``|`([\s\S]*?)([^\\]|\\\\)`)/;
OPERATOR = /^([+\*&|\/\-%=<>:!?]+)/;
WHITESPACE = /^([ \t]+)/;
@ -45,9 +46,6 @@
LAST_DENTS = /\n([ \t]*)/g;
LAST_DENT = /\n([ \t]*)/;
ASSIGNMENT = /^(:|=)$/;
// Interpolation matching regexes.
INTERPOLATED_EXPRESSION = /(^|[\s\S]*?(?:[\\]|\\\\)?)(\${[\s\S]*?(?:[^\\]|\\\\)})/;
INTERPOLATED_IDENTIFIER = /(^|[\s\S]*?(?:[\\]|\\\\)?)(\$([a-zA-Z_@]\w*))/;
// Token cleaning regexes.
JS_CLEANER = /(^`|`$)/g;
MULTILINER = /\n/g;
@ -408,7 +406,7 @@
// "Hello $name."
// "Hello ${name.capitalize()}."
Lexer.prototype.interpolate_string = function interpolate_string(str) {
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, before, contents, each, expression, expression_match, group, i, id, identifier, identifier_match, lexer, nested, prev, quote, tok, tokens;
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, before, contents, each, group, i, interp, lexer, match, nested, prev, quote, tok, tokens;
if (str.length < 3 || str.substring(0, 1) !== '"') {
return this.token('STRING', str);
} else {
@ -417,75 +415,59 @@
quote = str.substring(0, 1);
str = str.substring(1, str.length - 1);
while (str.length) {
expression_match = str.match(INTERPOLATED_EXPRESSION);
if (expression_match) {
_a = expression_match;
match = str.match(INTERPOLATION);
if (match) {
_a = match;
group = _a[0];
before = _a[1];
expression = _a[2];
interp = _a[2];
if (before.substring(before.length - 1) === '\\') {
if (before.length) {
tokens.push(['STRING', quote + before.substring(0, before.length - 1) + expression + quote]);
tokens.push(['STRING', quote + before.substring(0, before.length - 1) + '$' + interp + quote]);
}
} else {
if (before.length) {
tokens.push(['STRING', quote + before + quote]);
}
nested = lexer.tokenize('(' + expression.substring(2, expression.length - 1) + ')', {
rewrite: false
});
nested.pop();
tokens.push(['TOKENS', nested]);
if (interp.substring(0, 1) === '{') {
nested = lexer.tokenize('(' + interp.substring(1, interp.length - 1) + ')', {
rewrite: false
});
nested.pop();
tokens.push(['TOKENS', nested]);
} else {
if (interp.substring(0, 1) === '@') {
interp = 'this.' + interp.substring(1);
}
tokens.push(['IDENTIFIER', interp]);
}
}
str = str.substring(group.length);
} else {
identifier_match = str.match(INTERPOLATED_IDENTIFIER);
if (identifier_match) {
_b = identifier_match;
group = _b[0];
before = _b[1];
identifier = _b[2];
if (before.substring(before.length - 1) === '\\') {
if (before.length) {
tokens.push(['STRING', quote + before.substring(0, before.length - 1) + identifier + quote]);
}
} else {
if (before.length) {
tokens.push(['STRING', quote + before + quote]);
}
id = identifier.substring(1);
if (id.substring(0, 1) === '@') {
id = 'this.' + id.substring(1);
}
tokens.push(['IDENTIFIER', id]);
}
str = str.substring(group.length);
} else {
tokens.push(['STRING', quote + str + quote]);
str = '';
}
tokens.push(['STRING', quote + str + quote]);
str = '';
}
}
if (tokens.length > 1) {
_e = tokens.length - 1; _f = 1;
for (_d = 0, i = _e; (_e <= _f ? i <= _f : i >= _f); (_e <= _f ? i += 1 : i -= 1), _d++) {
_g = [tokens[i - 1], tokens[i]];
prev = _g[0];
tok = _g[1];
_d = tokens.length - 1; _e = 1;
for (_c = 0, i = _d; (_d <= _e ? i <= _e : i >= _e); (_d <= _e ? i += 1 : i -= 1), _c++) {
_f = [tokens[i - 1], tokens[i]];
prev = _f[0];
tok = _f[1];
if (tok[0] === 'STRING' && prev[0] === 'STRING') {
contents = quote + prev[1].substring(1, prev[1].length - 1) + tok[1].substring(1, tok[1].length - 1) + quote;
tokens.splice(i - 1, 2, ['STRING', contents]);
}
}
}
_h = []; _i = tokens;
for (i = 0, _j = _i.length; i < _j; i++) {
each = _i[i];
_h.push((function() {
_g = []; _h = tokens;
for (i = 0, _i = _h.length; i < _i; i++) {
each = _h[i];
_g.push((function() {
if (each[0] === 'TOKENS') {
_k = each[1];
for (_l = 0, _m = _k.length; _l < _m; _l++) {
nested = _k[_l];
_j = each[1];
for (_k = 0, _l = _j.length; _k < _l; _k++) {
nested = _j[_k];
this.token(nested[0], nested[1]);
}
} else {
@ -496,7 +478,7 @@
}
}).call(this));
}
return _h;
return _g;
}
};
// Helpers

View File

@ -257,13 +257,13 @@
return node instanceof ValueNode && node.is_arguments();
});
if (args) {
code = this.idt() + "arguments = Array.prototype.slice.call(arguments, 0);\n" + code;
code = (this.idt()) + "arguments = Array.prototype.slice.call(arguments, 0);\n" + code;
}
if (o.scope.has_assignments(this)) {
code = this.idt() + 'var ' + o.scope.compiled_assignments() + ";\n" + code;
code = (this.idt()) + "var " + (o.scope.compiled_assignments()) + ";\n" + code;
}
if (o.scope.has_declarations(this)) {
code = this.idt() + 'var ' + o.scope.compiled_declarations() + ";\n" + code;
code = (this.idt()) + "var " + (o.scope.compiled_declarations()) + ";\n" + code;
}
return code;
};
@ -287,7 +287,7 @@
}));
}
// Otherwise, we can just return the value of the expression.
return this.idt() + 'return ' + node.compile(o) + ';';
return (this.idt()) + "return " + (node.compile(o)) + ";";
};
return Expressions;
}).call(this);
@ -321,7 +321,7 @@
return idt + this.value + end;
};
LiteralNode.prototype.toString = function toString(idt) {
return ' "' + this.value + '"';
return " \"" + this.value + "\"";
};
return LiteralNode;
}).call(this);
@ -339,7 +339,7 @@
returns: true
}));
}
return this.idt() + 'return ' + this.expression.compile(o) + ';';
return (this.idt()) + "return " + (this.expression.compile(o)) + ";";
};
return ReturnNode;
}).call(this);
@ -391,7 +391,7 @@
props = only ? this.properties.slice(0, this.properties.length - 1) : this.properties;
baseline = this.base.compile(o);
if (this.base instanceof ObjectNode && this.has_properties()) {
baseline = '(' + baseline + ')';
baseline = "(" + baseline + ")";
}
complete = (this.last = baseline);
_a = props;
@ -402,7 +402,7 @@
soaked = true;
if (this.base instanceof CallNode && prop === props[0]) {
temp = o.scope.free_variable();
complete = '(' + temp + ' = ' + complete + ')' + this.SOAK + ((baseline = temp + prop.compile(o)));
complete = "(" + temp + " = " + complete + ")" + this.SOAK + ((baseline = temp + prop.compile(o)));
} else {
complete = complete + this.SOAK + (baseline += prop.compile(o));
}
@ -413,7 +413,7 @@
this.last = part;
}
}
return op && soaked ? '(' + complete + ')' : complete;
return op && soaked ? "(" + complete + ")" : complete;
};
return ValueNode;
}).call(this);
@ -428,7 +428,7 @@
__extends(CommentNode, BaseNode);
CommentNode.prototype.type = 'Comment';
CommentNode.prototype.compile_node = function compile_node(o) {
return this.idt() + '//' + this.lines.join('\n' + this.idt() + '//');
return (this.idt()) + "//" + this.lines.join("\n" + (this.idt()) + "//");
};
return CommentNode;
}).call(this);
@ -469,6 +469,7 @@
if (this.variable === 'super') {
return this.compile_super(args, o);
}
// "$@prefix${@variable.compile(o)}($args)"
return this.prefix + this.variable.compile(o) + '(' + args + ')';
};
// Compile a call against the superclass's implementation of the current function.

View File

@ -56,24 +56,21 @@ RESERVED: [
JS_FORBIDDEN: JS_KEYWORDS.concat RESERVED
# Token matching regexes.
IDENTIFIER : /^([a-zA-Z$_](\w|\$)*)/
NUMBER : /^(\b((0(x|X)[0-9a-fA-F]+)|([0-9]+(\.[0-9]+)?(e[+\-]?[0-9]+)?)))\b/i
STRING : /^(""|''|"([\s\S]*?)([^\\]|\\\\)"|'([\s\S]*?)([^\\]|\\\\)')/
HEREDOC : /^("{6}|'{6}|"{3}\n?([\s\S]*?)\n?([ \t]*)"{3}|'{3}\n?([\s\S]*?)\n?([ \t]*)'{3})/
JS : /^(``|`([\s\S]*?)([^\\]|\\\\)`)/
OPERATOR : /^([+\*&|\/\-%=<>:!?]+)/
WHITESPACE : /^([ \t]+)/
COMMENT : /^(((\n?[ \t]*)?#[^\n]*)+)/
CODE : /^((-|=)>)/
REGEX : /^(\/(\S.*?)?([^\\]|\\\\)\/[imgy]{0,4})/
MULTI_DENT : /^((\n([ \t]*))+)(\.)?/
LAST_DENTS : /\n([ \t]*)/g
LAST_DENT : /\n([ \t]*)/
ASSIGNMENT : /^(:|=)$/
# Interpolation matching regexes.
INTERPOLATED_EXPRESSION: /(^|[\s\S]*?(?:[\\]|\\\\)?)(\${[\s\S]*?(?:[^\\]|\\\\)})/
INTERPOLATED_IDENTIFIER: /(^|[\s\S]*?(?:[\\]|\\\\)?)(\$([a-zA-Z_@]\w*))/
IDENTIFIER : /^([a-zA-Z$_](\w|\$)*)/
NUMBER : /^(\b((0(x|X)[0-9a-fA-F]+)|([0-9]+(\.[0-9]+)?(e[+\-]?[0-9]+)?)))\b/i
STRING : /^(""|''|"([\s\S]*?)([^\\]|\\\\)"|'([\s\S]*?)([^\\]|\\\\)')/
HEREDOC : /^("{6}|'{6}|"{3}\n?([\s\S]*?)\n?([ \t]*)"{3}|'{3}\n?([\s\S]*?)\n?([ \t]*)'{3})/
INTERPOLATION : /(^|[\s\S]*?(?:[\\]|\\\\)?)\$([a-zA-Z_@]\w*|{[\s\S]*?(?:[^\\]|\\\\)})/
JS : /^(``|`([\s\S]*?)([^\\]|\\\\)`)/
OPERATOR : /^([+\*&|\/\-%=<>:!?]+)/
WHITESPACE : /^([ \t]+)/
COMMENT : /^(((\n?[ \t]*)?#[^\n]*)+)/
CODE : /^((-|=)>)/
REGEX : /^(\/(\S.*?)?([^\\]|\\\\)\/[imgy]{0,4})/
MULTI_DENT : /^((\n([ \t]*))+)(\.)?/
LAST_DENTS : /\n([ \t]*)/g
LAST_DENT : /\n([ \t]*)/
ASSIGNMENT : /^(:|=)$/
# Token cleaning regexes.
JS_CLEANER : /(^`|`$)/g
@ -360,32 +357,24 @@ exports.Lexer: class Lexer
quote: str.substring(0, 1)
str: str.substring(1, str.length - 1)
while str.length
expression_match: str.match INTERPOLATED_EXPRESSION
if expression_match
[group, before, expression]: expression_match
match: str.match INTERPOLATION
if match
[group, before, interp]: match
if before.substring(before.length - 1) is '\\'
tokens.push ['STRING', quote + before.substring(0, before.length - 1) + expression + quote] if before.length
tokens.push ['STRING', quote + before.substring(0, before.length - 1) + '$' + interp + quote] if before.length
else
tokens.push ['STRING', quote + before + quote] if before.length
nested: lexer.tokenize '(' + expression.substring(2, expression.length - 1) + ')', {rewrite: no}
nested.pop()
tokens.push ['TOKENS', nested]
if interp.substring(0, 1) is '{'
nested: lexer.tokenize '(' + interp.substring(1, interp.length - 1) + ')', {rewrite: no}
nested.pop()
tokens.push ['TOKENS', nested]
else
interp: 'this.' + interp.substring(1) if interp.substring(0, 1) is '@'
tokens.push ['IDENTIFIER', interp]
str: str.substring(group.length)
else
identifier_match: str.match INTERPOLATED_IDENTIFIER
if identifier_match
[group, before, identifier]: identifier_match
if before.substring(before.length - 1) is '\\'
tokens.push ['STRING', quote + before.substring(0, before.length - 1) + identifier + quote] if before.length
else
tokens.push ['STRING', quote + before + quote] if before.length
id: identifier.substring(1)
id: 'this.' + id.substring(1) if id.substring(0, 1) is '@'
tokens.push ['IDENTIFIER', id]
str: str.substring(group.length)
else
tokens.push ['STRING', quote + str + quote]
str: ''
tokens.push ['STRING', quote + str + quote]
str: ''
if tokens.length > 1
for i in [tokens.length - 1..1]
[prev, tok]: [tokens[i - 1], tokens[i]]

View File

@ -161,16 +161,16 @@ exports.Expressions: class Expressions extends BaseNode
o.scope: new Scope(null, this, null)
code: if o.globals then @compile_node(o) else @compile_with_declarations(o)
code: code.replace(TRAILING_WHITESPACE, '')
if o.no_wrap then code else "(function(){\n"+code+"\n})();\n"
if o.no_wrap then code else "(function(){\n$code\n})();\n"
# Compile the expressions body, with declarations of all inner variables
# pushed up to the top.
compile_with_declarations: (o) ->
code: @compile_node(o)
args: @contains (node) -> node instanceof ValueNode and node.is_arguments()
code: @idt() + "arguments = Array.prototype.slice.call(arguments, 0);\n" + code if args
code: @idt() + 'var ' + o.scope.compiled_assignments() + ";\n" + code if o.scope.has_assignments(this)
code: @idt() + 'var ' + o.scope.compiled_declarations() + ";\n" + code if o.scope.has_declarations(this)
code: "${@idt()}arguments = Array.prototype.slice.call(arguments, 0);\n$code" if args
code: "${@idt()}var ${o.scope.compiled_assignments()};\n$code" if o.scope.has_assignments(this)
code: "${@idt()}var ${o.scope.compiled_declarations()};\n$code" if o.scope.has_declarations(this)
code
# Compiles a single expression within the expressions body.
@ -184,7 +184,7 @@ exports.Expressions: class Expressions extends BaseNode
# If it's a statement, the node knows how to return itself.
return node.compile(merge(o, {returns: true})) if node.is_statement()
# Otherwise, we can just return the value of the expression.
return @idt() + 'return ' + node.compile(o) + ';'
return "${@idt()}return ${node.compile(o)};"
# Wrap up a node as an Expressions, unless it already is one.
Expressions.wrap: (nodes) ->
@ -212,10 +212,10 @@ exports.LiteralNode: class LiteralNode extends BaseNode
compile_node: (o) ->
idt: if @is_statement() then @idt() else ''
end: if @is_statement() then ';' else ''
idt + @value + end
"$idt$@value$end"
toString: (idt) ->
' "' + @value + '"'
" \"$@value\""
# Return an expression, or wrap it in a closure and return it.
@ -227,7 +227,7 @@ exports.ReturnNode: class ReturnNode extends BaseNode
compile_node: (o) ->
return @expression.compile(merge(o, {returns: true})) if @expression.is_statement()
@idt() + 'return ' + @expression.compile(o) + ';'
"${@idt()}return ${@expression.compile(o)};"
statement ReturnNode, true
@ -277,7 +277,7 @@ exports.ValueNode: class ValueNode extends BaseNode
op: del(o, 'operation')
props: if only then @properties[0...@properties.length - 1] else @properties
baseline: @base.compile o
baseline: '(' + baseline + ')' if @base instanceof ObjectNode and @has_properties()
baseline: "($baseline)" if @base instanceof ObjectNode and @has_properties()
complete: @last: baseline
for prop in props
@ -286,7 +286,7 @@ exports.ValueNode: class ValueNode extends BaseNode
soaked: true
if @base instanceof CallNode and prop is props[0]
temp: o.scope.free_variable()
complete: '(' + temp + ' = ' + complete + ')' + @SOAK + (baseline: temp + prop.compile(o))
complete: "($temp = $complete)$@SOAK" + (baseline: temp + prop.compile(o))
else
complete: complete + @SOAK + (baseline += prop.compile(o))
else
@ -295,7 +295,7 @@ exports.ValueNode: class ValueNode extends BaseNode
complete += part
@last: part
if op and soaked then '(' + complete + ')' else complete
if op and soaked then "($complete)" else complete
# Pass through CoffeeScript comments into JavaScript comments at the
@ -308,7 +308,7 @@ exports.CommentNode: class CommentNode extends BaseNode
this
compile_node: (o) ->
@idt() + '//' + @lines.join('\n' + @idt() + '//')
"${@idt()}//" + @lines.join("\n${@idt()}//")
statement CommentNode
@ -336,6 +336,7 @@ exports.CallNode: class CallNode extends BaseNode
return @compile_splat(o) if @args[@args.length - 1] instanceof SplatNode
args: (arg.compile(o) for arg in @args).join(', ')
return @compile_super(args, o) if @variable is 'super'
# "$@prefix${@variable.compile(o)}($args)"
@prefix + @variable.compile(o) + '(' + args + ')'
# Compile a call against the superclass's implementation of the current function.