first draft of CoffeeScript language extensions.

lib/coffee-script.js

@@ -1,5 +1,5 @@
 (function(){
-  var lexer, parser, path, process_scripts;
+  var Lexer, lexer, parser, path, process_scripts;
   // CoffeeScript can be used both on the server, as a command-line compiler based
   // on Node.js/V8, or to run CoffeeScripts directly in the browser. This module
   // contains the main entry functions for tokenzing, parsing, and compiling source
@@ -10,18 +10,20 @@
   if ((typeof process !== "undefined" && process !== null)) {
     process.mixin(require('nodes'));
     path = require('path');
-    lexer = new (require('lexer').Lexer)();
+    Lexer = require('lexer').Lexer;
     parser = require('parser').parser;
   } else {
-    lexer = new Lexer();
     parser = exports.parser;
     this.exports = (this.CoffeeScript = {});
   }
   // The current CoffeeScript version number.
   exports.VERSION = '0.5.5';
+  // Instantiate a Lexer for our use here.
+  lexer = new Lexer();
   // Compile a string of CoffeeScript code to JavaScript, using the Coffee/Jison
   // compiler.
   exports.compile = function compile(code, options) {
+    options = options || {};
     try {
       return (parser.parse(lexer.tokenize(code))).compile(options);
     } catch (err) {
@@ -49,6 +51,13 @@
     __dirname = path.dirname(__filename);
     return eval(exports.compile(code, options));
   };
+  // Extend CoffeeScript with a custom language extension. It should hook in to
+  // the **Lexer** (as a peer of any of the lexer's tokenizing methods), and
+  // push a token on to the stack that contains a **Node** as the value (as a
+  // peer of the nodes in [nodes.coffee](nodes.html)).
+  exports.extend = function extend(func) {
+    return Lexer.extensions.push(func);
+  };
   // The real Lexer produces a generic stream of tokens. This object provides a
   // thin wrapper around it, compatible with the Jison API. We can then pass it
   // directly as a "Jison lexer".

lib/grammar.js

@@ -67,7 +67,7 @@
     // CoffeeScript is the **Expression** -- you'll notice that there is no
     // "statement" nonterminal. Expressions serve as the building blocks
     // of many other rules, making them somewhat circular.
-    Expression: [o("Value"), o("Call"), o("Code"), o("Operation"), o("Assign"), o("If"), o("Try"), o("Throw"), o("Return"), o("While"), o("For"), o("Switch"), o("Extends"), o("Class"), o("Splat"), o("Existence"), o("Comment")],
+    Expression: [o("Value"), o("Call"), o("Code"), o("Operation"), o("Assign"), o("If"), o("Try"), o("Throw"), o("Return"), o("While"), o("For"), o("Switch"), o("Extends"), o("Class"), o("Splat"), o("Existence"), o("Comment"), o("Extension")],
     // A an indented block of expressions. Note that the [Rewriter](rewriter.html)
     // will convert some postfix forms into blocks for us, by adjusting the
     // token stream.
@@ -370,6 +370,12 @@
         return new ParentheticalNode($2);
       })
     ],
+    // A language extension to CoffeeScript from the outside. We simply pass
+    // it through unaltered.
+    Extension: [o("EXTENSION", function() {
+        return $1;
+      })
+    ],
     // The condition portion of a while loop.
     WhileSource: [o("WHILE Expression", function() {
         return new WhileNode($2);

lib/lexer.js

@@ -60,6 +60,9 @@
     // short-circuiting if any of them succeed. Their order determines precedence:
     // `@literal_token` is the fallback catch-all.
     Lexer.prototype.extract_next_token = function extract_next_token() {
+      if (this.extension_token()) {
+        return null;
+      }
       if (this.identifier_token()) {
         return null;
       }
@@ -91,6 +94,19 @@
     };
     // Tokenizers
     // ----------
+    // Language extensions get the highest priority, first chance to tag tokens
+    // as something else.
+    Lexer.prototype.extension_token = function extension_token() {
+      var _a, _b, _c, extension;
+      _a = Lexer.extensions;
+      for (_b = 0, _c = _a.length; _b < _c; _b++) {
+        extension = _a[_b];
+        if (extension.call(this)) {
+          return true;
+        }
+      }
+      return false;
+    };
     // Matches identifying literals: variables, keywords, method names, etc.
     // Check to ensure that JavaScript reserved words aren't being used as
     // identifiers. Because CoffeeScript reserves a handful of keywords that are
@@ -528,6 +544,8 @@
     };
     return Lexer;
   }).call(this);
+  // There are no exensions to the core lexer by default.
+  Lexer.extensions = [];
   // Constants
   // ---------
   // Keywords that CoffeeScript shares in common with JavaScript.

lib/nodes.js

@@ -772,32 +772,6 @@ idt += TAB
     return ClassNode;
   }).call(this);
   statement(ClassNode);
-  //### PushNode
-  // A faux-node that is never created by the grammar, but is used during
-  // code generation to generate a quick `array.push(value)` tree of nodes.
-  // Helpful for recording the result arrays from comprehensions.
-  PushNode = (exports.PushNode = {
-    wrap: function wrap(array, expressions) {
-      var expr;
-      expr = expressions.unwrap();
-      if (expr.is_pure_statement() || expr.contains(function(n) {
-        return n.is_pure_statement();
-      })) {
-        return expressions;
-      }
-      return Expressions.wrap([new CallNode(new ValueNode(literal(array), [new AccessorNode(literal('push'))]), [expr])]);
-    }
-  });
-  //### ClosureNode
-  // A faux-node used to wrap an expressions body in a closure.
-  ClosureNode = (exports.ClosureNode = {
-    wrap: function wrap(expressions, statement) {
-      var call, func;
-      func = new ParentheticalNode(new CodeNode([], Expressions.wrap([expressions])));
-      call = new CallNode(new ValueNode(func, [new AccessorNode(literal('call'))]), [literal('this')]);
-      return statement ? Expressions.wrap([call]) : call;
-    }
-  });
   //### AssignNode
   // The **AssignNode** is used to assign a local variable to value, or to set the
   // property of an object -- including within object literals.
@@ -1556,6 +1530,35 @@ idt += TAB
     };
     return IfNode;
   }).call(this);
+  // Faux-Nodes
+  // ----------
+  //### PushNode
+  // Faux-nodes are never created by the grammar, but are used during code
+  // generation to generate other combinations of nodes. The **PushNode** creates
+  // the tree for `array.push(value)`, which is helpful for recording the result
+  // arrays from comprehensions.
+  PushNode = (exports.PushNode = {
+    wrap: function wrap(array, expressions) {
+      var expr;
+      expr = expressions.unwrap();
+      if (expr.is_pure_statement() || expr.contains(function(n) {
+        return n.is_pure_statement();
+      })) {
+        return expressions;
+      }
+      return Expressions.wrap([new CallNode(new ValueNode(literal(array), [new AccessorNode(literal('push'))]), [expr])]);
+    }
+  });
+  //### ClosureNode
+  // A faux-node used to wrap an expressions body in a closure.
+  ClosureNode = (exports.ClosureNode = {
+    wrap: function wrap(expressions, statement) {
+      var call, func;
+      func = new ParentheticalNode(new CodeNode([], Expressions.wrap([expressions])));
+      call = new CallNode(new ValueNode(func, [new AccessorNode(literal('call'))]), [literal('this')]);
+      return statement ? Expressions.wrap([call]) : call;
+    }
+  });
   // Constants
   // ---------
   // Tabs are two spaces for pretty printing.

lib/rewriter.js

@@ -373,7 +373,7 @@
   // Tokens that, if followed by an `IMPLICIT_CALL`, indicate a function invocation.
   IMPLICIT_FUNC = ['IDENTIFIER', 'SUPER', ')', 'CALL_END', ']', 'INDEX_END'];
   // If preceded by an `IMPLICIT_FUNC`, indicates a function invocation.
-  IMPLICIT_CALL = ['IDENTIFIER', 'NUMBER', 'STRING', 'JS', 'REGEX', 'NEW', 'PARAM_START', 'TRY', 'DELETE', 'TYPEOF', 'SWITCH', 'TRUE', 'FALSE', 'YES', 'NO', 'ON', 'OFF', '!', '!!', 'NOT', '@', '->', '=>', '[', '(', '{'];
+  IMPLICIT_CALL = ['IDENTIFIER', 'NUMBER', 'STRING', 'JS', 'REGEX', 'NEW', 'PARAM_START', 'TRY', 'DELETE', 'TYPEOF', 'SWITCH', 'EXTENSION', 'TRUE', 'FALSE', 'YES', 'NO', 'ON', 'OFF', '!', '!!', 'NOT', '@', '->', '=>', '[', '(', '{'];
   // Tokens indicating that the implicit call must enclose a block of expressions.
   IMPLICIT_BLOCK = ['->', '=>', '{', '[', ','];
   // Tokens that always mark the end of an implicit call for single-liners.

src/coffee-script.coffee

@@ -10,19 +10,22 @@
 if process?
   process.mixin require 'nodes'
   path:         require 'path'
-  lexer:   new (require('lexer').Lexer)()
+  Lexer:        require('lexer').Lexer
   parser:       require('parser').parser
 else
-  lexer: new Lexer()
   parser: exports.parser
   this.exports: this.CoffeeScript: {}
 
 # The current CoffeeScript version number.
 exports.VERSION: '0.5.5'
 
+# Instantiate a Lexer for our use here.
+lexer: new Lexer()
+
 # Compile a string of CoffeeScript code to JavaScript, using the Coffee/Jison
 # compiler.
 exports.compile: (code, options) ->
+  options ||= {}
   try
     (parser.parse lexer.tokenize code).compile options
   catch err
@@ -46,6 +49,13 @@ exports.run: (code, options) ->
   __dirname: path.dirname __filename
   eval exports.compile code, options
 
+# Extend CoffeeScript with a custom language extension. It should hook in to
+# the **Lexer** (as a peer of any of the lexer's tokenizing methods), and
+# push a token on to the stack that contains a **Node** as the value (as a
+# peer of the nodes in [nodes.coffee](nodes.html)).
+exports.extend: (func) ->
+  Lexer.extensions.push func
+
 # The real Lexer produces a generic stream of tokens. This object provides a
 # thin wrapper around it, compatible with the Jison API. We can then pass it
 # directly as a "Jison lexer".

src/grammar.coffee

@@ -89,6 +89,7 @@ grammar: {
     o "Splat"
     o "Existence"
     o "Comment"
+    o "Extension"
   ]
 
   # A an indented block of expressions. Note that the [Rewriter](rewriter.html)
@@ -353,6 +354,12 @@ grammar: {
     o "( Expression )",                         -> new ParentheticalNode $2
   ]
 
+  # A language extension to CoffeeScript from the outside. We simply pass
+  # it through unaltered.
+  Extension: [
+    o "EXTENSION",                              -> $1
+  ]
+
   # The condition portion of a while loop.
   WhileSource: [
     o "WHILE Expression",                       -> new WhileNode $2

src/lexer.coffee

@@ -54,6 +54,7 @@ exports.Lexer: class Lexer
   # short-circuiting if any of them succeed. Their order determines precedence:
   # `@literal_token` is the fallback catch-all.
   extract_next_token: ->
+    return if @extension_token()
     return if @identifier_token()
     return if @number_token()
     return if @heredoc_token()
@@ -68,6 +69,13 @@ exports.Lexer: class Lexer
   # Tokenizers
   # ----------
 
+  # Language extensions get the highest priority, first chance to tag tokens
+  # as something else.
+  extension_token: ->
+    for extension in Lexer.extensions
+      return true if extension.call this
+    false
+
   # Matches identifying literals: variables, keywords, method names, etc.
   # Check to ensure that JavaScript reserved words aren't being used as
   # identifiers. Because CoffeeScript reserves a handful of keywords that are
@@ -387,6 +395,9 @@ exports.Lexer: class Lexer
     return false unless m: @chunk.match(regex)
     if m then m[index] else false
 
+# There are no exensions to the core lexer by default.
+Lexer.extensions: []
+
 # Constants
 # ---------
 

src/nodes.coffee

@@ -594,34 +594,6 @@ exports.ClassNode: class ClassNode extends BaseNode
 
 statement ClassNode
 
-#### PushNode
-
-# A faux-node that is never created by the grammar, but is used during
-# code generation to generate a quick `array.push(value)` tree of nodes.
-# Helpful for recording the result arrays from comprehensions.
-PushNode: exports.PushNode: {
-
-  wrap: (array, expressions) ->
-    expr: expressions.unwrap()
-    return expressions if expr.is_pure_statement() or expr.contains (n) -> n.is_pure_statement()
-    Expressions.wrap([new CallNode(
-      new ValueNode(literal(array), [new AccessorNode(literal('push'))]), [expr]
-    )])
-
-}
-
-#### ClosureNode
-
-# A faux-node used to wrap an expressions body in a closure.
-ClosureNode: exports.ClosureNode: {
-
-  wrap: (expressions, statement) ->
-    func: new ParentheticalNode(new CodeNode([], Expressions.wrap([expressions])))
-    call: new CallNode(new ValueNode(func, [new AccessorNode(literal('call'))]), [literal('this')])
-    if statement then Expressions.wrap([call]) else call
-
-}
-
 #### AssignNode
 
 # The **AssignNode** is used to assign a local variable to value, or to set the
@@ -1182,6 +1154,38 @@ exports.IfNode: class IfNode extends BaseNode
     else_part:  if @else_body then @else_body.compile(o) else 'null'
     "$if_part : $else_part"
 
+# Faux-Nodes
+# ----------
+
+#### PushNode
+
+# Faux-nodes are never created by the grammar, but are used during code
+# generation to generate other combinations of nodes. The **PushNode** creates
+# the tree for `array.push(value)`, which is helpful for recording the result
+# arrays from comprehensions.
+PushNode: exports.PushNode: {
+
+  wrap: (array, expressions) ->
+    expr: expressions.unwrap()
+    return expressions if expr.is_pure_statement() or expr.contains (n) -> n.is_pure_statement()
+    Expressions.wrap([new CallNode(
+      new ValueNode(literal(array), [new AccessorNode(literal('push'))]), [expr]
+    )])
+
+}
+
+#### ClosureNode
+
+# A faux-node used to wrap an expressions body in a closure.
+ClosureNode: exports.ClosureNode: {
+
+  wrap: (expressions, statement) ->
+    func: new ParentheticalNode(new CodeNode([], Expressions.wrap([expressions])))
+    call: new CallNode(new ValueNode(func, [new AccessorNode(literal('call'))]), [literal('this')])
+    if statement then Expressions.wrap([call]) else call
+
+}
+
 # Constants
 # ---------
 

src/rewriter.coffee

@@ -251,7 +251,7 @@ IMPLICIT_FUNC:  ['IDENTIFIER', 'SUPER', ')', 'CALL_END', ']', 'INDEX_END']
 
 # If preceded by an `IMPLICIT_FUNC`, indicates a function invocation.
 IMPLICIT_CALL:  ['IDENTIFIER', 'NUMBER', 'STRING', 'JS', 'REGEX', 'NEW', 'PARAM_START',
-                 'TRY', 'DELETE', 'TYPEOF', 'SWITCH',
+                 'TRY', 'DELETE', 'TYPEOF', 'SWITCH', 'EXTENSION',
                  'TRUE', 'FALSE', 'YES', 'NO', 'ON', 'OFF', '!', '!!', 'NOT',
                  '@', '->', '=>', '[', '(', '{']
 

test/test_compilation.coffee

@@ -1,3 +1,29 @@
+# Ensure that carriage returns don't break compilation on Windows.
+
 js: CoffeeScript.compile("one\r\ntwo", {no_wrap: on})
 
-ok js is "one;\ntwo;"
+ok js is "one;\ntwo;"
+
+
+# Try out language extensions to CoffeeScript.
+#
+# class SplitNode extends BaseNode
+#   type: 'Split'
+#
+#   constructor: (variable) ->
+#     @variable: variable
+#
+#   compile_node: (o) ->
+#     "${variable}.split('')"
+#
+# CoffeeScript.extend ->
+#   return false unless variable: @match /^--(\w+)--/, 1
+#   @i += variable.length + 4
+#   @token 'EXTENSION', new SplitNode(variable)
+#   true
+#
+# js: CoffeeScript.nodes('print --name--', {no_wrap: on})
+#
+# p js.children[0].children
+#
+# Lexer.extensions: []