waypoint ... somethings still a little off with the parser

lib/coffee_script/grammar.js

@@ -1,5 +1,5 @@
 (function(){
-  var Parser, __a, __b, __c, __d, __e, __f, bnf, grammar, name, non_terminal, o, operators, option, parser, part, posix, tokens, unwrap;
+  var Parser, __a, __b, __c, __d, __e, __f, bnf, grammar, js, name, non_terminal, o, operators, option, parser, part, posix, tokens, unwrap;
   var __hasProp = Object.prototype.hasOwnProperty;
   Parser = require('jison').Parser;
   // DSL ===================================================================
@@ -24,14 +24,20 @@
         return new Expressions();
       }), o("TERMINATOR", function() {
         return new Expressions();
-      }), o("Expressions"), o("Block TERMINATOR")
+      }), o("Expressions", function() {
+        return $1;
+      }), o("Block TERMINATOR", function() {
+        return $1;
+      })
     ],
     // Any list of expressions or method body, seperated by line breaks or semis.
     Expressions: [o("Expression", function() {
         return Expressions.wrap([$1]);
       }), o("Expressions TERMINATOR Expression", function() {
         return $1.push($3);
-      }), o("Expressions TERMINATOR")
+      }), o("Expressions TERMINATOR", function() {
+        return $1;
+      })
     ],
     // All types of expressions in our language. The basic unit of CoffeeScript
     // is the expression.
@@ -558,10 +564,11 @@
   }, {
     debug: false
   });
+  js = parser.generate();
   // Save the parser to a file.
   // puts parser.generate()
   posix = require('posix');
   posix.open('lib/coffee_script/parser.js', process.O_CREAT | process.O_WRONLY, 0755).addCallback(function(fd) {
-    return posix.write(fd, parser.generate());
+    return posix.write(fd, js);
   });
 })();

lib/coffee_script/lexer.js

@@ -228,7 +228,9 @@
       this.token('OUTDENT', last_indent);
       move_out -= last_indent;
     }
-    this.token('TERMINATOR', "\n");
+    if (!(this.tag() === 'TERMINATOR')) {
+      this.token('TERMINATOR', "\n");
+    }
     return true;
   };
   // Matches and consumes non-meaningful whitespace.
@@ -244,7 +246,7 @@
   // Multiple newlines get merged together.
   // Use a trailing \ to escape newlines.
   lex.prototype.newline_token = function newline_token(newlines) {
-    if (!(this.value() === "\n")) {
+    if (!(this.tag() === 'TERMINATOR')) {
       this.token('TERMINATOR', "\n");
     }
     return true;

lib/coffee_script/rewriter.js

@@ -203,7 +203,7 @@
           idx = token[0] === 'OUTDENT' ? i + 1 : i;
           __k = 0; __l = stack[stack.length - 1];
           for (__j=0, tmp=__k; (__k <= __l ? tmp < __l : tmp > __l); (__k <= __l ? tmp += 1 : tmp -= 1), __j++) {
-            this.tokens.splice(idx, 0, ['CALL_END', ')']);
+            this.tokens.splice(idx, 0, ['CALL_END', ')', token[2]]);
           }
           size = stack[stack.length - 1] + 1;
           stack[stack.length - 1] = 0;
@@ -212,7 +212,7 @@
         if (!(prev && IMPLICIT_FUNC.indexOf(prev[0]) >= 0 && IMPLICIT_CALL.indexOf(token[0]) >= 0)) {
           return 1;
         }
-        this.tokens.splice(i, 0, ['CALL_START', '(']);
+        this.tokens.splice(i, 0, ['CALL_START', '(', token[2]]);
         stack[stack.length - 1] += 1;
         return 2;
       };
@@ -233,7 +233,7 @@
           return 1;
         }
         starter = token[0];
-        this.tokens.splice(i + 1, 0, ['INDENT', 2]);
+        this.tokens.splice(i + 1, 0, ['INDENT', 2, token[2]]);
         idx = i + 1;
         parens = 0;
         while (true) {
@@ -241,7 +241,7 @@
           tok = this.tokens[idx];
           if ((!tok || SINGLE_CLOSERS.indexOf(tok[0]) >= 0 || (tok[0] === ')' && parens === 0)) && !(starter === 'ELSE' && tok[0] === 'ELSE')) {
             insertion = this.tokens[idx - 1][0] === "," ? idx - 1 : idx;
-            this.tokens.splice(insertion, 0, ['OUTDENT', 2]);
+            this.tokens.splice(insertion, 0, ['OUTDENT', 2, token[2]]);
             break;
           }
           if (tok[0] === '(') {

src/grammar.coffee

@@ -45,15 +45,15 @@ grammar: {
   Root: [
     o "",                                  -> new Expressions()
     o "TERMINATOR",                        -> new Expressions()
-    o "Expressions"
+    o "Expressions",                       -> $1
-    o "Block TERMINATOR"
+    o "Block TERMINATOR",                  -> $1
   ]
 
   # Any list of expressions or method body, seperated by line breaks or semis.
   Expressions: [
     o "Expression",                        -> Expressions.wrap([$1])
     o "Expressions TERMINATOR Expression", -> $1.push($3)
-    o "Expressions TERMINATOR"
+    o "Expressions TERMINATOR",            -> $1
   ]
 
   # All types of expressions in our language. The basic unit of CoffeeScript
@@ -461,9 +461,10 @@ for name, non_terminal of grammar
     option
 tokens: tokens.join(" ")
 parser: new Parser({tokens: tokens, bnf: bnf, operators: operators.reverse(), startSymbol: 'Root'}, {debug: false})
+js: parser.generate()
 
 # Save the parser to a file.
 # puts parser.generate()
 posix: require 'posix'
 posix.open('lib/coffee_script/parser.js', process.O_CREAT | process.O_WRONLY, 0755).addCallback (fd) ->
-  posix.write(fd, parser.generate())
+  posix.write(fd, js)

src/lexer.coffee

@@ -188,7 +188,7 @@ lex::outdent_token: (move_out) ->
     last_indent: @indents.pop()
     @token 'OUTDENT', last_indent
     move_out -= last_indent
-  @token 'TERMINATOR', "\n"
+  @token 'TERMINATOR', "\n" unless @tag() is 'TERMINATOR'
   true
 
 # Matches and consumes non-meaningful whitespace.
@@ -201,7 +201,7 @@ lex::whitespace_token: ->
 # Multiple newlines get merged together.
 # Use a trailing \ to escape newlines.
 lex::newline_token: (newlines) ->
-  @token 'TERMINATOR', "\n" unless @value() is "\n"
+  @token 'TERMINATOR', "\n" unless @tag() is 'TERMINATOR'
   true
 
 # Tokens to explicitly escape newlines are removed once their job is done.

src/rewriter.coffee

@@ -145,12 +145,12 @@ re::add_implicit_parentheses: ->
     if stack[stack.length - 1] > 0 and (IMPLICIT_END.indexOf(token[0]) >= 0 or !post?)
       idx: if token[0] is 'OUTDENT' then i + 1 else i
       for tmp in [0...stack[stack.length - 1]]
-        this.tokens.splice(idx, 0, ['CALL_END', ')'])
+        this.tokens.splice(idx, 0, ['CALL_END', ')', token[2]])
       size: stack[stack.length - 1] + 1
       stack[stack.length - 1]: 0
       return size
     return 1 unless prev and IMPLICIT_FUNC.indexOf(prev[0]) >= 0 and IMPLICIT_CALL.indexOf(token[0]) >= 0
-    this.tokens.splice(i, 0, ['CALL_START', '('])
+    this.tokens.splice(i, 0, ['CALL_START', '(', token[2]])
     stack[stack.length - 1] += 1
     return 2
 
@@ -163,7 +163,7 @@ re::add_implicit_indentation: ->
     return 1 unless SINGLE_LINERS.indexOf(token[0]) >= 0 and post[0] isnt 'INDENT' and
       not (token[0] is 'ELSE' and post[0] is 'IF')
     starter: token[0]
-    this.tokens.splice(i + 1, 0, ['INDENT', 2])
+    this.tokens.splice(i + 1, 0, ['INDENT', 2, token[2]])
     idx: i + 1
     parens: 0
     while true
@@ -173,7 +173,7 @@ re::add_implicit_indentation: ->
           (tok[0] is ')' && parens is 0)) and
           not (starter is 'ELSE' and tok[0] is 'ELSE')
         insertion: if this.tokens[idx - 1][0] is "," then idx - 1 else idx
-        this.tokens.splice(insertion, 0, ['OUTDENT', 2])
+        this.tokens.splice(insertion, 0, ['OUTDENT', 2, token[2]])
         break
       parens += 1 if tok[0] is '('
       parens -= 1 if tok[0] is ')'