Fix tabbed Literate CoffeeScript (#4345)

* failing test case

* add markdown parser for literate coffeescript

this should help to handle ligitimate markdown with indentation correctly

* Update generated code

* Update package.json

* Add `marked` dependency to browser version of CoffeeScript

* Update invertLiterate to use a randomly-generated token that we check for uniqueness, rather than a magic number that we hope might not occur in the code

* Fix typos

Cakefile

@@ -140,6 +140,14 @@ task 'build:parser', 'rebuild the Jison parser (run build first)', ->
 
 task 'build:browser', 'rebuild the merged script for inclusion in the browser', ->
   code = ''
+  for {name, src} in [{name: 'marked', src: 'lib/marked.js'}]
+    code += """
+      require['#{name}'] = (function() {
+        var exports = {}, module = {exports: exports};
+        #{fs.readFileSync "node_modules/#{name}/#{src}"}
+        return module.exports;
+      })();
+    """
   for name in ['helpers', 'rewriter', 'lexer', 'parser', 'scope', 'nodes', 'sourcemap', 'coffee-script', 'browser']
     code += """
       require['./#{name}'] = (function() {

lib/coffee-script/helpers.js

@@ -1,6 +1,8 @@
 // Generated by CoffeeScript 2.0.0-alpha
 (function() {
-  var buildLocationData, extend, flatten, ref, repeat, syntaxErrorToString;
+  var buildLocationData, extend, flatten, marked, ref, repeat, syntaxErrorToString;
+
+  marked = require('marked');
 
   exports.starts = function(string, literal, start) {
     return literal === string.substr(start, literal.length);
@@ -96,25 +98,24 @@
   };
 
   exports.invertLiterate = function(code) {
-    var line, lines, maybe_code;
-    maybe_code = true;
-    lines = (function() {
-      var i, len1, ref1, results;
-      ref1 = code.split('\n');
-      results = [];
-      for (i = 0, len1 = ref1.length; i < len1; i++) {
-        line = ref1[i];
-        if (maybe_code && /^([ ]{4}|[ ]{0,3}\t)/.test(line)) {
-          results.push(line);
-        } else if (maybe_code = /^\s*$/.test(line)) {
-          results.push(line);
-        } else {
-          results.push('# ' + line);
-        }
+    var generateRandomToken, i, item, len1, out, ref1, token;
+    generateRandomToken = function() {
+      return "" + (Math.random() * Date.now());
+    };
+    while (token === void 0 || code.indexOf(token) !== -1) {
+      token = generateRandomToken();
+    }
+    code = code.replace("\t", token);
+    out = "";
+    ref1 = marked.lexer(code, {});
+    for (i = 0, len1 = ref1.length; i < len1; i++) {
+      item = ref1[i];
+      if (item.type === 'code') {
+        out += item.text + "\n";
       }
-      return results;
-    })();
-    return lines.join('\n');
+    }
+    out.replace(token, "\t");
+    return out;
   };
 
   buildLocationData = function(first, last) {

package.json

@@ -44,5 +44,8 @@
     "highlight.js": "~9.7.0",
     "underscore": "~1.8.3",
     "docco": "~0.7.0"
+  },
+  "dependencies": {
+    "marked": "~0.3.6"
   }
 }

src/helpers.coffee

@@ -2,6 +2,17 @@
 # the **Lexer**, **Rewriter**, and the **Nodes**. Merge objects, flatten
 # arrays, count characters, that sort of thing.
 
+marked = require 'marked'
+# marked.setOptions
+#   renderer: new marked.Renderer()
+#   gfm: true
+#   tables: true
+#   breaks: false
+#   pedantic: false
+#   sanitize: true
+#   smartLists: true
+#   smartypants: false
+
 # Peek at the beginning of a given string to see if it matches a sequence.
 exports.starts = (string, literal, start) ->
   literal is string.substr start, literal.length
@@ -67,19 +78,25 @@ exports.some = Array::some ? (fn) ->
   return true for e in this when fn e
   false
 
-# Simple function for inverting Literate CoffeeScript code by putting the
-# documentation in comments, producing a string of CoffeeScript code that
-# can be compiled "normally".
+# Simple function for extracting code from Literate CoffeeScript by stripping
+# out all non-code blocks, producing a string of CoffeeScript code that can
+# be compiled “normally.”
 exports.invertLiterate = (code) ->
-  maybe_code = true
-  lines = for line in code.split('\n')
-    if maybe_code and /^([ ]{4}|[ ]{0,3}\t)/.test line
-      line
-    else if maybe_code = /^\s*$/.test line
-      line
-    else
-      '# ' + line
-  lines.join '\n'
+  # Create a placeholder for tabs, that isn’t used anywhere in `code`, and then
+  # re-insert the tabs after code extraction.
+  generateRandomToken = ->
+    "#{Math.random() * Date.now()}"
+  while token is undefined or code.indexOf(token) isnt -1
+    token = generateRandomToken()
+
+  code = code.replace "\t", token
+  # Parse as markdown, discard everything except code blocks.
+  out = ""
+  for item in marked.lexer code, {}
+    out += "#{item.text}\n" if item.type is 'code'
+  # Put the tabs back in.
+  out.replace token, "\t"
+  out
 
 # Merge two jison-style location data objects together.
 # If `last` is not provided, this will simply return `first`.

test/literate.litcoffee

@@ -55,3 +55,111 @@ Tabs work too:
 
 				test "tabbed code", ->
 					ok yes
+
+---
+
+    # keep track of whether code blocks are executed or not
+    executed = false
+
+<p>
+
+    executed = true # should not execute, this is just HTML para, not code!
+
+</p>
+
+    test "should ignore indented sections inside HTML", ->
+      eq executed, false
+
+---
+
+*   A list item with a code block:
+
+        test "basic literate CoffeeScript parsing", ->
+          ok yes
+
+---
+
+*   Lorem ipsum dolor sit amet, consectetuer adipiscing elit.
+    Aliquam hendrerit mi posuere lectus. Vestibulum enim wisi,
+    viverra nec, fringilla in, laoreet vitae, risus.
+
+*   Donec sit amet nisl. Aliquam semper ipsum sit amet velit.
+    Suspendisse id sem consectetuer libero luctus adipiscing.
+
+---
+
+1.  This is a list item with two paragraphs. Lorem ipsum dolor
+    sit amet, consectetuer adipiscing elit. Aliquam hendrerit
+    mi posuere lectus.
+
+    Vestibulum enim wisi, viverra nec, fringilla in, laoreet
+    vitae, risus. Donec sit amet nisl. Aliquam semper ipsum
+    sit amet velit.
+
+2.  Suspendisse id sem consectetuer libero luctus adipiscing.
+
+---
+
+1.  This is a list item with two paragraphs. Lorem ipsum dolor
+    sit amet, consectetuer adipiscing elit. Aliquam hendrerit
+    mi posuere lectus.
+
+    Vestibulum enim wisi, viverra nec, fringilla in, laoreet
+    vitae, risus. Donec sit amet nisl. Aliquam semper ipsum
+    sit amet velit.
+
+2.  Suspendisse id sem consectetuer libero luctus adipiscing.
+
+---
+
+*   A list item with a blockquote:
+
+    > This is a blockquote
+    > inside a list item.
+
+---
+
+This next one probably passes because a string is inoffensive in compiled js, also, can't get `marked` to parse it correctly, and not sure if empty line is permitted between title and reference
+
+This is [an example][id] reference-style link.
+[id]: http://example.com/
+
+    "Optional Title Here"
+
+---
+
+    executed = no
+
+1986. What a great season.
+          executed = yes
+
+and test...
+
+    test "should recognise indented code blocks in lists", ->
+      ok executed
+
+---
+
+    executed = no
+
+1986. What a great season.
+
+          executed = yes
+
+and test...
+
+    test "should recognise indented code blocks in lists with empty line as separator", ->
+      ok executed
+
+---
+
+    executed = no
+
+1986\. What a great season.
+           executed = yes
+
+and test...
+
+    test "should ignore indented code in escaped list like number", ->
+      eq executed, no
+