CS1 tagged template literals (and CS2 interpolated strings as template literals) (#4352)

* Add initial support for template literals with no
interpolation

* Change ‘unexpected string’ error message tests to
use number not identifier prefix.

Identifer prefixes are now valid as tagged
template literals

* Test tagged template literals for non-interpolated
strings and tag function.

* Tagged template literals work for pure Strings.

Pull tagged template definition up to Invocation
level in grammar, enabling chained invocation calls.

We can view a tagged template is a special form
of function call.

* Readying for StringWithInterpolations work.

* Tweaks.

* Fix style

* Pass StringWithInterpolations parameter straight
into Call constructor.

StringWithInterpolations will be output as
template literal, so already in correct form for
outputting tagged template literal.

* Strip down compileNode for StringWithInterpolations

* Done StringLiteral case for interpolated Strings

* Remove need for TemplateLiteral

* Simplify code.

* Small code tidy

* Interpolated strings now outputting as template literals.

Still needs comprehensive testing.

* Move error message tests into error_messages.coffee; remove test that is testing for a Node runtime error

* Split up tests that were testing multiple things per test, so that each test tests only one thing

* Edge cases: tagged template literals containing interpolated strings or even internal tagged template literals

* Make more concise, more idiomatic style

* Pull back extreme indentation

* Restore and fix commented-out tests

* Edge case: tagged template literal with empty string

* Only use new ES2015 interpolated string syntax if we’re inside a tagged template literal; this keeps this PR safe to merge into CoffeeScript 1.x. Remove the code from this commit to make all interpolated strings use ES2015 syntax, for CoffeeScript 2.

* Compiler now _doesn’t_ use template literals.

* Expand tagged template literal tests

* Move ‘Unexpected string’ error message tests into
tagged template literal section.

‘Unexpected string’ is not reported in these test
scenarios anymore. Instead, we error that the
prefixing literal is not a function.

* Don’t unwrap StringWithInterpolations.

Saw bug with program consisting of “#{2}” not
compiling with template literals. Root cause was
that Block.compileNode was unwrapping interpolated
string and so didn’t use compileNode logic at
StringWithInterpolations level.

* No need to bracket interpolated strings any more.

When interpolated string looks like `hello ${2}`,
no extract brackets are needed, as the `s mark the
beginning and end.

* Show html templating with tagged template literals

* Multiline should match multiline

* Comment out unnecessary `unwrap`, which is only needed for CoffeeScript 2 all-ES2015 syntax output

lib/coffee-script/grammar.js

@@ -392,7 +392,9 @@
       })
     ],
     Invocation: [
-      o('Value OptFuncExist Arguments', function() {
+      o('Value OptFuncExist String', function() {
+        return new TaggedTemplateCall($1, $3, $2);
+      }), o('Value OptFuncExist Arguments', function() {
         return new Call($1, $3, $2);
       }), o('Invocation OptFuncExist Arguments', function() {
         return new Call($1, $3, $2);

lib/coffee-script/nodes.js

@@ -1,6 +1,6 @@
 // Generated by CoffeeScript 1.11.1
 (function() {
-  var Access, Arr, Assign, Base, Block, BooleanLiteral, Call, Class, Code, CodeFragment, Comment, Existence, Expansion, ExportAllDeclaration, ExportDeclaration, ExportDefaultDeclaration, ExportNamedDeclaration, ExportSpecifier, ExportSpecifierList, Extends, For, IdentifierLiteral, If, ImportClause, ImportDeclaration, ImportDefaultSpecifier, ImportNamespaceSpecifier, ImportSpecifier, ImportSpecifierList, In, Index, InfinityLiteral, JS_FORBIDDEN, LEVEL_ACCESS, LEVEL_COND, LEVEL_LIST, LEVEL_OP, LEVEL_PAREN, LEVEL_TOP, Literal, ModuleDeclaration, ModuleSpecifier, ModuleSpecifierList, NEGATE, NO, NaNLiteral, NullLiteral, NumberLiteral, Obj, Op, Param, Parens, PassthroughLiteral, PropertyName, Range, RegexLiteral, RegexWithInterpolations, Return, SIMPLENUM, Scope, Slice, Splat, StatementLiteral, StringLiteral, StringWithInterpolations, SuperCall, Switch, TAB, THIS, ThisLiteral, Throw, Try, UTILITIES, UndefinedLiteral, Value, While, YES, YieldReturn, addLocationDataFn, compact, del, ends, extend, flatten, fragmentsToText, isComplexOrAssignable, isLiteralArguments, isLiteralThis, isUnassignable, locationDataToString, merge, multident, ref1, ref2, some, starts, throwSyntaxError, unfoldSoak, utility,
+  var Access, Arr, Assign, Base, Block, BooleanLiteral, Call, Class, Code, CodeFragment, Comment, Existence, Expansion, ExportAllDeclaration, ExportDeclaration, ExportDefaultDeclaration, ExportNamedDeclaration, ExportSpecifier, ExportSpecifierList, Extends, For, IdentifierLiteral, If, ImportClause, ImportDeclaration, ImportDefaultSpecifier, ImportNamespaceSpecifier, ImportSpecifier, ImportSpecifierList, In, Index, InfinityLiteral, JS_FORBIDDEN, LEVEL_ACCESS, LEVEL_COND, LEVEL_LIST, LEVEL_OP, LEVEL_PAREN, LEVEL_TOP, Literal, ModuleDeclaration, ModuleSpecifier, ModuleSpecifierList, NEGATE, NO, NaNLiteral, NullLiteral, NumberLiteral, Obj, Op, Param, Parens, PassthroughLiteral, PropertyName, Range, RegexLiteral, RegexWithInterpolations, Return, SIMPLENUM, Scope, Slice, Splat, StatementLiteral, StringLiteral, StringWithInterpolations, SuperCall, Switch, TAB, THIS, TaggedTemplateCall, ThisLiteral, Throw, Try, UTILITIES, UndefinedLiteral, Value, While, YES, YieldReturn, addLocationDataFn, compact, del, ends, extend, flatten, fragmentsToText, isComplexOrAssignable, isLiteralArguments, isLiteralThis, isUnassignable, locationDataToString, merge, multident, ref1, ref2, some, starts, throwSyntaxError, unfoldSoak, utility,
     extend1 = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; },
     hasProp = {}.hasOwnProperty,
     indexOf = [].indexOf || function(item) { for (var i = 0, l = this.length; i < l; i++) { if (i in this && this[i] === item) return i; } return -1; },
@@ -1012,10 +1012,10 @@
   exports.Call = Call = (function(superClass1) {
     extend1(Call, superClass1);
 
-    function Call(variable1, args1, soak) {
+    function Call(variable1, args1, soak1) {
       this.variable = variable1;
       this.args = args1 != null ? args1 : [];
-      this.soak = soak;
+      this.soak = soak1;
       this.isNew = false;
       if (this.variable instanceof Value && this.variable.isNotCallable()) {
         this.variable.error("literal is not a function");
@@ -1218,6 +1218,25 @@
 
   })(Call);
 
+  exports.TaggedTemplateCall = TaggedTemplateCall = (function(superClass1) {
+    extend1(TaggedTemplateCall, superClass1);
+
+    function TaggedTemplateCall(variable, arg, soak) {
+      if (arg instanceof StringLiteral) {
+        arg = new StringWithInterpolations(Block.wrap([new Value(arg)]));
+      }
+      TaggedTemplateCall.__super__.constructor.call(this, variable, [arg], soak);
+    }
+
+    TaggedTemplateCall.prototype.compileNode = function(o) {
+      o.inTaggedTemplateCall = true;
+      return this.variable.compileToFragments(o, LEVEL_ACCESS).concat(this.args[0].compileToFragments(o, LEVEL_LIST));
+    };
+
+    return TaggedTemplateCall;
+
+  })(Call);
+
   exports.Extends = Extends = (function(superClass1) {
     extend1(Extends, superClass1);
 
@@ -3331,6 +3350,39 @@
       return StringWithInterpolations.__super__.constructor.apply(this, arguments);
     }
 
+    StringWithInterpolations.prototype.compileNode = function(o) {
+      var element, elements, expr, fragments, j, len1;
+      if (!o.inTaggedTemplateCall) {
+        return StringWithInterpolations.__super__.compileNode.apply(this, arguments);
+      }
+      expr = this.body.unwrap();
+      elements = [];
+      expr.traverseChildren(false, function(node) {
+        if (node instanceof StringLiteral) {
+          elements.push(node);
+          return true;
+        } else if (node instanceof Parens) {
+          elements.push(node);
+          return false;
+        }
+        return true;
+      });
+      fragments = [];
+      fragments.push(this.makeCode('`'));
+      for (j = 0, len1 = elements.length; j < len1; j++) {
+        element = elements[j];
+        if (element instanceof StringLiteral) {
+          fragments.push(this.makeCode(element.value.slice(1, -1)));
+        } else {
+          fragments.push(this.makeCode('${'));
+          fragments.push.apply(fragments, element.compileToFragments(o, LEVEL_PAREN));
+          fragments.push(this.makeCode('}'));
+        }
+      }
+      fragments.push(this.makeCode('`'));
+      return fragments;
+    };
+
     return StringWithInterpolations;
 
   })(Parens);

src/grammar.coffee

@@ -413,6 +413,7 @@ grammar =
 
   # Ordinary function invocation, or a chained series of calls.
   Invocation: [
+    o 'Value OptFuncExist String',              -> new TaggedTemplateCall $1, $3, $2
     o 'Value OptFuncExist Arguments',           -> new Call $1, $3, $2
     o 'Invocation OptFuncExist Arguments',      -> new Call $1, $3, $2
     o 'Super'

src/nodes.coffee

@@ -787,6 +787,17 @@ exports.RegexWithInterpolations = class RegexWithInterpolations extends Call
   constructor: (args = []) ->
     super (new Value new IdentifierLiteral 'RegExp'), args, false
 
+#### TaggedTemplateCall
+
+exports.TaggedTemplateCall = class TaggedTemplateCall extends Call
+  constructor: (variable, arg, soak) ->
+    arg = new StringWithInterpolations Block.wrap([ new Value arg ]) if arg instanceof StringLiteral
+    super variable, [ arg ], soak
+
+  compileNode: (o) ->
+    o.inTaggedTemplateCall = yes # Tell StringWithInterpolations whether to compile as ES2015 or not; remove in CoffeeScript 2
+    @variable.compileToFragments(o, LEVEL_ACCESS).concat @args[0].compileToFragments(o, LEVEL_LIST)
+
 #### Extends
 
 # Node to extend an object's prototype with an ancestor object.
@@ -2233,6 +2244,44 @@ exports.Parens = class Parens extends Base
 # string concatenation inside.
 
 exports.StringWithInterpolations = class StringWithInterpolations extends Parens
+  # Uncomment the following line in CoffeeScript 2, to allow all interpolated
+  # strings to be output using the ES2015 syntax:
+  # unwrap: -> this
+
+  compileNode: (o) ->
+    # This method produces an interpolated string using the new ES2015 syntax,
+    # which is opt-in by using tagged template literals. If this
+    # StringWithInterpolations isn’t inside a tagged template literal,
+    # fall back to the CoffeeScript 1.x output.
+    # (Remove this check in CoffeeScript 2.)
+    unless o.inTaggedTemplateCall
+      return super
+
+    # Assumption: expr is Value>StringLiteral or Op
+    expr = @body.unwrap()
+
+    elements = []
+    expr.traverseChildren no, (node) ->
+      if node instanceof StringLiteral
+        elements.push node
+        return yes
+      else if node instanceof Parens
+        elements.push node
+        return no
+      return yes
+
+    fragments = []
+    fragments.push @makeCode '`'
+    for element in elements
+      if element instanceof StringLiteral
+        fragments.push @makeCode element.value.slice(1, -1)
+      else
+        fragments.push @makeCode '${'
+        fragments.push element.compileToFragments(o, LEVEL_PAREN)...
+        fragments.push @makeCode '}'
+    fragments.push @makeCode '`'
+
+    fragments
 
 #### For
 

test/error_messages.coffee

@@ -137,46 +137,6 @@ test "#1096: unexpected generated tokens", ->
      ^^^^^^^^^^^
   '''
   # Unexpected string
-  assertErrorFormat "a''", '''
-    [stdin]:1:2: error: unexpected string
-    a''
-     ^^
-  '''
-  assertErrorFormat 'a""', '''
-    [stdin]:1:2: error: unexpected string
-    a""
-     ^^
-  '''
-  assertErrorFormat "a'b'", '''
-    [stdin]:1:2: error: unexpected string
-    a'b'
-     ^^^
-  '''
-  assertErrorFormat 'a"b"', '''
-    [stdin]:1:2: error: unexpected string
-    a"b"
-     ^^^
-  '''
-  assertErrorFormat "a'''b'''", """
-    [stdin]:1:2: error: unexpected string
-    a'''b'''
-     ^^^^^^^
-  """
-  assertErrorFormat 'a"""b"""', '''
-    [stdin]:1:2: error: unexpected string
-    a"""b"""
-     ^^^^^^^
-  '''
-  assertErrorFormat 'a"#{b}"', '''
-    [stdin]:1:2: error: unexpected string
-    a"#{b}"
-     ^^^^^^
-  '''
-  assertErrorFormat 'a"""#{b}"""', '''
-    [stdin]:1:2: error: unexpected string
-    a"""#{b}"""
-     ^^^^^^^^^^
-  '''
   assertErrorFormat 'import foo from "lib-#{version}"', '''
     [stdin]:1:17: error: the name of the module to be imported from must be an uninterpolated string
     import foo from "lib-#{version}"
@@ -1188,3 +1148,45 @@ test "own is not supported in for-from loops", ->
     x for own x from [1, 2, 3]
           ^^^
     '''
+
+test "tagged template literals must be called by an identifier", ->
+  assertErrorFormat "1''", '''
+    [stdin]:1:1: error: literal is not a function
+    1''
+    ^
+  '''
+  assertErrorFormat '1""', '''
+    [stdin]:1:1: error: literal is not a function
+    1""
+    ^
+  '''
+  assertErrorFormat "1'b'", '''
+    [stdin]:1:1: error: literal is not a function
+    1'b'
+    ^
+  '''
+  assertErrorFormat '1"b"', '''
+    [stdin]:1:1: error: literal is not a function
+    1"b"
+    ^
+  '''
+  assertErrorFormat "1'''b'''", """
+    [stdin]:1:1: error: literal is not a function
+    1'''b'''
+    ^
+  """
+  assertErrorFormat '1"""b"""', '''
+    [stdin]:1:1: error: literal is not a function
+    1"""b"""
+    ^
+  '''
+  assertErrorFormat '1"#{b}"', '''
+    [stdin]:1:1: error: literal is not a function
+    1"#{b}"
+    ^
+  '''
+  assertErrorFormat '1"""#{b}"""', '''
+    [stdin]:1:1: error: literal is not a function
+    1"""#{b}"""
+    ^
+  '''

test/tagged_template_literals.coffee

@@ -0,0 +1,139 @@
+# Tagged template literals
+# ------------------------
+
+# NOTES:
+# A tagged template literal is a string that is passed to a prefixing function for
+# post-processing. There's a bunch of different angles that need testing:
+# - Prefixing function, which can be any form of function call:
+#   - function: func'Hello'
+#   - object property with dot notation: outerobj.obj.func'Hello'
+#   - object property with bracket notation: outerobj['obj']['func']'Hello'
+# - String form: single quotes, double quotes and block strings
+# - String is single-line or multi-line
+# - String is interpolated or not
+
+func = (text, expressions...) ->
+  "text: [#{text.join '|'}] expressions: [#{expressions.join '|'}]"
+
+outerobj =
+  obj:
+    func: func
+
+# Example use
+test "tagged template literal for html templating", ->
+  html = (htmlFragments, expressions...) ->
+    htmlFragments.reduce (fullHtml, htmlFragment, i) ->
+      fullHtml + "#{expressions[i - 1]}#{htmlFragment}"
+
+  state =
+    name: 'Greg'
+    adjective: 'awesome'
+
+  eq """
+      <p>
+        Hi Greg. You're looking awesome!
+      </p>
+    """,
+    html"""
+      <p>
+        Hi ${state.name}. You're looking ${state.adjective}!
+      </p>
+    """
+
+# Simple, non-interpolated strings
+test "tagged template literal with a single-line single-quote string", ->
+  eq 'text: [single-line single quotes] expressions: []',
+  func'single-line single quotes'
+
+test "tagged template literal with a single-line double-quote string", ->
+  eq 'text: [single-line double quotes] expressions: []',
+  func"single-line double quotes"
+
+test "tagged template literal with a single-line single-quote block string", ->
+  eq 'text: [single-line block string] expressions: []',
+  func'''single-line block string'''
+
+test "tagged template literal with a single-line double-quote block string", ->
+  eq 'text: [single-line block string] expressions: []',
+  func"""single-line block string"""
+
+test "tagged template literal with a multi-line single-quote string", ->
+  eq 'text: [multi-line single quotes] expressions: []',
+  func'multi-line
+                                                              single quotes'
+
+test "tagged template literal with a multi-line double-quote string", ->
+  eq 'text: [multi-line double quotes] expressions: []',
+  func"multi-line
+       double quotes"
+
+test "tagged template literal with a multi-line single-quote block string", ->
+  eq 'text: [multi-line\nblock string] expressions: []',
+  func'''
+      multi-line
+      block string
+      '''
+
+test "tagged template literal with a multi-line double-quote block string", ->
+  eq 'text: [multi-line\nblock string] expressions: []',
+  func"""
+      multi-line
+      block string
+      """
+
+# Interpolated strings with expressions
+test "tagged template literal with a single-line double-quote interpolated string", ->
+  eq 'text: [single-line | double quotes | interpolation] expressions: [36|42]',
+  func"single-line #{6 * 6} double quotes #{6 * 7} interpolation"
+
+test "tagged template literal with a single-line double-quote block interpolated string", ->
+  eq 'text: [single-line | block string | interpolation] expressions: [incredible|48]',
+  func"""single-line #{'incredible'} block string #{6 * 8} interpolation"""
+
+test "tagged template literal with a multi-line double-quote interpolated string", ->
+  eq 'text: [multi-line | double quotes | interpolation] expressions: [2|awesome]',
+  func"multi-line #{4/2}
+       double quotes #{'awesome'} interpolation"
+
+test "tagged template literal with a multi-line double-quote block interpolated string", ->
+  eq 'text: [multi-line |\nblock string |] expressions: [/abc/|32]',
+  func"""
+      multi-line #{/abc/}
+      block string #{2 * 16}
+      """
+
+
+# Tagged template literal must use a callable function
+test "tagged template literal dot notation recognized as a callable function", ->
+  eq 'text: [dot notation] expressions: []',
+  outerobj.obj.func'dot notation'
+
+test "tagged template literal bracket notation recognized as a callable function", ->
+  eq 'text: [bracket notation] expressions: []',
+  outerobj['obj']['func']'bracket notation'
+
+test "tagged template literal mixed dot and bracket notation recognized as a callable function", ->
+  eq 'text: [mixed notation] expressions: []',
+  outerobj['obj'].func'mixed notation'
+
+
+# Edge cases
+test "tagged template literal with an empty string", ->
+  eq 'text: [] expressions: []',
+  func''
+
+test "tagged template literal with an empty interpolated string", ->
+  eq 'text: [] expressions: []',
+  func"#{}"
+
+test "tagged template literal as single interpolated expression", ->
+  eq 'text: [|] expressions: [3]',
+  func"#{3}"
+
+test "tagged template literal with an interpolated string that itself contains an interpolated string", ->
+  eq 'text: [inner | string] expressions: [interpolated]',
+  func"inner #{"#{'inter'}polated"} string"
+
+test "tagged template literal with an interpolated string that contains a tagged template literal", ->
+  eq 'text: [inner tagged | literal] expressions: [text: [|] expressions: [template]]',
+  func"inner tagged #{func"#{'template'}"} literal"