jashkenas--coffeescript/lib/coffeescript/lexer.js

// Generated by CoffeeScript 2.7.0
(function() {
  // The CoffeeScript Lexer. Uses a series of token-matching regexes to attempt
  // matches against the beginning of the source code. When a match is found,
  // a token is produced, we consume the match, and start again. Tokens are in the
  // form:

  //     [tag, value, locationData]

  // where locationData is {first_line, first_column, last_line, last_column, last_line_exclusive, last_column_exclusive}, which is a
  // format that can be fed directly into [Jison](https://github.com/zaach/jison).  These
  // are read by jison in the `parser.lexer` function defined in coffeescript.coffee.
  var BOM, BOOL, CALLABLE, CODE, COFFEE_ALIASES, COFFEE_ALIAS_MAP, COFFEE_KEYWORDS, COMMENT, COMPARABLE_LEFT_SIDE, COMPARE, COMPOUND_ASSIGN, HERECOMMENT_ILLEGAL, HEREDOC_DOUBLE, HEREDOC_INDENT, HEREDOC_SINGLE, HEREGEX, HEREGEX_COMMENT, HERE_JSTOKEN, IDENTIFIER, INDENTABLE_CLOSERS, INDEXABLE, INSIDE_JSX, INVERSES, JSTOKEN, JSX_ATTRIBUTE, JSX_FRAGMENT_IDENTIFIER, JSX_IDENTIFIER, JSX_IDENTIFIER_PART, JSX_INTERPOLATION, JS_KEYWORDS, LINE_BREAK, LINE_CONTINUER, Lexer, MATH, MULTI_DENT, NOT_REGEX, NUMBER, OPERATOR, POSSIBLY_DIVISION, REGEX, REGEX_FLAGS, REGEX_ILLEGAL, REGEX_INVALID_ESCAPE, RELATION, RESERVED, Rewriter, SHIFT, STRICT_PROSCRIBED, STRING_DOUBLE, STRING_INVALID_ESCAPE, STRING_SINGLE, STRING_START, TRAILING_SPACES, UNARY, UNARY_MATH, UNFINISHED, VALID_FLAGS, WHITESPACE, addTokenData, attachCommentsToNode, compact, count, flatten, invertLiterate, isForFrom, isUnassignable, key, locationDataToString, merge, parseNumber, repeat, replaceUnicodeCodePointEscapes, starts, throwSyntaxError,
    indexOf = [].indexOf,
    slice = [].slice;

  ({Rewriter, INVERSES, UNFINISHED} = require('./rewriter'));

  // Import the helpers we need.
  ({count, starts, compact, repeat, invertLiterate, merge, attachCommentsToNode, locationDataToString, throwSyntaxError, replaceUnicodeCodePointEscapes, flatten, parseNumber} = require('./helpers'));

  // The Lexer Class
  // ---------------

  // The Lexer class reads a stream of CoffeeScript and divvies it up into tagged
  // tokens. Some potential ambiguity in the grammar has been avoided by
  // pushing some extra smarts into the Lexer.
  exports.Lexer = Lexer = class Lexer {
    constructor() {
      // Throws an error at either a given offset from the current chunk or at the
      // location of a token (`token[2]`).
      this.error = this.error.bind(this);
    }

    // **tokenize** is the Lexer's main method. Scan by attempting to match tokens
    // one at a time, using a regular expression anchored at the start of the
    // remaining code, or a custom recursive token-matching method
    // (for interpolations). When the next token has been recorded, we move forward
    // within the code past the token, and begin again.

    // Each tokenizing method is responsible for returning the number of characters
    // it has consumed.

    // Before returning the token stream, run it through the [Rewriter](rewriter.html).
    tokenize(code, opts = {}) {
      var consumed, end, i, ref;
      this.literate = opts.literate; // Are we lexing literate CoffeeScript?
      this.indent = 0; // The current indentation level.
      this.baseIndent = 0; // The overall minimum indentation level.
      this.continuationLineAdditionalIndent = 0; // The over-indentation at the current level.
      this.outdebt = 0; // The under-outdentation at the current level.
      this.indents = []; // The stack of all current indentation levels.
      this.indentLiteral = ''; // The indentation.
      this.ends = []; // The stack for pairing up tokens.
      this.tokens = []; // Stream of parsed tokens in the form `['TYPE', value, location data]`.
      this.seenFor = false; // Used to recognize `FORIN`, `FOROF` and `FORFROM` tokens.
      this.seenImport = false; // Used to recognize `IMPORT FROM? AS?` tokens.
      this.seenExport = false; // Used to recognize `EXPORT FROM? AS?` tokens.
      this.importSpecifierList = false; // Used to identify when in an `IMPORT {...} FROM? ...`.
      this.exportSpecifierList = false; // Used to identify when in an `EXPORT {...} FROM? ...`.
      this.jsxDepth = 0; // Used to optimize JSX checks, how deep in JSX we are.
      this.jsxObjAttribute = {}; // Used to detect if JSX attributes is wrapped in {} (<div {props...} />).
      this.chunkLine = opts.line || 0; // The start line for the current @chunk.
      this.chunkColumn = opts.column || 0; // The start column of the current @chunk.
      this.chunkOffset = opts.offset || 0; // The start offset for the current @chunk.
      this.locationDataCompensations = opts.locationDataCompensations || {};
      code = this.clean(code); // The stripped, cleaned original source code.
      
      // At every position, run through this list of attempted matches,
      // short-circuiting if any of them succeed. Their order determines precedence:
      // `@literalToken` is the fallback catch-all.
      i = 0;
      while (this.chunk = code.slice(i)) {
        consumed = this.identifierToken() || this.commentToken() || this.whitespaceToken() || this.lineToken() || this.stringToken() || this.numberToken() || this.jsxToken() || this.regexToken() || this.jsToken() || this.literalToken();
        // Update position.
        [this.chunkLine, this.chunkColumn, this.chunkOffset] = this.getLineAndColumnFromChunk(consumed);
        i += consumed;
        if (opts.untilBalanced && this.ends.length === 0) {
          return {
            tokens: this.tokens,
            index: i
          };
        }
      }
      this.closeIndentation();
      if (end = this.ends.pop()) {
        this.error(`missing ${end.tag}`, ((ref = end.origin) != null ? ref : end)[2]);
      }
      if (opts.rewrite === false) {
        return this.tokens;
      }
      return (new Rewriter()).rewrite(this.tokens);
    }

    // Preprocess the code to remove leading and trailing whitespace, carriage
    // returns, etc. If we’re lexing literate CoffeeScript, strip external Markdown
    // by removing all lines that aren’t indented by at least four spaces or a tab.
    clean(code) {
      var base, thusFar;
      thusFar = 0;
      if (code.charCodeAt(0) === BOM) {
        code = code.slice(1);
        this.locationDataCompensations[0] = 1;
        thusFar += 1;
      }
      if (WHITESPACE.test(code)) {
        code = `\n${code}`;
        this.chunkLine--;
        if ((base = this.locationDataCompensations)[0] == null) {
          base[0] = 0;
        }
        this.locationDataCompensations[0] -= 1;
      }
      code = code.replace(/\r/g, (match, offset) => {
        this.locationDataCompensations[thusFar + offset] = 1;
        return '';
      }).replace(TRAILING_SPACES, '');
      if (this.literate) {
        code = invertLiterate(code);
      }
      return code;
    }

    // Tokenizers
    // ----------

      // Matches identifying literals: variables, keywords, method names, etc.
    // Check to ensure that JavaScript reserved words aren’t being used as
    // identifiers. Because CoffeeScript reserves a handful of keywords that are
    // allowed in JavaScript, we’re careful not to tag them as keywords when
    // referenced as property names here, so you can still do `jQuery.is()` even
    // though `is` means `===` otherwise.
    identifierToken() {
      var alias, colon, colonOffset, colonToken, id, idLength, inJSXTag, input, match, poppedToken, prev, prevprev, ref, ref1, ref10, ref11, ref12, ref2, ref3, ref4, ref5, ref6, ref7, ref8, ref9, regExSuper, regex, sup, tag, tagToken, tokenData;
      inJSXTag = this.atJSXTag();
      regex = inJSXTag ? JSX_ATTRIBUTE : IDENTIFIER;
      if (!(match = regex.exec(this.chunk))) {
        return 0;
      }
      [input, id, colon] = match;
      // Preserve length of id for location data
      idLength = id.length;
      poppedToken = void 0;
      if (id === 'own' && this.tag() === 'FOR') {
        this.token('OWN', id);
        return id.length;
      }
      if (id === 'from' && this.tag() === 'YIELD') {
        this.token('FROM', id);
        return id.length;
      }
      if (id === 'as' && this.seenImport) {
        if (this.value() === '*') {
          this.tokens[this.tokens.length - 1][0] = 'IMPORT_ALL';
        } else if (ref = this.value(true), indexOf.call(COFFEE_KEYWORDS, ref) >= 0) {
          prev = this.prev();
          [prev[0], prev[1]] = ['IDENTIFIER', this.value(true)];
        }
        if ((ref1 = this.tag()) === 'DEFAULT' || ref1 === 'IMPORT_ALL' || ref1 === 'IDENTIFIER') {
          this.token('AS', id);
          return id.length;
        }
      }
      if (id === 'as' && this.seenExport) {
        if ((ref2 = this.tag()) === 'IDENTIFIER' || ref2 === 'DEFAULT') {
          this.token('AS', id);
          return id.length;
        }
        if (ref3 = this.value(true), indexOf.call(COFFEE_KEYWORDS, ref3) >= 0) {
          prev = this.prev();
          [prev[0], prev[1]] = ['IDENTIFIER', this.value(true)];
          this.token('AS', id);
          return id.length;
        }
      }
      if (id === 'default' && this.seenExport && ((ref4 = this.tag()) === 'EXPORT' || ref4 === 'AS')) {
        this.token('DEFAULT', id);
        return id.length;
      }
      if (id === 'assert' && (this.seenImport || this.seenExport) && this.tag() === 'STRING') {
        this.token('ASSERT', id);
        return id.length;
      }
      if (id === 'do' && (regExSuper = /^(\s*super)(?!\(\))/.exec(this.chunk.slice(3)))) {
        this.token('SUPER', 'super');
        this.token('CALL_START', '(');
        this.token('CALL_END', ')');
        [input, sup] = regExSuper;
        return sup.length + 3;
      }
      prev = this.prev();
      tag = colon || (prev != null) && (((ref5 = prev[0]) === '.' || ref5 === '?.' || ref5 === '::' || ref5 === '?::') || !prev.spaced && prev[0] === '@') ? 'PROPERTY' : 'IDENTIFIER';
      tokenData = {};
      if (tag === 'IDENTIFIER' && (indexOf.call(JS_KEYWORDS, id) >= 0 || indexOf.call(COFFEE_KEYWORDS, id) >= 0) && !(this.exportSpecifierList && indexOf.call(COFFEE_KEYWORDS, id) >= 0)) {
        tag = id.toUpperCase();
        if (tag === 'WHEN' && (ref6 = this.tag(), indexOf.call(LINE_BREAK, ref6) >= 0)) {
          tag = 'LEADING_WHEN';
        } else if (tag === 'FOR') {
          this.seenFor = {
            endsLength: this.ends.length
          };
        } else if (tag === 'UNLESS') {
          tag = 'IF';
        } else if (tag === 'IMPORT') {
          this.seenImport = true;
        } else if (tag === 'EXPORT') {
          this.seenExport = true;
        } else if (indexOf.call(UNARY, tag) >= 0) {
          tag = 'UNARY';
        } else if (indexOf.call(RELATION, tag) >= 0) {
          if (tag !== 'INSTANCEOF' && this.seenFor) {
            tag = 'FOR' + tag;
            this.seenFor = false;
          } else {
            tag = 'RELATION';
            if (this.value() === '!') {
              poppedToken = this.tokens.pop();
              tokenData.invert = (ref7 = (ref8 = poppedToken.data) != null ? ref8.original : void 0) != null ? ref7 : poppedToken[1];
            }
          }
        }
      } else if (tag === 'IDENTIFIER' && this.seenFor && id === 'from' && isForFrom(prev)) {
        tag = 'FORFROM';
        this.seenFor = false;
      // Throw an error on attempts to use `get` or `set` as keywords, or
      // what CoffeeScript would normally interpret as calls to functions named
      // `get` or `set`, i.e. `get({foo: function () {}})`.
      } else if (tag === 'PROPERTY' && prev) {
        if (prev.spaced && (ref9 = prev[0], indexOf.call(CALLABLE, ref9) >= 0) && /^[gs]et$/.test(prev[1]) && this.tokens.length > 1 && ((ref10 = this.tokens[this.tokens.length - 2][0]) !== '.' && ref10 !== '?.' && ref10 !== '@')) {
          this.error(`'${prev[1]}' cannot be used as a keyword, or as a function call without parentheses`, prev[2]);
        } else if (prev[0] === '.' && this.tokens.length > 1 && (prevprev = this.tokens[this.tokens.length - 2])[0] === 'UNARY' && prevprev[1] === 'new') {
          prevprev[0] = 'NEW_TARGET';
        } else if (prev[0] === '.' && this.tokens.length > 1 && (prevprev = this.tokens[this.tokens.length - 2])[0] === 'IMPORT' && prevprev[1] === 'import') {
          this.seenImport = false;
          prevprev[0] = 'IMPORT_META';
        } else if (this.tokens.length > 2) {
          prevprev = this.tokens[this.tokens.length - 2];
          if (((ref11 = prev[0]) === '@' || ref11 === 'THIS') && prevprev && prevprev.spaced && /^[gs]et$/.test(prevprev[1]) && ((ref12 = this.tokens[this.tokens.length - 3][0]) !== '.' && ref12 !== '?.' && ref12 !== '@')) {
            this.error(`'${prevprev[1]}' cannot be used as a keyword, or as a function call without parentheses`, prevprev[2]);
          }
        }
      }
      if (tag === 'IDENTIFIER' && indexOf.call(RESERVED, id) >= 0 && !inJSXTag) {
        this.error(`reserved word '${id}'`, {
          length: id.length
        });
      }
      if (!(tag === 'PROPERTY' || this.exportSpecifierList || this.importSpecifierList)) {
        if (indexOf.call(COFFEE_ALIASES, id) >= 0) {
          alias = id;
          id = COFFEE_ALIAS_MAP[id];
          tokenData.original = alias;
        }
        tag = (function() {
          switch (id) {
            case '!':
              return 'UNARY';
            case '==':
            case '!=':
              return 'COMPARE';
            case 'true':
            case 'false':
              return 'BOOL';
            case 'break':
            case 'continue':
            case 'debugger':
              return 'STATEMENT';
            case '&&':
            case '||':
              return id;
            default:
              return tag;
          }
        })();
      }
      tagToken = this.token(tag, id, {
        length: idLength,
        data: tokenData
      });
      if (alias) {
        tagToken.origin = [tag, alias, tagToken[2]];
      }
      if (poppedToken) {
        [tagToken[2].first_line, tagToken[2].first_column, tagToken[2].range[0]] = [poppedToken[2].first_line, poppedToken[2].first_column, poppedToken[2].range[0]];
      }
      if (colon) {
        colonOffset = input.lastIndexOf(inJSXTag ? '=' : ':');
        colonToken = this.token(':', ':', {
          offset: colonOffset
        });
        if (inJSXTag) { // used by rewriter
          colonToken.jsxColon = true;
        }
      }
      if (inJSXTag && tag === 'IDENTIFIER' && prev[0] !== ':') {
        this.token(',', ',', {
          length: 0,
          origin: tagToken,
          generated: true
        });
      }
      return input.length;
    }

    // Matches numbers, including decimals, hex, and exponential notation.
    // Be careful not to interfere with ranges in progress.
    numberToken() {
      var lexedLength, match, number, parsedValue, tag, tokenData;
      if (!(match = NUMBER.exec(this.chunk))) {
        return 0;
      }
      number = match[0];
      lexedLength = number.length;
      switch (false) {
        case !/^0[BOX]/.test(number):
          this.error(`radix prefix in '${number}' must be lowercase`, {
            offset: 1
          });
          break;
        case !/^0\d*[89]/.test(number):
          this.error(`decimal literal '${number}' must not be prefixed with '0'`, {
            length: lexedLength
          });
          break;
        case !/^0\d+/.test(number):
          this.error(`octal literal '${number}' must be prefixed with '0o'`, {
            length: lexedLength
          });
      }
      parsedValue = parseNumber(number);
      tokenData = {parsedValue};
      tag = parsedValue === 2e308 ? 'INFINITY' : 'NUMBER';
      if (tag === 'INFINITY') {
        tokenData.original = number;
      }
      this.token(tag, number, {
        length: lexedLength,
        data: tokenData
      });
      return lexedLength;
    }

    // Matches strings, including multiline strings, as well as heredocs, with or without
    // interpolation.
    stringToken() {
      var attempt, delimiter, doc, end, heredoc, i, indent, match, prev, quote, ref, regex, token, tokens;
      [quote] = STRING_START.exec(this.chunk) || [];
      if (!quote) {
        return 0;
      }
      // If the preceding token is `from` and this is an import or export statement,
      // properly tag the `from`.
      prev = this.prev();
      if (prev && this.value() === 'from' && (this.seenImport || this.seenExport)) {
        prev[0] = 'FROM';
      }
      regex = (function() {
        switch (quote) {
          case "'":
            return STRING_SINGLE;
          case '"':
            return STRING_DOUBLE;
          case "'''":
            return HEREDOC_SINGLE;
          case '"""':
            return HEREDOC_DOUBLE;
        }
      })();
      ({
        tokens,
        index: end
      } = this.matchWithInterpolations(regex, quote));
      heredoc = quote.length === 3;
      if (heredoc) {
        // Find the smallest indentation. It will be removed from all lines later.
        indent = null;
        doc = ((function() {
          var j, len, results;
          results = [];
          for (i = j = 0, len = tokens.length; j < len; i = ++j) {
            token = tokens[i];
            if (token[0] === 'NEOSTRING') {
              results.push(token[1]);
            }
          }
          return results;
        })()).join('#{}');
        while (match = HEREDOC_INDENT.exec(doc)) {
          attempt = match[1];
          if (indent === null || (0 < (ref = attempt.length) && ref < indent.length)) {
            indent = attempt;
          }
        }
      }
      delimiter = quote.charAt(0);
      this.mergeInterpolationTokens(tokens, {
        quote,
        indent,
        endOffset: end
      }, (value) => {
        return this.validateUnicodeCodePointEscapes(value, {
          delimiter: quote
        });
      });
      if (this.atJSXTag()) {
        this.token(',', ',', {
          length: 0,
          origin: this.prev,
          generated: true
        });
      }
      return end;
    }

    // Matches and consumes comments. The comments are taken out of the token
    // stream and saved for later, to be reinserted into the output after
    // everything has been parsed and the JavaScript code generated.
    commentToken(chunk = this.chunk, {heregex, returnCommentTokens = false, offsetInChunk = 0} = {}) {
      var commentAttachment, commentAttachments, commentWithSurroundingWhitespace, content, contents, getIndentSize, hasSeenFirstCommentLine, hereComment, hereLeadingWhitespace, hereTrailingWhitespace, i, indentSize, leadingNewline, leadingNewlineOffset, leadingNewlines, leadingWhitespace, length, lineComment, match, matchIllegal, noIndent, nonInitial, placeholderToken, precededByBlankLine, precedingNonCommentLines, prev;
      if (!(match = chunk.match(COMMENT))) {
        return 0;
      }
      [commentWithSurroundingWhitespace, hereLeadingWhitespace, hereComment, hereTrailingWhitespace, lineComment] = match;
      contents = null;
      // Does this comment follow code on the same line?
      leadingNewline = /^\s*\n+\s*#/.test(commentWithSurroundingWhitespace);
      if (hereComment) {
        matchIllegal = HERECOMMENT_ILLEGAL.exec(hereComment);
        if (matchIllegal) {
          this.error(`block comments cannot contain ${matchIllegal[0]}`, {
            offset: '###'.length + matchIllegal.index,
            length: matchIllegal[0].length
          });
        }
        // Parse indentation or outdentation as if this block comment didn’t exist.
        chunk = chunk.replace(`###${hereComment}###`, '');
        // Remove leading newlines, like `Rewriter::removeLeadingNewlines`, to
        // avoid the creation of unwanted `TERMINATOR` tokens.
        chunk = chunk.replace(/^\n+/, '');
        this.lineToken({chunk});
        // Pull out the ###-style comment’s content, and format it.
        content = hereComment;
        contents = [
          {
            content,
            length: commentWithSurroundingWhitespace.length - hereLeadingWhitespace.length - hereTrailingWhitespace.length,
            leadingWhitespace: hereLeadingWhitespace
          }
        ];
      } else {
        // The `COMMENT` regex captures successive line comments as one token.
        // Remove any leading newlines before the first comment, but preserve
        // blank lines between line comments.
        leadingNewlines = '';
        content = lineComment.replace(/^(\n*)/, function(leading) {
          leadingNewlines = leading;
          return '';
        });
        precedingNonCommentLines = '';
        hasSeenFirstCommentLine = false;
        contents = content.split('\n').map(function(line, index) {
          var comment, leadingWhitespace;
          if (!(line.indexOf('#') > -1)) {
            precedingNonCommentLines += `\n${line}`;
            return;
          }
          leadingWhitespace = '';
          content = line.replace(/^([ |\t]*)#/, function(_, whitespace) {
            leadingWhitespace = whitespace;
            return '';
          });
          comment = {
            content,
            length: '#'.length + content.length,
            leadingWhitespace: `${!hasSeenFirstCommentLine ? leadingNewlines : ''}${precedingNonCommentLines}${leadingWhitespace}`,
            precededByBlankLine: !!precedingNonCommentLines
          };
          hasSeenFirstCommentLine = true;
          precedingNonCommentLines = '';
          return comment;
        }).filter(function(comment) {
          return comment;
        });
      }
      getIndentSize = function({leadingWhitespace, nonInitial}) {
        var lastNewlineIndex;
        lastNewlineIndex = leadingWhitespace.lastIndexOf('\n');
        if ((hereComment != null) || !nonInitial) {
          if (!(lastNewlineIndex > -1)) {
            return null;
          }
        } else {
          if (lastNewlineIndex == null) {
            lastNewlineIndex = -1;
          }
        }
        return leadingWhitespace.length - 1 - lastNewlineIndex;
      };
      commentAttachments = (function() {
        var j, len, results;
        results = [];
        for (i = j = 0, len = contents.length; j < len; i = ++j) {
          ({content, length, leadingWhitespace, precededByBlankLine} = contents[i]);
          nonInitial = i !== 0;
          leadingNewlineOffset = nonInitial ? 1 : 0;
          offsetInChunk += leadingNewlineOffset + leadingWhitespace.length;
          indentSize = getIndentSize({leadingWhitespace, nonInitial});
          noIndent = (indentSize == null) || indentSize === -1;
          commentAttachment = {
            content,
            here: hereComment != null,
            newLine: leadingNewline || nonInitial, // Line comments after the first one start new lines, by definition.
            locationData: this.makeLocationData({offsetInChunk, length}),
            precededByBlankLine,
            indentSize,
            indented: !noIndent && indentSize > this.indent,
            outdented: !noIndent && indentSize < this.indent
          };
          if (heregex) {
            commentAttachment.heregex = true;
          }
          offsetInChunk += length;
          results.push(commentAttachment);
        }
        return results;
      }).call(this);
      prev = this.prev();
      if (!prev) {
        // If there’s no previous token, create a placeholder token to attach
        // this comment to; and follow with a newline.
        commentAttachments[0].newLine = true;
        this.lineToken({
          chunk: this.chunk.slice(commentWithSurroundingWhitespace.length),
          offset: commentWithSurroundingWhitespace.length // Set the indent.
        });
        placeholderToken = this.makeToken('JS', '', {
          offset: commentWithSurroundingWhitespace.length,
          generated: true
        });
        placeholderToken.comments = commentAttachments;
        this.tokens.push(placeholderToken);
        this.newlineToken(commentWithSurroundingWhitespace.length);
      } else {
        attachCommentsToNode(commentAttachments, prev);
      }
      if (returnCommentTokens) {
        return commentAttachments;
      }
      return commentWithSurroundingWhitespace.length;
    }

    // Matches JavaScript interpolated directly into the source via backticks.
    jsToken() {
      var length, match, matchedHere, script;
      if (!(this.chunk.charAt(0) === '`' && (match = (matchedHere = HERE_JSTOKEN.exec(this.chunk)) || JSTOKEN.exec(this.chunk)))) {
        return 0;
      }
      // Convert escaped backticks to backticks, and escaped backslashes
      // just before escaped backticks to backslashes
      script = match[1];
      ({length} = match[0]);
      this.token('JS', script, {
        length,
        data: {
          here: !!matchedHere
        }
      });
      return length;
    }

    // Matches regular expression literals, as well as multiline extended ones.
    // Lexing regular expressions is difficult to distinguish from division, so we
    // borrow some basic heuristics from JavaScript and Ruby.
    regexToken() {
      var body, closed, comment, commentIndex, commentOpts, commentTokens, comments, delimiter, end, flags, fullMatch, index, leadingWhitespace, match, matchedComment, origin, prev, ref, ref1, regex, tokens;
      switch (false) {
        case !(match = REGEX_ILLEGAL.exec(this.chunk)):
          this.error(`regular expressions cannot begin with ${match[2]}`, {
            offset: match.index + match[1].length
          });
          break;
        case !(match = this.matchWithInterpolations(HEREGEX, '///')):
          ({tokens, index} = match);
          comments = [];
          while (matchedComment = HEREGEX_COMMENT.exec(this.chunk.slice(0, index))) {
            ({
              index: commentIndex
            } = matchedComment);
            [fullMatch, leadingWhitespace, comment] = matchedComment;
            comments.push({
              comment,
              offsetInChunk: commentIndex + leadingWhitespace.length
            });
          }
          commentTokens = flatten((function() {
            var j, len, results;
            results = [];
            for (j = 0, len = comments.length; j < len; j++) {
              commentOpts = comments[j];
              results.push(this.commentToken(commentOpts.comment, Object.assign(commentOpts, {
                heregex: true,
                returnCommentTokens: true
              })));
            }
            return results;
          }).call(this));
          break;
        case !(match = REGEX.exec(this.chunk)):
          [regex, body, closed] = match;
          this.validateEscapes(body, {
            isRegex: true,
            offsetInChunk: 1
          });
          index = regex.length;
          prev = this.prev();
          if (prev) {
            if (prev.spaced && (ref = prev[0], indexOf.call(CALLABLE, ref) >= 0)) {
              if (!closed || POSSIBLY_DIVISION.test(regex)) {
                return 0;
              }
            } else if (ref1 = prev[0], indexOf.call(NOT_REGEX, ref1) >= 0) {
              return 0;
            }
          }
          if (!closed) {
            this.error('missing / (unclosed regex)');
          }
          break;
        default:
          return 0;
      }
      [flags] = REGEX_FLAGS.exec(this.chunk.slice(index));
      end = index + flags.length;
      origin = this.makeToken('REGEX', null, {
        length: end
      });
      switch (false) {
        case !!VALID_FLAGS.test(flags):
          this.error(`invalid regular expression flags ${flags}`, {
            offset: index,
            length: flags.length
          });
          break;
        case !(regex || tokens.length === 1):
          delimiter = body ? '/' : '///';
          if (body == null) {
            body = tokens[0][1];
          }
          this.validateUnicodeCodePointEscapes(body, {delimiter});
          this.token('REGEX', `/${body}/${flags}`, {
            length: end,
            origin,
            data: {delimiter}
          });
          break;
        default:
          this.token('REGEX_START', '(', {
            length: 0,
            origin,
            generated: true
          });
          this.token('IDENTIFIER', 'RegExp', {
            length: 0,
            generated: true
          });
          this.token('CALL_START', '(', {
            length: 0,
            generated: true
          });
          this.mergeInterpolationTokens(tokens, {
            double: true,
            heregex: {flags},
            endOffset: end - flags.length,
            quote: '///'
          }, (str) => {
            return this.validateUnicodeCodePointEscapes(str, {delimiter});
          });
          if (flags) {
            this.token(',', ',', {
              offset: index - 1,
              length: 0,
              generated: true
            });
            this.token('STRING', '"' + flags + '"', {
              offset: index,
              length: flags.length
            });
          }
          this.token(')', ')', {
            offset: end,
            length: 0,
            generated: true
          });
          this.token('REGEX_END', ')', {
            offset: end,
            length: 0,
            generated: true
          });
      }
      // Explicitly attach any heregex comments to the REGEX/REGEX_END token.
      if (commentTokens != null ? commentTokens.length : void 0) {
        addTokenData(this.tokens[this.tokens.length - 1], {
          heregexCommentTokens: commentTokens
        });
      }
      return end;
    }

    // Matches newlines, indents, and outdents, and determines which is which.
    // If we can detect that the current line is continued onto the next line,
    // then the newline is suppressed:

    //     elements
    //       .each( ... )
    //       .map( ... )

    // Keeps track of the level of indentation, because a single outdent token
    // can close multiple indents, so we need to know how far in we happen to be.
    lineToken({chunk = this.chunk, offset = 0} = {}) {
      var backslash, diff, endsContinuationLineIndentation, indent, match, minLiteralLength, newIndentLiteral, noNewlines, prev, ref, size;
      if (!(match = MULTI_DENT.exec(chunk))) {
        return 0;
      }
      indent = match[0];
      prev = this.prev();
      backslash = (prev != null ? prev[0] : void 0) === '\\';
      if (!((backslash || ((ref = this.seenFor) != null ? ref.endsLength : void 0) < this.ends.length) && this.seenFor)) {
        this.seenFor = false;
      }
      if (!((backslash && this.seenImport) || this.importSpecifierList)) {
        this.seenImport = false;
      }
      if (!((backslash && this.seenExport) || this.exportSpecifierList)) {
        this.seenExport = false;
      }
      size = indent.length - 1 - indent.lastIndexOf('\n');
      noNewlines = this.unfinished();
      newIndentLiteral = size > 0 ? indent.slice(-size) : '';
      if (!/^(.?)\1*$/.exec(newIndentLiteral)) {
        this.error('mixed indentation', {
          offset: indent.length
        });
        return indent.length;
      }
      minLiteralLength = Math.min(newIndentLiteral.length, this.indentLiteral.length);
      if (newIndentLiteral.slice(0, minLiteralLength) !== this.indentLiteral.slice(0, minLiteralLength)) {
        this.error('indentation mismatch', {
          offset: indent.length
        });
        return indent.length;
      }
      if (size - this.continuationLineAdditionalIndent === this.indent) {
        if (noNewlines) {
          this.suppressNewlines();
        } else {
          this.newlineToken(offset);
        }
        return indent.length;
      }
      if (size > this.indent) {
        if (noNewlines) {
          if (!backslash) {
            this.continuationLineAdditionalIndent = size - this.indent;
          }
          if (this.continuationLineAdditionalIndent) {
            prev.continuationLineIndent = this.indent + this.continuationLineAdditionalIndent;
          }
          this.suppressNewlines();
          return indent.length;
        }
        if (!this.tokens.length) {
          this.baseIndent = this.indent = size;
          this.indentLiteral = newIndentLiteral;
          return indent.length;
        }
        diff = size - this.indent + this.outdebt;
        this.token('INDENT', diff, {
          offset: offset + indent.length - size,
          length: size
        });
        this.indents.push(diff);
        this.ends.push({
          tag: 'OUTDENT'
        });
        this.outdebt = this.continuationLineAdditionalIndent = 0;
        this.indent = size;
        this.indentLiteral = newIndentLiteral;
      } else if (size < this.baseIndent) {
        this.error('missing indentation', {
          offset: offset + indent.length
        });
      } else {
        endsContinuationLineIndentation = this.continuationLineAdditionalIndent > 0;
        this.continuationLineAdditionalIndent = 0;
        this.outdentToken({
          moveOut: this.indent - size,
          noNewlines,
          outdentLength: indent.length,
          offset,
          indentSize: size,
          endsContinuationLineIndentation
        });
      }
      return indent.length;
    }

    // Record an outdent token or multiple tokens, if we happen to be moving back
    // inwards past several recorded indents. Sets new @indent value.
    outdentToken({moveOut, noNewlines, outdentLength = 0, offset = 0, indentSize, endsContinuationLineIndentation}) {
      var decreasedIndent, dent, lastIndent, ref, terminatorToken;
      decreasedIndent = this.indent - moveOut;
      while (moveOut > 0) {
        lastIndent = this.indents[this.indents.length - 1];
        if (!lastIndent) {
          this.outdebt = moveOut = 0;
        } else if (this.outdebt && moveOut <= this.outdebt) {
          this.outdebt -= moveOut;
          moveOut = 0;
        } else {
          dent = this.indents.pop() + this.outdebt;
          if (outdentLength && (ref = this.chunk[outdentLength], indexOf.call(INDENTABLE_CLOSERS, ref) >= 0)) {
            decreasedIndent -= dent - moveOut;
            moveOut = dent;
          }
          this.outdebt = 0;
          // pair might call outdentToken, so preserve decreasedIndent
          this.pair('OUTDENT');
          this.token('OUTDENT', moveOut, {
            length: outdentLength,
            indentSize: indentSize + moveOut - dent
          });
          moveOut -= dent;
        }
      }
      if (dent) {
        this.outdebt -= moveOut;
      }
      this.suppressSemicolons();
      if (!(this.tag() === 'TERMINATOR' || noNewlines)) {
        terminatorToken = this.token('TERMINATOR', '\n', {
          offset: offset + outdentLength,
          length: 0
        });
        if (endsContinuationLineIndentation) {
          terminatorToken.endsContinuationLineIndentation = {
            preContinuationLineIndent: this.indent
          };
        }
      }
      this.indent = decreasedIndent;
      this.indentLiteral = this.indentLiteral.slice(0, decreasedIndent);
      return this;
    }

    // Matches and consumes non-meaningful whitespace. Tag the previous token
    // as being “spaced”, because there are some cases where it makes a difference.
    whitespaceToken() {
      var match, nline, prev;
      if (!((match = WHITESPACE.exec(this.chunk)) || (nline = this.chunk.charAt(0) === '\n'))) {
        return 0;
      }
      prev = this.prev();
      if (prev) {
        prev[match ? 'spaced' : 'newLine'] = true;
      }
      if (match) {
        return match[0].length;
      } else {
        return 0;
      }
    }

    // Generate a newline token. Consecutive newlines get merged together.
    newlineToken(offset) {
      this.suppressSemicolons();
      if (this.tag() !== 'TERMINATOR') {
        this.token('TERMINATOR', '\n', {
          offset,
          length: 0
        });
      }
      return this;
    }

    // Use a `\` at a line-ending to suppress the newline.
    // The slash is removed here once its job is done.
    suppressNewlines() {
      var prev;
      prev = this.prev();
      if (prev[1] === '\\') {
        if (prev.comments && this.tokens.length > 1) {
          // `@tokens.length` should be at least 2 (some code, then `\`).
          // If something puts a `\` after nothing, they deserve to lose any
          // comments that trail it.
          attachCommentsToNode(prev.comments, this.tokens[this.tokens.length - 2]);
        }
        this.tokens.pop();
      }
      return this;
    }

    jsxToken() {
      var afterTag, end, endToken, firstChar, fullId, fullTagName, id, input, j, jsxTag, len, match, offset, openingTagToken, prev, prevChar, properties, property, ref, tagToken, token, tokens;
      firstChar = this.chunk[0];
      // Check the previous token to detect if attribute is spread.
      prevChar = this.tokens.length > 0 ? this.tokens[this.tokens.length - 1][0] : '';
      if (firstChar === '<') {
        match = JSX_IDENTIFIER.exec(this.chunk.slice(1)) || JSX_FRAGMENT_IDENTIFIER.exec(this.chunk.slice(1));
        // Not the right hand side of an unspaced comparison (i.e. `a<b`).
        if (!(match && (this.jsxDepth > 0 || !(prev = this.prev()) || prev.spaced || (ref = prev[0], indexOf.call(COMPARABLE_LEFT_SIDE, ref) < 0)))) {
          return 0;
        }
        [input, id] = match;
        fullId = id;
        if (indexOf.call(id, '.') >= 0) {
          [id, ...properties] = id.split('.');
        } else {
          properties = [];
        }
        tagToken = this.token('JSX_TAG', id, {
          length: id.length + 1,
          data: {
            openingBracketToken: this.makeToken('<', '<'),
            tagNameToken: this.makeToken('IDENTIFIER', id, {
              offset: 1
            })
          }
        });
        offset = id.length + 1;
        for (j = 0, len = properties.length; j < len; j++) {
          property = properties[j];
          this.token('.', '.', {offset});
          offset += 1;
          this.token('PROPERTY', property, {offset});
          offset += property.length;
        }
        this.token('CALL_START', '(', {
          generated: true
        });
        this.token('[', '[', {
          generated: true
        });
        this.ends.push({
          tag: '/>',
          origin: tagToken,
          name: id,
          properties
        });
        this.jsxDepth++;
        return fullId.length + 1;
      } else if (jsxTag = this.atJSXTag()) {
        if (this.chunk.slice(0, 2) === '/>') { // Self-closing tag.
          this.pair('/>');
          this.token(']', ']', {
            length: 2,
            generated: true
          });
          this.token('CALL_END', ')', {
            length: 2,
            generated: true,
            data: {
              selfClosingSlashToken: this.makeToken('/', '/'),
              closingBracketToken: this.makeToken('>', '>', {
                offset: 1
              })
            }
          });
          this.jsxDepth--;
          return 2;
        } else if (firstChar === '{') {
          if (prevChar === ':') {
            // This token represents the start of a JSX attribute value
            // that’s an expression (e.g. the `{b}` in `<div a={b} />`).
            // Our grammar represents the beginnings of expressions as `(`
            // tokens, so make this into a `(` token that displays as `{`.
            token = this.token('(', '{');
            this.jsxObjAttribute[this.jsxDepth] = false;
            // tag attribute name as JSX
            addTokenData(this.tokens[this.tokens.length - 3], {
              jsx: true
            });
          } else {
            token = this.token('{', '{');
            this.jsxObjAttribute[this.jsxDepth] = true;
          }
          this.ends.push({
            tag: '}',
            origin: token
          });
          return 1;
        } else if (firstChar === '>') { // end of opening tag
          ({
            // Ignore terminators inside a tag.
            origin: openingTagToken
          } = this.pair('/>')); // As if the current tag was self-closing.
          this.token(']', ']', {
            generated: true,
            data: {
              closingBracketToken: this.makeToken('>', '>')
            }
          });
          this.token(',', 'JSX_COMMA', {
            generated: true
          });
          ({
            tokens,
            index: end
          } = this.matchWithInterpolations(INSIDE_JSX, '>', '</', JSX_INTERPOLATION));
          this.mergeInterpolationTokens(tokens, {
            endOffset: end,
            jsx: true
          }, (value) => {
            return this.validateUnicodeCodePointEscapes(value, {
              delimiter: '>'
            });
          });
          match = JSX_IDENTIFIER.exec(this.chunk.slice(end)) || JSX_FRAGMENT_IDENTIFIER.exec(this.chunk.slice(end));
          if (!match || match[1] !== `${jsxTag.name}${((function() {
            var k, len1, ref1, results;
            ref1 = jsxTag.properties;
            results = [];
            for (k = 0, len1 = ref1.length; k < len1; k++) {
              property = ref1[k];
              results.push(`.${property}`);
            }
            return results;
          })()).join('')}`) {
            this.error(`expected corresponding JSX closing tag for ${jsxTag.name}`, jsxTag.origin.data.tagNameToken[2]);
          }
          [, fullTagName] = match;
          afterTag = end + fullTagName.length;
          if (this.chunk[afterTag] !== '>') {
            this.error("missing closing > after tag name", {
              offset: afterTag,
              length: 1
            });
          }
          // -2/+2 for the opening `</` and +1 for the closing `>`.
          endToken = this.token('CALL_END', ')', {
            offset: end - 2,
            length: fullTagName.length + 3,
            generated: true,
            data: {
              closingTagOpeningBracketToken: this.makeToken('<', '<', {
                offset: end - 2
              }),
              closingTagSlashToken: this.makeToken('/', '/', {
                offset: end - 1
              }),
              // TODO: individual tokens for complex tag name? eg < / A . B >
              closingTagNameToken: this.makeToken('IDENTIFIER', fullTagName, {
                offset: end
              }),
              closingTagClosingBracketToken: this.makeToken('>', '>', {
                offset: end + fullTagName.length
              })
            }
          });
          // make the closing tag location data more easily accessible to the grammar
          addTokenData(openingTagToken, endToken.data);
          this.jsxDepth--;
          return afterTag + 1;
        } else {
          return 0;
        }
      } else if (this.atJSXTag(1)) {
        if (firstChar === '}') {
          this.pair(firstChar);
          if (this.jsxObjAttribute[this.jsxDepth]) {
            this.token('}', '}');
            this.jsxObjAttribute[this.jsxDepth] = false;
          } else {
            this.token(')', '}');
          }
          this.token(',', ',', {
            generated: true
          });
          return 1;
        } else {
          return 0;
        }
      } else {
        return 0;
      }
    }

    atJSXTag(depth = 0) {
      var i, last, ref;
      if (this.jsxDepth === 0) {
        return false;
      }
      i = this.ends.length - 1;
      while (((ref = this.ends[i]) != null ? ref.tag : void 0) === 'OUTDENT' || depth-- > 0) { // Ignore indents.
        i--;
      }
      last = this.ends[i];
      return (last != null ? last.tag : void 0) === '/>' && last;
    }

    // We treat all other single characters as a token. E.g.: `( ) , . !`
    // Multi-character operators are also literal tokens, so that Jison can assign
    // the proper order of operations. There are some symbols that we tag specially
    // here. `;` and newlines are both treated as a `TERMINATOR`, we distinguish
    // parentheses that indicate a method call from regular parentheses, and so on.
    literalToken() {
      var match, message, origin, prev, ref, ref1, ref2, ref3, ref4, ref5, skipToken, tag, token, value;
      if (match = OPERATOR.exec(this.chunk)) {
        [value] = match;
        if (CODE.test(value)) {
          this.tagParameters();
        }
      } else {
        value = this.chunk.charAt(0);
      }
      tag = value;
      prev = this.prev();
      if (prev && indexOf.call(['=', ...COMPOUND_ASSIGN], value) >= 0) {
        skipToken = false;
        if (value === '=' && ((ref = prev[1]) === '||' || ref === '&&') && !prev.spaced) {
          prev[0] = 'COMPOUND_ASSIGN';
          prev[1] += '=';
          if ((ref1 = prev.data) != null ? ref1.original : void 0) {
            prev.data.original += '=';
          }
          prev[2].range = [prev[2].range[0], prev[2].range[1] + 1];
          prev[2].last_column += 1;
          prev[2].last_column_exclusive += 1;
          prev = this.tokens[this.tokens.length - 2];
          skipToken = true;
        }
        if (prev && prev[0] !== 'PROPERTY') {
          origin = (ref2 = prev.origin) != null ? ref2 : prev;
          message = isUnassignable(prev[1], origin[1]);
          if (message) {
            this.error(message, origin[2]);
          }
        }
        if (skipToken) {
          return value.length;
        }
      }
      if (value === '(' && (prev != null ? prev[0] : void 0) === 'IMPORT') {
        prev[0] = 'DYNAMIC_IMPORT';
      }
      if (value === '{' && this.seenImport) {
        this.importSpecifierList = true;
      } else if (this.importSpecifierList && value === '}') {
        this.importSpecifierList = false;
      } else if (value === '{' && (prev != null ? prev[0] : void 0) === 'EXPORT') {
        this.exportSpecifierList = true;
      } else if (this.exportSpecifierList && value === '}') {
        this.exportSpecifierList = false;
      }
      if (value === ';') {
        if (ref3 = prev != null ? prev[0] : void 0, indexOf.call(['=', ...UNFINISHED], ref3) >= 0) {
          this.error('unexpected ;');
        }
        this.seenFor = this.seenImport = this.seenExport = false;
        tag = 'TERMINATOR';
      } else if (value === '*' && (prev != null ? prev[0] : void 0) === 'EXPORT') {
        tag = 'EXPORT_ALL';
      } else if (indexOf.call(MATH, value) >= 0) {
        tag = 'MATH';
      } else if (indexOf.call(COMPARE, value) >= 0) {
        tag = 'COMPARE';
      } else if (indexOf.call(COMPOUND_ASSIGN, value) >= 0) {
        tag = 'COMPOUND_ASSIGN';
      } else if (indexOf.call(UNARY, value) >= 0) {
        tag = 'UNARY';
      } else if (indexOf.call(UNARY_MATH, value) >= 0) {
        tag = 'UNARY_MATH';
      } else if (indexOf.call(SHIFT, value) >= 0) {
        tag = 'SHIFT';
      } else if (value === '?' && (prev != null ? prev.spaced : void 0)) {
        tag = 'BIN?';
      } else if (prev) {
        if (value === '(' && !prev.spaced && (ref4 = prev[0], indexOf.call(CALLABLE, ref4) >= 0)) {
          if (prev[0] === '?') {
            prev[0] = 'FUNC_EXIST';
          }
          tag = 'CALL_START';
        } else if (value === '[' && (((ref5 = prev[0], indexOf.call(INDEXABLE, ref5) >= 0) && !prev.spaced) || (prev[0] === '::'))) { // `.prototype` can’t be a method you can call.
          tag = 'INDEX_START';
          switch (prev[0]) {
            case '?':
              prev[0] = 'INDEX_SOAK';
          }
        }
      }
      token = this.makeToken(tag, value);
      switch (value) {
        case '(':
        case '{':
        case '[':
          this.ends.push({
            tag: INVERSES[value],
            origin: token
          });
          break;
        case ')':
        case '}':
        case ']':
          this.pair(value);
      }
      this.tokens.push(this.makeToken(tag, value));
      return value.length;
    }

    // Token Manipulators
    // ------------------

      // A source of ambiguity in our grammar used to be parameter lists in function
    // definitions versus argument lists in function calls. Walk backwards, tagging
    // parameters specially in order to make things easier for the parser.
    tagParameters() {
      var i, paramEndToken, stack, tok, tokens;
      if (this.tag() !== ')') {
        return this.tagDoIife();
      }
      stack = [];
      ({tokens} = this);
      i = tokens.length;
      paramEndToken = tokens[--i];
      paramEndToken[0] = 'PARAM_END';
      while (tok = tokens[--i]) {
        switch (tok[0]) {
          case ')':
            stack.push(tok);
            break;
          case '(':
          case 'CALL_START':
            if (stack.length) {
              stack.pop();
            } else if (tok[0] === '(') {
              tok[0] = 'PARAM_START';
              return this.tagDoIife(i - 1);
            } else {
              paramEndToken[0] = 'CALL_END';
              return this;
            }
        }
      }
      return this;
    }

    // Tag `do` followed by a function differently than `do` followed by eg an
    // identifier to allow for different grammar precedence
    tagDoIife(tokenIndex) {
      var tok;
      tok = this.tokens[tokenIndex != null ? tokenIndex : this.tokens.length - 1];
      if ((tok != null ? tok[0] : void 0) !== 'DO') {
        return this;
      }
      tok[0] = 'DO_IIFE';
      return this;
    }

    // Close up all remaining open blocks at the end of the file.
    closeIndentation() {
      return this.outdentToken({
        moveOut: this.indent,
        indentSize: 0
      });
    }

    // Match the contents of a delimited token and expand variables and expressions
    // inside it using Ruby-like notation for substitution of arbitrary
    // expressions.

    //     "Hello #{name.capitalize()}."

    // If it encounters an interpolation, this method will recursively create a new
    // Lexer and tokenize until the `{` of `#{` is balanced with a `}`.

    //  - `regex` matches the contents of a token (but not `delimiter`, and not
    //    `#{` if interpolations are desired).
    //  - `delimiter` is the delimiter of the token. Examples are `'`, `"`, `'''`,
    //    `"""` and `///`.
    //  - `closingDelimiter` is different from `delimiter` only in JSX
    //  - `interpolators` matches the start of an interpolation, for JSX it's both
    //    `{` and `<` (i.e. nested JSX tag)

    // This method allows us to have strings within interpolations within strings,
    // ad infinitum.
    matchWithInterpolations(regex, delimiter, closingDelimiter = delimiter, interpolators = /^#\{/) {
      var braceInterpolator, close, column, index, interpolationOffset, interpolator, line, match, nested, offset, offsetInChunk, open, ref, ref1, rest, str, strPart, tokens;
      tokens = [];
      offsetInChunk = delimiter.length;
      if (this.chunk.slice(0, offsetInChunk) !== delimiter) {
        return null;
      }
      str = this.chunk.slice(offsetInChunk);
      while (true) {
        [strPart] = regex.exec(str);
        this.validateEscapes(strPart, {
          isRegex: delimiter.charAt(0) === '/',
          offsetInChunk
        });
        // Push a fake `'NEOSTRING'` token, which will get turned into a real string later.
        tokens.push(this.makeToken('NEOSTRING', strPart, {
          offset: offsetInChunk
        }));
        str = str.slice(strPart.length);
        offsetInChunk += strPart.length;
        if (!(match = interpolators.exec(str))) {
          break;
        }
        [interpolator] = match;
        // To remove the `#` in `#{`.
        interpolationOffset = interpolator.length - 1;
        [line, column, offset] = this.getLineAndColumnFromChunk(offsetInChunk + interpolationOffset);
        rest = str.slice(interpolationOffset);
        ({
          tokens: nested,
          index
        } = new Lexer().tokenize(rest, {
          line,
          column,
          offset,
          untilBalanced: true,
          locationDataCompensations: this.locationDataCompensations
        }));
        // Account for the `#` in `#{`.
        index += interpolationOffset;
        braceInterpolator = str[index - 1] === '}';
        if (braceInterpolator) {
          // Turn the leading and trailing `{` and `}` into parentheses. Unnecessary
          // parentheses will be removed later.
          [open] = nested, [close] = slice.call(nested, -1);
          open[0] = 'INTERPOLATION_START';
          open[1] = '(';
          open[2].first_column -= interpolationOffset;
          open[2].range = [open[2].range[0] - interpolationOffset, open[2].range[1]];
          close[0] = 'INTERPOLATION_END';
          close[1] = ')';
          close.origin = ['', 'end of interpolation', close[2]];
        }
        if (((ref = nested[1]) != null ? ref[0] : void 0) === 'TERMINATOR') {
          // Remove leading `'TERMINATOR'` (if any).
          nested.splice(1, 1);
        }
        if (((ref1 = nested[nested.length - 3]) != null ? ref1[0] : void 0) === 'INDENT' && nested[nested.length - 2][0] === 'OUTDENT') {
          // Remove trailing `'INDENT'/'OUTDENT'` pair (if any).
          nested.splice(-3, 2);
        }
        if (!braceInterpolator) {
          // We are not using `{` and `}`, so wrap the interpolated tokens instead.
          open = this.makeToken('INTERPOLATION_START', '(', {
            offset: offsetInChunk,
            length: 0,
            generated: true
          });
          close = this.makeToken('INTERPOLATION_END', ')', {
            offset: offsetInChunk + index,
            length: 0,
            generated: true
          });
          nested = [open, ...nested, close];
        }
        // Push a fake `'TOKENS'` token, which will get turned into real tokens later.
        tokens.push(['TOKENS', nested]);
        str = str.slice(index);
        offsetInChunk += index;
      }
      if (str.slice(0, closingDelimiter.length) !== closingDelimiter) {
        this.error(`missing ${closingDelimiter}`, {
          length: delimiter.length
        });
      }
      return {
        tokens,
        index: offsetInChunk + closingDelimiter.length
      };
    }

    // Merge the array `tokens` of the fake token types `'TOKENS'` and `'NEOSTRING'`
    // (as returned by `matchWithInterpolations`) into the token stream. The value
    // of `'NEOSTRING'`s are converted using `fn` and turned into strings using
    // `options` first.
    mergeInterpolationTokens(tokens, options, fn) {
      var $, converted, double, endOffset, firstIndex, heregex, i, indent, j, jsx, k, lastToken, len, len1, locationToken, lparen, placeholderToken, quote, ref, ref1, rparen, tag, token, tokensToPush, val, value;
      ({quote, indent, double, heregex, endOffset, jsx} = options);
      if (tokens.length > 1) {
        lparen = this.token('STRING_START', '(', {
          length: (ref = quote != null ? quote.length : void 0) != null ? ref : 0,
          data: {quote},
          generated: !(quote != null ? quote.length : void 0)
        });
      }
      firstIndex = this.tokens.length;
      $ = tokens.length - 1;
      for (i = j = 0, len = tokens.length; j < len; i = ++j) {
        token = tokens[i];
        [tag, value] = token;
        switch (tag) {
          case 'TOKENS':
            // There are comments (and nothing else) in this interpolation.
            if (value.length === 2 && (value[0].comments || value[1].comments)) {
              placeholderToken = this.makeToken('JS', '', {
                generated: true
              });
              // Use the same location data as the first parenthesis.
              placeholderToken[2] = value[0][2];
              for (k = 0, len1 = value.length; k < len1; k++) {
                val = value[k];
                if (!val.comments) {
                  continue;
                }
                if (placeholderToken.comments == null) {
                  placeholderToken.comments = [];
                }
                placeholderToken.comments.push(...val.comments);
              }
              value.splice(1, 0, placeholderToken);
            }
            // Push all the tokens in the fake `'TOKENS'` token. These already have
            // sane location data.
            locationToken = value[0];
            tokensToPush = value;
            break;
          case 'NEOSTRING':
            // Convert `'NEOSTRING'` into `'STRING'`.
            converted = fn.call(this, token[1], i);
            if (i === 0) {
              addTokenData(token, {
                initialChunk: true
              });
            }
            if (i === $) {
              addTokenData(token, {
                finalChunk: true
              });
            }
            addTokenData(token, {indent, quote, double});
            if (heregex) {
              addTokenData(token, {heregex});
            }
            if (jsx) {
              addTokenData(token, {jsx});
            }
            token[0] = 'STRING';
            token[1] = '"' + converted + '"';
            if (tokens.length === 1 && (quote != null)) {
              token[2].first_column -= quote.length;
              if (token[1].substr(-2, 1) === '\n') {
                token[2].last_line += 1;
                token[2].last_column = quote.length - 1;
              } else {
                token[2].last_column += quote.length;
                if (token[1].length === 2) {
                  token[2].last_column -= 1;
                }
              }
              token[2].last_column_exclusive += quote.length;
              token[2].range = [token[2].range[0] - quote.length, token[2].range[1] + quote.length];
            }
            locationToken = token;
            tokensToPush = [token];
        }
        this.tokens.push(...tokensToPush);
      }
      if (lparen) {
        [lastToken] = slice.call(tokens, -1);
        lparen.origin = [
          'STRING',
          null,
          {
            first_line: lparen[2].first_line,
            first_column: lparen[2].first_column,
            last_line: lastToken[2].last_line,
            last_column: lastToken[2].last_column,
            last_line_exclusive: lastToken[2].last_line_exclusive,
            last_column_exclusive: lastToken[2].last_column_exclusive,
            range: [lparen[2].range[0],
          lastToken[2].range[1]]
          }
        ];
        if (!(quote != null ? quote.length : void 0)) {
          lparen[2] = lparen.origin[2];
        }
        return rparen = this.token('STRING_END', ')', {
          offset: endOffset - (quote != null ? quote : '').length,
          length: (ref1 = quote != null ? quote.length : void 0) != null ? ref1 : 0,
          generated: !(quote != null ? quote.length : void 0)
        });
      }
    }

    // Pairs up a closing token, ensuring that all listed pairs of tokens are
    // correctly balanced throughout the course of the token stream.
    pair(tag) {
      var lastIndent, prev, ref, ref1, wanted;
      ref = this.ends, [prev] = slice.call(ref, -1);
      if (tag !== (wanted = prev != null ? prev.tag : void 0)) {
        if ('OUTDENT' !== wanted) {
          this.error(`unmatched ${tag}`);
        }
        // Auto-close `INDENT` to support syntax like this:

        //     el.click((event) ->
        //       el.hide())

        ref1 = this.indents, [lastIndent] = slice.call(ref1, -1);
        this.outdentToken({
          moveOut: lastIndent,
          noNewlines: true
        });
        return this.pair(tag);
      }
      return this.ends.pop();
    }

    // Helpers
    // -------

      // Compensate for the things we strip out initially (e.g. carriage returns)
    // so that location data stays accurate with respect to the original source file.
    getLocationDataCompensation(start, end) {
      var compensation, current, initialEnd, totalCompensation;
      totalCompensation = 0;
      initialEnd = end;
      current = start;
      while (current <= end) {
        if (current === end && start !== initialEnd) {
          break;
        }
        compensation = this.locationDataCompensations[current];
        if (compensation != null) {
          totalCompensation += compensation;
          end += compensation;
        }
        current++;
      }
      return totalCompensation;
    }

    // Returns the line and column number from an offset into the current chunk.

    // `offset` is a number of characters into `@chunk`.
    getLineAndColumnFromChunk(offset) {
      var column, columnCompensation, compensation, lastLine, lineCount, previousLinesCompensation, ref, string;
      compensation = this.getLocationDataCompensation(this.chunkOffset, this.chunkOffset + offset);
      if (offset === 0) {
        return [this.chunkLine, this.chunkColumn + compensation, this.chunkOffset + compensation];
      }
      if (offset >= this.chunk.length) {
        string = this.chunk;
      } else {
        string = this.chunk.slice(0, +(offset - 1) + 1 || 9e9);
      }
      lineCount = count(string, '\n');
      column = this.chunkColumn;
      if (lineCount > 0) {
        ref = string.split('\n'), [lastLine] = slice.call(ref, -1);
        column = lastLine.length;
        previousLinesCompensation = this.getLocationDataCompensation(this.chunkOffset, this.chunkOffset + offset - column);
        if (previousLinesCompensation < 0) {
          // Don't recompensate for initially inserted newline.
          previousLinesCompensation = 0;
        }
        columnCompensation = this.getLocationDataCompensation(this.chunkOffset + offset + previousLinesCompensation - column, this.chunkOffset + offset + previousLinesCompensation);
      } else {
        column += string.length;
        columnCompensation = compensation;
      }
      return [this.chunkLine + lineCount, column + columnCompensation, this.chunkOffset + offset + compensation];
    }

    makeLocationData({offsetInChunk, length}) {
      var endOffset, lastCharacter, locationData;
      locationData = {
        range: []
      };
      [locationData.first_line, locationData.first_column, locationData.range[0]] = this.getLineAndColumnFromChunk(offsetInChunk);
      // Use length - 1 for the final offset - we’re supplying the last_line and the last_column,
      // so if last_column == first_column, then we’re looking at a character of length 1.
      lastCharacter = length > 0 ? length - 1 : 0;
      [locationData.last_line, locationData.last_column, endOffset] = this.getLineAndColumnFromChunk(offsetInChunk + lastCharacter);
      [locationData.last_line_exclusive, locationData.last_column_exclusive] = this.getLineAndColumnFromChunk(offsetInChunk + lastCharacter + (length > 0 ? 1 : 0));
      locationData.range[1] = length > 0 ? endOffset + 1 : endOffset;
      return locationData;
    }

    // Same as `token`, except this just returns the token without adding it
    // to the results.
    makeToken(tag, value, {
        offset: offsetInChunk = 0,
        length = value.length,
        origin,
        generated,
        indentSize
      } = {}) {
      var token;
      token = [tag, value, this.makeLocationData({offsetInChunk, length})];
      if (origin) {
        token.origin = origin;
      }
      if (generated) {
        token.generated = true;
      }
      if (indentSize != null) {
        token.indentSize = indentSize;
      }
      return token;
    }

    // Add a token to the results.
    // `offset` is the offset into the current `@chunk` where the token starts.
    // `length` is the length of the token in the `@chunk`, after the offset.  If
    // not specified, the length of `value` will be used.

    // Returns the new token.
    token(tag, value, {offset, length, origin, data, generated, indentSize} = {}) {
      var token;
      token = this.makeToken(tag, value, {offset, length, origin, generated, indentSize});
      if (data) {
        addTokenData(token, data);
      }
      this.tokens.push(token);
      return token;
    }

    // Peek at the last tag in the token stream.
    tag() {
      var ref, token;
      ref = this.tokens, [token] = slice.call(ref, -1);
      return token != null ? token[0] : void 0;
    }

    // Peek at the last value in the token stream.
    value(useOrigin = false) {
      var ref, token;
      ref = this.tokens, [token] = slice.call(ref, -1);
      if (useOrigin && ((token != null ? token.origin : void 0) != null)) {
        return token.origin[1];
      } else {
        return token != null ? token[1] : void 0;
      }
    }

    // Get the previous token in the token stream.
    prev() {
      return this.tokens[this.tokens.length - 1];
    }

    // Are we in the midst of an unfinished expression?
    unfinished() {
      var ref;
      return LINE_CONTINUER.test(this.chunk) || (ref = this.tag(), indexOf.call(UNFINISHED, ref) >= 0);
    }

    validateUnicodeCodePointEscapes(str, options) {
      return replaceUnicodeCodePointEscapes(str, merge(options, {error: this.error}));
    }

    // Validates escapes in strings and regexes.
    validateEscapes(str, options = {}) {
      var before, hex, invalidEscape, invalidEscapeRegex, match, message, octal, ref, unicode, unicodeCodePoint;
      invalidEscapeRegex = options.isRegex ? REGEX_INVALID_ESCAPE : STRING_INVALID_ESCAPE;
      match = invalidEscapeRegex.exec(str);
      if (!match) {
        return;
      }
      match[0], before = match[1], octal = match[2], hex = match[3], unicodeCodePoint = match[4], unicode = match[5];
      message = octal ? "octal escape sequences are not allowed" : "invalid escape sequence";
      invalidEscape = `\\${octal || hex || unicodeCodePoint || unicode}`;
      return this.error(`${message} ${invalidEscape}`, {
        offset: ((ref = options.offsetInChunk) != null ? ref : 0) + match.index + before.length,
        length: invalidEscape.length
      });
    }

    suppressSemicolons() {
      var ref, ref1, results;
      results = [];
      while (this.value() === ';') {
        this.tokens.pop();
        if (ref = (ref1 = this.prev()) != null ? ref1[0] : void 0, indexOf.call(['=', ...UNFINISHED], ref) >= 0) {
          results.push(this.error('unexpected ;'));
        } else {
          results.push(void 0);
        }
      }
      return results;
    }

    error(message, options = {}) {
      var first_column, first_line, location, ref, ref1;
      location = 'first_line' in options ? options : ([first_line, first_column] = this.getLineAndColumnFromChunk((ref = options.offset) != null ? ref : 0), {
        first_line,
        first_column,
        last_column: first_column + ((ref1 = options.length) != null ? ref1 : 1) - 1
      });
      return throwSyntaxError(message, location);
    }

  };

  // Helper functions
  // ----------------
  isUnassignable = function(name, displayName = name) {
    switch (false) {
      case indexOf.call([...JS_KEYWORDS, ...COFFEE_KEYWORDS], name) < 0:
        return `keyword '${displayName}' can't be assigned`;
      case indexOf.call(STRICT_PROSCRIBED, name) < 0:
        return `'${displayName}' can't be assigned`;
      case indexOf.call(RESERVED, name) < 0:
        return `reserved word '${displayName}' can't be assigned`;
      default:
        return false;
    }
  };

  exports.isUnassignable = isUnassignable;

  // `from` isn’t a CoffeeScript keyword, but it behaves like one in `import` and
  // `export` statements (handled above) and in the declaration line of a `for`
  // loop. Try to detect when `from` is a variable identifier and when it is this
  // “sometimes” keyword.
  isForFrom = function(prev) {
    var ref;
    // `for i from iterable`
    if (prev[0] === 'IDENTIFIER') {
      return true;
    // `for from…`
    } else if (prev[0] === 'FOR') {
      return false;
    // `for {from}…`, `for [from]…`, `for {a, from}…`, `for {a: from}…`
    } else if ((ref = prev[1]) === '{' || ref === '[' || ref === ',' || ref === ':') {
      return false;
    } else {
      return true;
    }
  };

  addTokenData = function(token, data) {
    return Object.assign((token.data != null ? token.data : token.data = {}), data);
  };

  // Constants
  // ---------

  // Keywords that CoffeeScript shares in common with JavaScript.
  JS_KEYWORDS = ['true', 'false', 'null', 'this', 'new', 'delete', 'typeof', 'in', 'instanceof', 'return', 'throw', 'break', 'continue', 'debugger', 'yield', 'await', 'if', 'else', 'switch', 'for', 'while', 'do', 'try', 'catch', 'finally', 'class', 'extends', 'super', 'import', 'export', 'default'];

  // CoffeeScript-only keywords.
  COFFEE_KEYWORDS = ['undefined', 'Infinity', 'NaN', 'then', 'unless', 'until', 'loop', 'of', 'by', 'when'];

  COFFEE_ALIAS_MAP = {
    and: '&&',
    or: '||',
    is: '==',
    isnt: '!=',
    not: '!',
    yes: 'true',
    no: 'false',
    on: 'true',
    off: 'false'
  };

  COFFEE_ALIASES = (function() {
    var results;
    results = [];
    for (key in COFFEE_ALIAS_MAP) {
      results.push(key);
    }
    return results;
  })();

  COFFEE_KEYWORDS = COFFEE_KEYWORDS.concat(COFFEE_ALIASES);

  // The list of keywords that are reserved by JavaScript, but not used, or are
  // used by CoffeeScript internally. We throw an error when these are encountered,
  // to avoid having a JavaScript error at runtime.
  RESERVED = ['case', 'function', 'var', 'void', 'with', 'const', 'let', 'enum', 'native', 'implements', 'interface', 'package', 'private', 'protected', 'public', 'static'];

  STRICT_PROSCRIBED = ['arguments', 'eval'];

  // The superset of both JavaScript keywords and reserved words, none of which may
  // be used as identifiers or properties.
  exports.JS_FORBIDDEN = JS_KEYWORDS.concat(RESERVED).concat(STRICT_PROSCRIBED);

  // The character code of the nasty Microsoft madness otherwise known as the BOM.
  BOM = 65279;

  // Token matching regexes.
  IDENTIFIER = /^(?!\d)((?:(?!\s)[$\w\x7f-\uffff])+)([^\n\S]*:(?!:))?/; // Is this a property name?

  // Like `IDENTIFIER`, but includes `-`s
  JSX_IDENTIFIER_PART = /(?:(?!\s)[\-$\w\x7f-\uffff])+/.source;

  // In https://facebook.github.io/jsx/ spec, JSXElementName can be
  // JSXIdentifier, JSXNamespacedName (JSXIdentifier : JSXIdentifier), or
  // JSXMemberExpression (two or more JSXIdentifier connected by `.`s).
  JSX_IDENTIFIER = RegExp(`^(?![\\d<])(${JSX_IDENTIFIER_PART // Must not start with `<`.
  // JSXNamespacedName
  // JSXMemberExpression
}(?:\\s*:\\s*${JSX_IDENTIFIER_PART}|(?:\\s*\\.\\s*${JSX_IDENTIFIER_PART})+)?)`);

  // Fragment: <></>
  JSX_FRAGMENT_IDENTIFIER = /^()>/; // Ends immediately with `>`.

  // In https://facebook.github.io/jsx/ spec, JSXAttributeName can be either
  // JSXIdentifier or JSXNamespacedName which is JSXIdentifier : JSXIdentifier
  JSX_ATTRIBUTE = RegExp(`^(?!\\d)(${JSX_IDENTIFIER_PART // JSXNamespacedName
  // Is this an attribute with a value?
}(?:\\s*:\\s*${JSX_IDENTIFIER_PART})?)([^\\S]*=(?!=))?`);

  NUMBER = /^0b[01](?:_?[01])*n?|^0o[0-7](?:_?[0-7])*n?|^0x[\da-f](?:_?[\da-f])*n?|^\d+(?:_\d+)*n|^(?:\d+(?:_\d+)*)?\.?\d+(?:_\d+)*(?:e[+-]?\d+(?:_\d+)*)?/i; // binary
  // octal
  // hex
  // decimal bigint
  // decimal
  // decimal without support for numeric literal separators for reference:
  // \d*\.?\d+ (?:e[+-]?\d+)?

  OPERATOR = /^(?:[-=]>|[-+*\/%<>&|^!?=]=|>>>=?|([-+:])\1|([&|<>*\/%])\2=?|\?(\.|::)|\.{2,3})/; // function
  // compound assign / compare
  // zero-fill right shift
  // doubles
  // logic / shift / power / floor division / modulo
  // soak access
  // range or splat

  WHITESPACE = /^[^\n\S]+/;

  COMMENT = /^(\s*)###([^#][\s\S]*?)(?:###([^\n\S]*)|###$)|^((?:\s*#(?!##[^#]).*)+)/;

  CODE = /^[-=]>/;

  MULTI_DENT = /^(?:\n[^\n\S]*)+/;

  JSTOKEN = /^`(?!``)((?:[^`\\]|\\[\s\S])*)`/;

  HERE_JSTOKEN = /^```((?:[^`\\]|\\[\s\S]|`(?!``))*)```/;

  // String-matching-regexes.
  STRING_START = /^(?:'''|"""|'|")/;

  STRING_SINGLE = /^(?:[^\\']|\\[\s\S])*/;

  STRING_DOUBLE = /^(?:[^\\"#]|\\[\s\S]|\#(?!\{))*/;

  HEREDOC_SINGLE = /^(?:[^\\']|\\[\s\S]|'(?!''))*/;

  HEREDOC_DOUBLE = /^(?:[^\\"#]|\\[\s\S]|"(?!"")|\#(?!\{))*/;

  INSIDE_JSX = /^(?:[^\{<])*/; // Start of CoffeeScript interpolation. // Similar to `HEREDOC_DOUBLE` but there is no escaping.
  // Maybe JSX tag (`<` not allowed even if bare).

  JSX_INTERPOLATION = /^(?:\{|<(?!\/))/; // CoffeeScript interpolation.
  // JSX opening tag.

  HEREDOC_INDENT = /\n+([^\n\S]*)(?=\S)/g;

  // Regex-matching-regexes.
  REGEX = /^\/(?!\/)((?:[^[\/\n\\]|\\[^\n]|\[(?:\\[^\n]|[^\]\n\\])*\])*)(\/)?/; // Every other thing.
  // Anything but newlines escaped.
  // Character class.

  REGEX_FLAGS = /^\w*/;

  VALID_FLAGS = /^(?!.*(.).*\1)[gimsuy]*$/;

  HEREGEX = /^(?:[^\\\/#\s]|\\[\s\S]|\/(?!\/\/)|\#(?!\{)|\s+(?:#(?!\{).*)?)*/; // Match any character, except those that need special handling below.
  // Match `\` followed by any character.
  // Match any `/` except `///`.
  // Match `#` which is not part of interpolation, e.g. `#{}`.
  // Comments consume everything until the end of the line, including `///`.

  HEREGEX_COMMENT = /(\s+)(#(?!{).*)/gm;

  REGEX_ILLEGAL = /^(\/|\/{3}\s*)(\*)/;

  POSSIBLY_DIVISION = /^\/=?\s/;

  // Other regexes.
  HERECOMMENT_ILLEGAL = /\*\//;

  LINE_CONTINUER = /^\s*(?:,|\??\.(?![.\d])|\??::)/;

  STRING_INVALID_ESCAPE = /((?:^|[^\\])(?:\\\\)*)\\(?:(0\d|[1-7])|(x(?![\da-fA-F]{2}).{0,2})|(u\{(?![\da-fA-F]{1,}\})[^}]*\}?)|(u(?!\{|[\da-fA-F]{4}).{0,4}))/; // Make sure the escape isn’t escaped.
  // octal escape
  // hex escape
  // unicode code point escape
  // unicode escape

  REGEX_INVALID_ESCAPE = /((?:^|[^\\])(?:\\\\)*)\\(?:(0\d)|(x(?![\da-fA-F]{2}).{0,2})|(u\{(?![\da-fA-F]{1,}\})[^}]*\}?)|(u(?!\{|[\da-fA-F]{4}).{0,4}))/; // Make sure the escape isn’t escaped.
  // octal escape
  // hex escape
  // unicode code point escape
  // unicode escape

  TRAILING_SPACES = /\s+$/;

  // Compound assignment tokens.
  COMPOUND_ASSIGN = ['-=', '+=', '/=', '*=', '%=', '||=', '&&=', '?=', '<<=', '>>=', '>>>=', '&=', '^=', '|=', '**=', '//=', '%%='];

  // Unary tokens.
  UNARY = ['NEW', 'TYPEOF', 'DELETE'];

  UNARY_MATH = ['!', '~'];

  // Bit-shifting tokens.
  SHIFT = ['<<', '>>', '>>>'];

  // Comparison tokens.
  COMPARE = ['==', '!=', '<', '>', '<=', '>='];

  // Mathematical tokens.
  MATH = ['*', '/', '%', '//', '%%'];

  // Relational tokens that are negatable with `not` prefix.
  RELATION = ['IN', 'OF', 'INSTANCEOF'];

  // Boolean tokens.
  BOOL = ['TRUE', 'FALSE'];

  // Tokens which could legitimately be invoked or indexed. An opening
  // parentheses or bracket following these tokens will be recorded as the start
  // of a function invocation or indexing operation.
  CALLABLE = ['IDENTIFIER', 'PROPERTY', ')', ']', '?', '@', 'THIS', 'SUPER', 'DYNAMIC_IMPORT'];

  INDEXABLE = CALLABLE.concat(['NUMBER', 'INFINITY', 'NAN', 'STRING', 'STRING_END', 'REGEX', 'REGEX_END', 'BOOL', 'NULL', 'UNDEFINED', '}', '::']);

  // Tokens which can be the left-hand side of a less-than comparison, i.e. `a<b`.
  COMPARABLE_LEFT_SIDE = ['IDENTIFIER', ')', ']', 'NUMBER'];

  // Tokens which a regular expression will never immediately follow (except spaced
  // CALLABLEs in some cases), but which a division operator can.

  // See: http://www-archive.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions
  NOT_REGEX = INDEXABLE.concat(['++', '--']);

  // Tokens that, when immediately preceding a `WHEN`, indicate that the `WHEN`
  // occurs at the start of a line. We disambiguate these from trailing whens to
  // avoid an ambiguity in the grammar.
  LINE_BREAK = ['INDENT', 'OUTDENT', 'TERMINATOR'];

  // Additional indent in front of these is ignored.
  INDENTABLE_CLOSERS = [')', '}', ']'];

}).call(this);