documenting and cleaning up the Rewriter
This commit is contained in:
Jeremy Ashkenas
parent
45d8cf163e
commit
a4f7a5e248
|
|
@ -72,7 +72,7 @@ table td {
|
|||
max-width: 500px;
|
||||
min-width: 500px;
|
||||
min-height: 5px;
|
||||
padding: 10px 30px 1px 50px;
|
||||
padding: 10px 25px 1px 50px;
|
||||
vertical-align: top;
|
||||
text-align: left;
|
||||
}
|
||||
|
|
@ -103,7 +103,7 @@ table td {
|
|||
opacity: 1;
|
||||
}
|
||||
td.code, th.code {
|
||||
padding: 14px 15px 16px 35px;
|
||||
padding: 14px 15px 16px 50px;
|
||||
width: 100%;
|
||||
vertical-align: top;
|
||||
background: #f5f5ff;
|
||||
|
|
|
|||
|
|
@ -10,61 +10,9 @@ form:</p>
|
|||
<span class="nv">Rewriter: </span><span class="nx">require</span><span class="p">(</span><span class="s1">'./rewriter'</span><span class="p">).</span><span class="nx">Rewriter</span>
|
||||
<span class="k">else</span>
|
||||
<span class="k">this</span><span class="p">.</span><span class="nv">exports: </span><span class="k">this</span>
|
||||
<span class="nv">Rewriter: </span><span class="k">this</span><span class="p">.</span><span class="nx">Rewriter</span></pre></div> </td> </tr> <tr id="section-3"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-3">#</a> </div> <h2>Constants</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-4"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-4">#</a> </div> <p>Keywords that CoffeeScript shares in common with JavaScript.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">JS_KEYWORDS: </span><span class="p">[</span>
|
||||
<span class="s2">"if"</span><span class="p">,</span> <span class="s2">"else"</span><span class="p">,</span>
|
||||
<span class="s2">"true"</span><span class="p">,</span> <span class="s2">"false"</span><span class="p">,</span>
|
||||
<span class="s2">"new"</span><span class="p">,</span> <span class="s2">"return"</span><span class="p">,</span>
|
||||
<span class="s2">"try"</span><span class="p">,</span> <span class="s2">"catch"</span><span class="p">,</span> <span class="s2">"finally"</span><span class="p">,</span> <span class="s2">"throw"</span><span class="p">,</span>
|
||||
<span class="s2">"break"</span><span class="p">,</span> <span class="s2">"continue"</span><span class="p">,</span>
|
||||
<span class="s2">"for"</span><span class="p">,</span> <span class="s2">"in"</span><span class="p">,</span> <span class="s2">"while"</span><span class="p">,</span>
|
||||
<span class="s2">"delete"</span><span class="p">,</span> <span class="s2">"instanceof"</span><span class="p">,</span> <span class="s2">"typeof"</span><span class="p">,</span>
|
||||
<span class="s2">"switch"</span><span class="p">,</span> <span class="s2">"super"</span><span class="p">,</span> <span class="s2">"extends"</span><span class="p">,</span> <span class="s2">"class"</span>
|
||||
<span class="p">]</span></pre></div> </td> </tr> <tr id="section-5"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-5">#</a> </div> <p>CoffeeScript-only keywords, which we're more relaxed about allowing. They can't
|
||||
be used standalone, but you can reference them as an attached property.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">COFFEE_KEYWORDS: </span><span class="p">[</span>
|
||||
<span class="s2">"then"</span><span class="p">,</span> <span class="s2">"unless"</span><span class="p">,</span>
|
||||
<span class="s2">"yes"</span><span class="p">,</span> <span class="s2">"no"</span><span class="p">,</span> <span class="s2">"on"</span><span class="p">,</span> <span class="s2">"off"</span><span class="p">,</span>
|
||||
<span class="s2">"and"</span><span class="p">,</span> <span class="s2">"or"</span><span class="p">,</span> <span class="s2">"is"</span><span class="p">,</span> <span class="s2">"isnt"</span><span class="p">,</span> <span class="s2">"not"</span><span class="p">,</span>
|
||||
<span class="s2">"of"</span><span class="p">,</span> <span class="s2">"by"</span><span class="p">,</span> <span class="s2">"where"</span><span class="p">,</span> <span class="s2">"when"</span>
|
||||
<span class="p">]</span></pre></div> </td> </tr> <tr id="section-6"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-6">#</a> </div> <p>The combined list of keywords is the superset that gets passed verbatim to
|
||||
the parser.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">KEYWORDS: </span><span class="nx">JS_KEYWORDS</span><span class="p">.</span><span class="nx">concat</span> <span class="nx">COFFEE_KEYWORDS</span></pre></div> </td> </tr> <tr id="section-7"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-7">#</a> </div> <p>The list of keywords that are reserved by JavaScript, but not used, or are
|
||||
used by CoffeeScript internally. We throw an error when these are encountered,
|
||||
to avoid having a JavaScript error at runtime.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">RESERVED: </span><span class="p">[</span>
|
||||
<span class="s2">"case"</span><span class="p">,</span> <span class="s2">"default"</span><span class="p">,</span> <span class="s2">"do"</span><span class="p">,</span> <span class="s2">"function"</span><span class="p">,</span> <span class="s2">"var"</span><span class="p">,</span> <span class="s2">"void"</span><span class="p">,</span> <span class="s2">"with"</span>
|
||||
<span class="s2">"const"</span><span class="p">,</span> <span class="s2">"let"</span><span class="p">,</span> <span class="s2">"debugger"</span><span class="p">,</span> <span class="s2">"enum"</span><span class="p">,</span> <span class="s2">"export"</span><span class="p">,</span> <span class="s2">"import"</span><span class="p">,</span> <span class="s2">"native"</span><span class="p">,</span>
|
||||
<span class="s2">"__extends"</span><span class="p">,</span> <span class="s2">"__hasProp"</span>
|
||||
<span class="p">]</span></pre></div> </td> </tr> <tr id="section-8"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-8">#</a> </div> <p>The superset of both JavaScript keywords and reserved words, none of which may
|
||||
be used as identifiers or properties.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">JS_FORBIDDEN: </span><span class="nx">JS_KEYWORDS</span><span class="p">.</span><span class="nx">concat</span> <span class="nx">RESERVED</span></pre></div> </td> </tr> <tr id="section-9"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-9">#</a> </div> <p>Token matching regexes.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nx">IDENTIFIER</span> <span class="o">:</span> <span class="sr">/^([a-zA-Z$_](\w|\$)*)/</span>
|
||||
<span class="nx">NUMBER</span> <span class="o">:</span> <span class="sr">/^(\b((0(x|X)[0-9a-fA-F]+)|([0-9]+(\.[0-9]+)?(e[+\-]?[0-9]+)?)))\b/i</span>
|
||||
<span class="nx">HEREDOC</span> <span class="o">:</span> <span class="sr">/^("{6}|'{6}|"{3}\n?([\s\S]*?)\n?([ \t]*)"{3}|'{3}\n?([\s\S]*?)\n?([ \t]*)'{3})/</span>
|
||||
<span class="nx">INTERPOLATION</span> <span class="o">:</span> <span class="sr">/^\$([a-zA-Z_@]\w*)/</span>
|
||||
<span class="nx">OPERATOR</span> <span class="o">:</span> <span class="sr">/^([+\*&|\/\-%=<>:!?]+)/</span>
|
||||
<span class="nx">WHITESPACE</span> <span class="o">:</span> <span class="sr">/^([ \t]+)/</span>
|
||||
<span class="nx">COMMENT</span> <span class="o">:</span> <span class="sr">/^(((\n?[ \t]*)?#[^\n]*)+)/</span>
|
||||
<span class="nx">CODE</span> <span class="o">:</span> <span class="sr">/^((-|=)>)/</span>
|
||||
<span class="nx">REGEX</span> <span class="o">:</span> <span class="sr">/^(\/(\S.*?)?([^\\]|\\\\)\/[imgy]{0,4})/</span>
|
||||
<span class="nx">MULTI_DENT</span> <span class="o">:</span> <span class="sr">/^((\n([ \t]*))+)(\.)?/</span>
|
||||
<span class="nx">LAST_DENTS</span> <span class="o">:</span> <span class="sr">/\n([ \t]*)/g</span>
|
||||
<span class="nx">LAST_DENT</span> <span class="o">:</span> <span class="sr">/\n([ \t]*)/</span>
|
||||
<span class="nx">ASSIGNMENT</span> <span class="o">:</span> <span class="sr">/^(:|=)$/</span></pre></div> </td> </tr> <tr id="section-10"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-10">#</a> </div> <p>Token cleaning regexes.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nx">JS_CLEANER</span> <span class="o">:</span> <span class="sr">/(^`|`$)/g</span>
|
||||
<span class="nx">MULTILINER</span> <span class="o">:</span> <span class="sr">/\n/g</span>
|
||||
<span class="nx">STRING_NEWLINES</span> <span class="o">:</span> <span class="sr">/\n[ \t]*/g</span>
|
||||
<span class="nx">COMMENT_CLEANER</span> <span class="o">:</span> <span class="sr">/(^[ \t]*#|\n[ \t]*$)/mg</span>
|
||||
<span class="nx">NO_NEWLINE</span> <span class="o">:</span> <span class="sr">/^([+\*&|\/\-%=<>:!.\\][<>=&|]*|and|or|is|isnt|not|delete|typeof|instanceof)$/</span>
|
||||
<span class="nx">HEREDOC_INDENT</span> <span class="o">:</span> <span class="sr">/^[ \t]+/mg</span></pre></div> </td> </tr> <tr id="section-11"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-11">#</a> </div> <p>Tokens which a regular expression will never immediately follow, but which
|
||||
a division operator might.</p>
|
||||
|
||||
<p>See: <a href='http://www.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions'>http://www.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions</a></p>
|
||||
|
||||
<p>Our list is shorter, due to sans-parentheses method calls.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">NOT_REGEX: </span><span class="p">[</span>
|
||||
<span class="s1">'NUMBER'</span><span class="p">,</span> <span class="s1">'REGEX'</span><span class="p">,</span> <span class="s1">'++'</span><span class="p">,</span> <span class="s1">'--'</span><span class="p">,</span> <span class="s1">'FALSE'</span><span class="p">,</span> <span class="s1">'NULL'</span><span class="p">,</span> <span class="s1">'TRUE'</span>
|
||||
<span class="p">]</span></pre></div> </td> </tr> <tr id="section-12"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-12">#</a> </div> <p>Tokens which could legitimately be invoked or indexed. A opening
|
||||
parentheses or bracket following these tokens will be recorded as the start
|
||||
of a function invocation or indexing operation.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">CALLABLE: </span><span class="p">[</span><span class="s1">'IDENTIFIER'</span><span class="p">,</span> <span class="s1">'SUPER'</span><span class="p">,</span> <span class="s1">')'</span><span class="p">,</span> <span class="s1">']'</span><span class="p">,</span> <span class="s1">'}'</span><span class="p">,</span> <span class="s1">'STRING'</span><span class="p">,</span> <span class="s1">'@'</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-13"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-13">#</a> </div> <p>Tokens that indicate an access -- keywords immediately following will be
|
||||
treated as identifiers.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">ACCESSORS: </span><span class="p">[</span><span class="s1">'PROPERTY_ACCESS'</span><span class="p">,</span> <span class="s1">'PROTOTYPE_ACCESS'</span><span class="p">,</span> <span class="s1">'SOAK_ACCESS'</span><span class="p">,</span> <span class="s1">'@'</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-14"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-14">#</a> </div> <p>Tokens that, when immediately preceding a <code>WHEN</code>, indicate that the <code>WHEN</code>
|
||||
occurs at the start of a line. We disambiguate these from trailing whens to
|
||||
avoid an ambiguity in the grammar.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">BEFORE_WHEN: </span><span class="p">[</span><span class="s1">'INDENT'</span><span class="p">,</span> <span class="s1">'OUTDENT'</span><span class="p">,</span> <span class="s1">'TERMINATOR'</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-15"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-15">#</a> </div> <h2>The Lexer Class</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-16"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-16">#</a> </div> <p>The Lexer class reads a stream of CoffeeScript and divvys it up into tagged
|
||||
tokens. A minor bit of the ambiguity in the grammar has been avoided by
|
||||
pushing some extra smarts into the Lexer.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">exports.Lexer: </span><span class="nx">class</span> <span class="nx">Lexer</span></pre></div> </td> </tr> <tr id="section-17"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-17">#</a> </div> <p><strong>tokenize</strong> is the Lexer's main method. Scan by attempting to match tokens
|
||||
<span class="nv">Rewriter: </span><span class="k">this</span><span class="p">.</span><span class="nx">Rewriter</span></pre></div> </td> </tr> <tr id="section-3"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-3">#</a> </div> <h2>The Lexer Class</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-4"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-4">#</a> </div> <p>The Lexer class reads a stream of CoffeeScript and divvys it up into tagged
|
||||
tokens. Some potential ambiguity in the grammar has been avoided by
|
||||
pushing some extra smarts into the Lexer.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">exports.Lexer: </span><span class="nx">class</span> <span class="nx">Lexer</span></pre></div> </td> </tr> <tr id="section-5"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-5">#</a> </div> <p><strong>tokenize</strong> is the Lexer's main method. Scan by attempting to match tokens
|
||||
one at a time, using a regular expression anchored at the start of the
|
||||
remaining code, or a custom recursive token-matching method
|
||||
(for interpolations). When the next token has been recorded, we move forward
|
||||
|
|
@ -72,7 +20,10 @@ within the code past the token, and begin again.</p>
|
|||
|
||||
<p>Each tokenizing method is responsible for incrementing <code>@i</code> by the number of
|
||||
characters it has consumed. <code>@i</code> can be thought of as our finger on the page
|
||||
of source.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">tokenize: </span><span class="p">(</span><span class="nx">code</span><span class="p">,</span> <span class="nx">options</span><span class="p">)</span> <span class="o">-></span>
|
||||
of source.</p>
|
||||
|
||||
<p>Before returning the token stream, run it through the <a href="rewriter.html">Rewriter</a>
|
||||
unless explicitly asked not to.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">tokenize: </span><span class="p">(</span><span class="nx">code</span><span class="p">,</span> <span class="nx">options</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="nx">o</span> <span class="o">:</span> <span class="nx">options</span> <span class="o">or</span> <span class="p">{}</span>
|
||||
<span class="err">@</span><span class="nx">code</span> <span class="o">:</span> <span class="nx">code</span> <span class="c1"># The remainder of the source code.</span>
|
||||
<span class="err">@</span><span class="nx">i</span> <span class="o">:</span> <span class="mi">0</span> <span class="c1"># Current character position we're parsing.</span>
|
||||
|
|
@ -85,7 +36,7 @@ of source.</p> </td> <td class="code"> <di
|
|||
<span class="err">@</span><span class="nx">extract_next_token</span><span class="p">()</span>
|
||||
<span class="err">@</span><span class="nx">close_indentation</span><span class="p">()</span>
|
||||
<span class="k">return</span> <span class="err">@</span><span class="nx">tokens</span> <span class="k">if</span> <span class="nx">o</span><span class="p">.</span><span class="nx">rewrite</span> <span class="o">is</span> <span class="kc">off</span>
|
||||
<span class="p">(</span><span class="k">new</span> <span class="nx">Rewriter</span><span class="p">()).</span><span class="nx">rewrite</span> <span class="err">@</span><span class="nx">tokens</span></pre></div> </td> </tr> <tr id="section-18"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-18">#</a> </div> <p>At every position, run through this list of attempted matches,
|
||||
<span class="p">(</span><span class="k">new</span> <span class="nx">Rewriter</span><span class="p">()).</span><span class="nx">rewrite</span> <span class="err">@</span><span class="nx">tokens</span></pre></div> </td> </tr> <tr id="section-6"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-6">#</a> </div> <p>At every position, run through this list of attempted matches,
|
||||
short-circuiting if any of them succeed. Their order determines precedence:
|
||||
<code>@literal_token</code> is the fallback catch-all.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">extract_next_token: </span><span class="o">-></span>
|
||||
<span class="k">return</span> <span class="k">if</span> <span class="err">@</span><span class="nx">identifier_token</span><span class="p">()</span>
|
||||
|
|
@ -97,7 +48,7 @@ short-circuiting if any of them succeed. Their order determines precedence:
|
|||
<span class="k">return</span> <span class="k">if</span> <span class="err">@</span><span class="nx">whitespace_token</span><span class="p">()</span>
|
||||
<span class="k">return</span> <span class="k">if</span> <span class="err">@</span><span class="nx">js_token</span><span class="p">()</span>
|
||||
<span class="k">return</span> <span class="k">if</span> <span class="err">@</span><span class="nx">string_token</span><span class="p">()</span>
|
||||
<span class="k">return</span> <span class="err">@</span><span class="nx">literal_token</span><span class="p">()</span></pre></div> </td> </tr> <tr id="section-19"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-19">#</a> </div> <h2>Tokenizers</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-20"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-20">#</a> </div> <p>Matches identifying literals: variables, keywords, method names, etc.
|
||||
<span class="k">return</span> <span class="err">@</span><span class="nx">literal_token</span><span class="p">()</span></pre></div> </td> </tr> <tr id="section-7"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-7">#</a> </div> <h2>Tokenizers</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-8"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-8">#</a> </div> <p>Matches identifying literals: variables, keywords, method names, etc.
|
||||
Check to ensure that JavaScript reserved words aren't being used as
|
||||
identifiers. Because CoffeeScript reserves a handful of keywords that are
|
||||
allowed in JavaScript, we're careful not to tag them as keywords when
|
||||
|
|
@ -112,11 +63,11 @@ though <code>is</code> means <code>===</code> otherwise.</p> </td>
|
|||
<span class="nv">tag: </span><span class="s1">'LEADING_WHEN'</span> <span class="k">if</span> <span class="nx">tag</span> <span class="o">is</span> <span class="s1">'WHEN'</span> <span class="o">and</span> <span class="nx">include</span> <span class="nx">BEFORE_WHEN</span><span class="p">,</span> <span class="err">@</span><span class="nx">tag</span><span class="p">()</span>
|
||||
<span class="err">@</span><span class="nx">token</span><span class="p">(</span><span class="nx">tag</span><span class="p">,</span> <span class="nx">id</span><span class="p">)</span>
|
||||
<span class="err">@</span><span class="nx">i</span> <span class="o">+=</span> <span class="nx">id</span><span class="p">.</span><span class="nx">length</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-21"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-21">#</a> </div> <p>Matches numbers, including decimals, hex, and exponential notation.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">number_token: </span><span class="o">-></span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-9"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-9">#</a> </div> <p>Matches numbers, including decimals, hex, and exponential notation.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">number_token: </span><span class="o">-></span>
|
||||
<span class="k">return</span> <span class="kc">false</span> <span class="nx">unless</span> <span class="nv">number: </span><span class="err">@</span><span class="nx">match</span> <span class="nx">NUMBER</span><span class="p">,</span> <span class="mi">1</span>
|
||||
<span class="err">@</span><span class="nx">token</span> <span class="s1">'NUMBER'</span><span class="p">,</span> <span class="nx">number</span>
|
||||
<span class="err">@</span><span class="nx">i</span> <span class="o">+=</span> <span class="nx">number</span><span class="p">.</span><span class="nx">length</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-22"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-22">#</a> </div> <p>Matches strings, including multi-line strings. Ensures that quotation marks
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-10"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-10">#</a> </div> <p>Matches strings, including multi-line strings. Ensures that quotation marks
|
||||
are balanced within the string's contents, and within nested interpolations.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">string_token: </span><span class="o">-></span>
|
||||
<span class="k">return</span> <span class="kc">false</span> <span class="nx">unless</span> <span class="nx">starts</span><span class="p">(</span><span class="err">@</span><span class="nx">chunk</span><span class="p">,</span> <span class="s1">'"'</span><span class="p">)</span> <span class="o">or</span> <span class="nx">starts</span><span class="p">(</span><span class="err">@</span><span class="nx">chunk</span><span class="p">,</span> <span class="s2">"'"</span><span class="p">)</span>
|
||||
<span class="nv">string: </span><span class="err">@</span><span class="nx">balanced_token</span> <span class="p">[</span><span class="s1">'"'</span><span class="p">,</span> <span class="s1">'"'</span><span class="p">],</span> <span class="p">[</span><span class="s1">'${'</span><span class="p">,</span> <span class="s1">'}'</span><span class="p">]</span>
|
||||
|
|
@ -125,28 +76,28 @@ are balanced within the string's contents, and within nested interpolations.</p>
|
|||
<span class="err">@</span><span class="nx">interpolate_string</span> <span class="nx">string</span><span class="p">.</span><span class="nx">replace</span> <span class="nx">STRING_NEWLINES</span><span class="p">,</span> <span class="s2">" \\\n"</span>
|
||||
<span class="err">@</span><span class="nx">line</span> <span class="o">+=</span> <span class="nx">count</span> <span class="nx">string</span><span class="p">,</span> <span class="s2">"\n"</span>
|
||||
<span class="err">@</span><span class="nx">i</span> <span class="o">+=</span> <span class="nx">string</span><span class="p">.</span><span class="nx">length</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-23"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-23">#</a> </div> <p>Matches heredocs, adjusting indentation to the correct level, as heredocs
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-11"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-11">#</a> </div> <p>Matches heredocs, adjusting indentation to the correct level, as heredocs
|
||||
preserve whitespace, but ignore indentation to the left.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">heredoc_token: </span><span class="o">-></span>
|
||||
<span class="k">return</span> <span class="kc">false</span> <span class="nx">unless</span> <span class="nx">match</span> <span class="o">=</span> <span class="err">@</span><span class="nx">chunk</span><span class="p">.</span><span class="nx">match</span><span class="p">(</span><span class="nx">HEREDOC</span><span class="p">)</span>
|
||||
<span class="nv">doc: </span><span class="err">@</span><span class="nx">sanitize_heredoc</span> <span class="nx">match</span><span class="p">[</span><span class="mi">2</span><span class="p">]</span> <span class="o">or</span> <span class="nx">match</span><span class="p">[</span><span class="mi">4</span><span class="p">]</span>
|
||||
<span class="err">@</span><span class="nx">token</span> <span class="s1">'STRING'</span><span class="p">,</span> <span class="s2">"\"$doc\""</span>
|
||||
<span class="err">@</span><span class="nx">line</span> <span class="o">+=</span> <span class="nx">count</span> <span class="nx">match</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="s2">"\n"</span>
|
||||
<span class="err">@</span><span class="nx">i</span> <span class="o">+=</span> <span class="nx">match</span><span class="p">[</span><span class="mi">1</span><span class="p">].</span><span class="nx">length</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-24"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-24">#</a> </div> <p>Matches JavaScript interpolated directly into the source via backticks.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">js_token: </span><span class="o">-></span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-12"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-12">#</a> </div> <p>Matches JavaScript interpolated directly into the source via backticks.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">js_token: </span><span class="o">-></span>
|
||||
<span class="k">return</span> <span class="kc">false</span> <span class="nx">unless</span> <span class="nx">starts</span> <span class="err">@</span><span class="nx">chunk</span><span class="p">,</span> <span class="s1">'`'</span>
|
||||
<span class="k">return</span> <span class="kc">false</span> <span class="nx">unless</span> <span class="nv">script: </span><span class="err">@</span><span class="nx">balanced_token</span> <span class="p">[</span><span class="s1">'`'</span><span class="p">,</span> <span class="s1">'`'</span><span class="p">]</span>
|
||||
<span class="err">@</span><span class="nx">token</span> <span class="s1">'JS'</span><span class="p">,</span> <span class="nx">script</span><span class="p">.</span><span class="nx">replace</span><span class="p">(</span><span class="nx">JS_CLEANER</span><span class="p">,</span> <span class="s1">''</span><span class="p">)</span>
|
||||
<span class="err">@</span><span class="nx">i</span> <span class="o">+=</span> <span class="nx">script</span><span class="p">.</span><span class="nx">length</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-25"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-25">#</a> </div> <p>Matches regular expression literals. Lexing regular expressions is difficult
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-13"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-13">#</a> </div> <p>Matches regular expression literals. Lexing regular expressions is difficult
|
||||
to distinguish from division, so we borrow some basic heuristics from
|
||||
JavaScript and Ruby.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">regex_token: </span><span class="o">-></span>
|
||||
<span class="k">return</span> <span class="kc">false</span> <span class="nx">unless</span> <span class="nv">regex: </span><span class="err">@</span><span class="nx">match</span> <span class="nx">REGEX</span><span class="p">,</span> <span class="mi">1</span>
|
||||
<span class="k">return</span> <span class="kc">false</span> <span class="k">if</span> <span class="nx">include</span> <span class="nx">NOT_REGEX</span><span class="p">,</span> <span class="err">@</span><span class="nx">tag</span><span class="p">()</span>
|
||||
<span class="err">@</span><span class="nx">token</span> <span class="s1">'REGEX'</span><span class="p">,</span> <span class="nx">regex</span>
|
||||
<span class="err">@</span><span class="nx">i</span> <span class="o">+=</span> <span class="nx">regex</span><span class="p">.</span><span class="nx">length</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-26"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-26">#</a> </div> <p>Matches a token in which which the passed delimiter pairs must be correctly
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-14"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-14">#</a> </div> <p>Matches a token in which which the passed delimiter pairs must be correctly
|
||||
balanced (ie. strings, JS literals).</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">balanced_token: </span><span class="p">(</span><span class="nx">delimited</span><span class="p">...)</span> <span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">balanced_string</span> <span class="err">@</span><span class="nx">chunk</span><span class="p">,</span> <span class="nx">delimited</span><span class="p">...</span></pre></div> </td> </tr> <tr id="section-27"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-27">#</a> </div> <p>Matches and conumes comments. We pass through comments into JavaScript,
|
||||
<span class="err">@</span><span class="nx">balanced_string</span> <span class="err">@</span><span class="nx">chunk</span><span class="p">,</span> <span class="nx">delimited</span><span class="p">...</span></pre></div> </td> </tr> <tr id="section-15"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-15">#</a> </div> <p>Matches and conumes comments. We pass through comments into JavaScript,
|
||||
so they're treated as real tokens, like any other part of the language.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">comment_token: </span><span class="o">-></span>
|
||||
<span class="k">return</span> <span class="kc">false</span> <span class="nx">unless</span> <span class="nv">comment: </span><span class="err">@</span><span class="nx">match</span> <span class="nx">COMMENT</span><span class="p">,</span> <span class="mi">1</span>
|
||||
<span class="err">@</span><span class="nx">line</span> <span class="o">+=</span> <span class="p">(</span><span class="nx">comment</span><span class="p">.</span><span class="nx">match</span><span class="p">(</span><span class="nx">MULTILINER</span><span class="p">)</span> <span class="o">or</span> <span class="p">[]).</span><span class="nx">length</span>
|
||||
|
|
@ -154,7 +105,7 @@ so they're treated as real tokens, like any other part of the language.</p>
|
|||
<span class="err">@</span><span class="nx">token</span> <span class="s1">'COMMENT'</span><span class="p">,</span> <span class="nx">compact</span> <span class="nx">lines</span>
|
||||
<span class="err">@</span><span class="nx">token</span> <span class="s1">'TERMINATOR'</span><span class="p">,</span> <span class="s2">"\n"</span>
|
||||
<span class="err">@</span><span class="nx">i</span> <span class="o">+=</span> <span class="nx">comment</span><span class="p">.</span><span class="nx">length</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-28"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-28">#</a> </div> <p>Matches newlines, indents, and outdents, and determines which is which.
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-16"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-16">#</a> </div> <p>Matches newlines, indents, and outdents, and determines which is which.
|
||||
If we can detect that the current line is continued onto the the next line,
|
||||
then the newline is suppressed:</p>
|
||||
|
||||
|
|
@ -184,25 +135,25 @@ can close multiple indents, so we need to know how far in we happen to be.</p>
|
|||
<span class="k">else</span>
|
||||
<span class="err">@</span><span class="nx">outdent_token</span> <span class="err">@</span><span class="nx">indent</span> <span class="o">-</span> <span class="nx">size</span><span class="p">,</span> <span class="nx">no_newlines</span>
|
||||
<span class="err">@</span><span class="nv">indent: </span><span class="nx">size</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-29"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-29">#</a> </div> <p>Record an outdent token or multiple tokens, if we happen to be moving back
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-17"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-17">#</a> </div> <p>Record an outdent token or multiple tokens, if we happen to be moving back
|
||||
inwards past several recorded indents.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">outdent_token: </span><span class="p">(</span><span class="nx">move_out</span><span class="p">,</span> <span class="nx">no_newlines</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="k">while</span> <span class="nx">move_out</span> <span class="o">></span> <span class="mi">0</span> <span class="o">and</span> <span class="err">@</span><span class="nx">indents</span><span class="p">.</span><span class="nx">length</span>
|
||||
<span class="nv">last_indent: </span><span class="err">@</span><span class="nx">indents</span><span class="p">.</span><span class="nx">pop</span><span class="p">()</span>
|
||||
<span class="err">@</span><span class="nx">token</span> <span class="s1">'OUTDENT'</span><span class="p">,</span> <span class="nx">last_indent</span>
|
||||
<span class="nx">move_out</span> <span class="o">-=</span> <span class="nx">last_indent</span>
|
||||
<span class="err">@</span><span class="nx">token</span> <span class="s1">'TERMINATOR'</span><span class="p">,</span> <span class="s2">"\n"</span> <span class="nx">unless</span> <span class="err">@</span><span class="nx">tag</span><span class="p">()</span> <span class="o">is</span> <span class="s1">'TERMINATOR'</span> <span class="o">or</span> <span class="nx">no_newlines</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-30"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-30">#</a> </div> <p>Matches and consumes non-meaningful whitespace. Tag the previous token
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-18"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-18">#</a> </div> <p>Matches and consumes non-meaningful whitespace. Tag the previous token
|
||||
as being "spaced", because there are some cases where it makes a difference.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">whitespace_token: </span><span class="o">-></span>
|
||||
<span class="k">return</span> <span class="kc">false</span> <span class="nx">unless</span> <span class="nv">space: </span><span class="err">@</span><span class="nx">match</span> <span class="nx">WHITESPACE</span><span class="p">,</span> <span class="mi">1</span>
|
||||
<span class="nv">prev: </span><span class="err">@</span><span class="nx">prev</span><span class="p">()</span>
|
||||
<span class="nv">prev.spaced: </span><span class="kc">true</span> <span class="k">if</span> <span class="nx">prev</span>
|
||||
<span class="err">@</span><span class="nx">i</span> <span class="o">+=</span> <span class="nx">space</span><span class="p">.</span><span class="nx">length</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-31"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-31">#</a> </div> <p>Generate a newline token. Consecutive newlines get merged together.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">newline_token: </span><span class="p">(</span><span class="nx">newlines</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-19"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-19">#</a> </div> <p>Generate a newline token. Consecutive newlines get merged together.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">newline_token: </span><span class="p">(</span><span class="nx">newlines</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">token</span> <span class="s1">'TERMINATOR'</span><span class="p">,</span> <span class="s2">"\n"</span> <span class="nx">unless</span> <span class="err">@</span><span class="nx">tag</span><span class="p">()</span> <span class="o">is</span> <span class="s1">'TERMINATOR'</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-32"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-32">#</a> </div> <p>Use a <code>\</code> at a line-ending to suppress the newline.
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-20"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-20">#</a> </div> <p>Use a <code>\</code> at a line-ending to suppress the newline.
|
||||
The slash is removed here once its job is done.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">suppress_newlines: </span><span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">pop</span><span class="p">()</span> <span class="k">if</span> <span class="err">@</span><span class="nx">value</span><span class="p">()</span> <span class="o">is</span> <span class="s2">"\\"</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-33"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-33">#</a> </div> <p>We treat all other single characters as a token. Eg.: <code>( ) , . !</code>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-21"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-21">#</a> </div> <p>We treat all other single characters as a token. Eg.: <code>( ) , . !</code>
|
||||
Multi-character operators are also literal tokens, so that Jison can assign
|
||||
the proper order of operations. There are some symbols that we tag specially
|
||||
here. <code>;</code> and newlines are both treated as a <code>TERMINATOR</code>, we distinguish
|
||||
|
|
@ -230,7 +181,7 @@ parentheses that indicate a method call from regular parentheses, and so on.</p>
|
|||
<span class="nv">tag: </span><span class="s1">'INDEX_START'</span> <span class="k">if</span> <span class="nx">value</span> <span class="o">is</span> <span class="s1">'['</span>
|
||||
<span class="err">@</span><span class="nx">token</span> <span class="nx">tag</span><span class="p">,</span> <span class="nx">value</span>
|
||||
<span class="err">@</span><span class="nx">i</span> <span class="o">+=</span> <span class="nx">value</span><span class="p">.</span><span class="nx">length</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-34"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-34">#</a> </div> <h2>Token Manipulators</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-35"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-35">#</a> </div> <p>As we consume a new <code>IDENTIFIER</code>, look at the previous token to determine
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-22"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-22">#</a> </div> <h2>Token Manipulators</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-23"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-23">#</a> </div> <p>As we consume a new <code>IDENTIFIER</code>, look at the previous token to determine
|
||||
if it's a special kind of accessor.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">name_access_type: </span><span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">tag</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="s1">'PROTOTYPE_ACCESS'</span><span class="p">)</span> <span class="k">if</span> <span class="err">@</span><span class="nx">value</span><span class="p">()</span> <span class="o">is</span> <span class="s1">'::'</span>
|
||||
<span class="k">if</span> <span class="err">@</span><span class="nx">value</span><span class="p">()</span> <span class="o">is</span> <span class="s1">'.'</span> <span class="o">and</span> <span class="o">not</span> <span class="p">(</span><span class="err">@</span><span class="nx">value</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span> <span class="o">is</span> <span class="s1">'.'</span><span class="p">)</span>
|
||||
|
|
@ -238,12 +189,12 @@ if it's a special kind of accessor.</p> </td> <td class=
|
|||
<span class="err">@</span><span class="nx">tag</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="s1">'SOAK_ACCESS'</span><span class="p">)</span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">splice</span><span class="p">(</span><span class="o">-</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="k">else</span>
|
||||
<span class="err">@</span><span class="nx">tag</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'PROPERTY_ACCESS'</span></pre></div> </td> </tr> <tr id="section-36"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-36">#</a> </div> <p>Sanitize a heredoc by escaping internal double quotes and erasing all
|
||||
<span class="err">@</span><span class="nx">tag</span> <span class="mi">1</span><span class="p">,</span> <span class="s1">'PROPERTY_ACCESS'</span></pre></div> </td> </tr> <tr id="section-24"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-24">#</a> </div> <p>Sanitize a heredoc by escaping internal double quotes and erasing all
|
||||
external indentation on the left-hand side.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">sanitize_heredoc: </span><span class="p">(</span><span class="nx">doc</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="nv">indent: </span><span class="p">(</span><span class="nx">doc</span><span class="p">.</span><span class="nx">match</span><span class="p">(</span><span class="nx">HEREDOC_INDENT</span><span class="p">)</span> <span class="o">or</span> <span class="p">[</span><span class="s1">''</span><span class="p">]).</span><span class="nx">sort</span><span class="p">()[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="nx">doc</span><span class="p">.</span><span class="nx">replace</span><span class="p">(</span><span class="k">new</span> <span class="nb">RegExp</span><span class="p">(</span><span class="s2">"^"</span> <span class="o">+</span><span class="nx">indent</span><span class="p">,</span> <span class="s1">'gm'</span><span class="p">),</span> <span class="s1">''</span><span class="p">)</span>
|
||||
<span class="p">.</span><span class="nx">replace</span><span class="p">(</span><span class="nx">MULTILINER</span><span class="p">,</span> <span class="s2">"\\n"</span><span class="p">)</span>
|
||||
<span class="p">.</span><span class="nx">replace</span><span class="p">(</span><span class="sr">/"/g</span><span class="p">,</span> <span class="s1">'\\"'</span><span class="p">)</span></pre></div> </td> </tr> <tr id="section-37"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-37">#</a> </div> <p>A source of ambiguity in our grammar used to be parameter lists in function
|
||||
<span class="p">.</span><span class="nx">replace</span><span class="p">(</span><span class="sr">/"/g</span><span class="p">,</span> <span class="s1">'\\"'</span><span class="p">)</span></pre></div> </td> </tr> <tr id="section-25"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-25">#</a> </div> <p>A source of ambiguity in our grammar used to be parameter lists in function
|
||||
definitions versus argument lists in function calls. Walk backwards, tagging
|
||||
parameters specially in order to make things easier for the parser.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">tag_parameters: </span><span class="o">-></span>
|
||||
<span class="k">return</span> <span class="k">if</span> <span class="err">@</span><span class="nx">tag</span><span class="p">()</span> <span class="o">isnt</span> <span class="s1">')'</span>
|
||||
|
|
@ -256,12 +207,12 @@ parameters specially in order to make things easier for the parser.</p>
|
|||
<span class="k">when</span> <span class="s1">'IDENTIFIER'</span> <span class="k">then</span> <span class="nx">tok</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">:</span> <span class="s1">'PARAM'</span>
|
||||
<span class="k">when</span> <span class="s1">')'</span> <span class="k">then</span> <span class="nx">tok</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">:</span> <span class="s1">'PARAM_END'</span>
|
||||
<span class="k">when</span> <span class="s1">'('</span> <span class="k">then</span> <span class="k">return</span> <span class="nx">tok</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">:</span> <span class="s1">'PARAM_START'</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-38"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-38">#</a> </div> <p>Close up all remaining open blocks at the end of the file.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">close_indentation: </span><span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">outdent_token</span><span class="p">(</span><span class="err">@</span><span class="nx">indent</span><span class="p">)</span></pre></div> </td> </tr> <tr id="section-39"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-39">#</a> </div> <p>The error for when you try to use a forbidden word in JavaScript as
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-26"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-26">#</a> </div> <p>Close up all remaining open blocks at the end of the file.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">close_indentation: </span><span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">outdent_token</span><span class="p">(</span><span class="err">@</span><span class="nx">indent</span><span class="p">)</span></pre></div> </td> </tr> <tr id="section-27"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-27">#</a> </div> <p>The error for when you try to use a forbidden word in JavaScript as
|
||||
an identifier.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">identifier_error: </span><span class="p">(</span><span class="nx">word</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="k">throw</span> <span class="k">new</span> <span class="nb">Error</span> <span class="s2">"SyntaxError: Reserved word \"$word\" on line ${@line + 1}"</span></pre></div> </td> </tr> <tr id="section-40"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-40">#</a> </div> <p>The error for when you try to assign to a reserved word in JavaScript,
|
||||
<span class="k">throw</span> <span class="k">new</span> <span class="nb">Error</span> <span class="s2">"SyntaxError: Reserved word \"$word\" on line ${@line + 1}"</span></pre></div> </td> </tr> <tr id="section-28"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-28">#</a> </div> <p>The error for when you try to assign to a reserved word in JavaScript,
|
||||
like "function" or "default".</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">assignment_error: </span><span class="o">-></span>
|
||||
<span class="k">throw</span> <span class="k">new</span> <span class="nb">Error</span> <span class="s2">"SyntaxError: Reserved word \"${@value()}\" on line ${@line + 1} can't be assigned"</span></pre></div> </td> </tr> <tr id="section-41"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-41">#</a> </div> <p>Matches a balanced group such as a single or double-quoted string. Pass in
|
||||
<span class="k">throw</span> <span class="k">new</span> <span class="nb">Error</span> <span class="s2">"SyntaxError: Reserved word \"${@value()}\" on line ${@line + 1} can't be assigned"</span></pre></div> </td> </tr> <tr id="section-29"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-29">#</a> </div> <p>Matches a balanced group such as a single or double-quoted string. Pass in
|
||||
a series of delimiters, all of which must be nested correctly within the
|
||||
contents of the string. This method allows us to have strings within
|
||||
interpolations within strings etc...</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">balanced_string: </span><span class="p">(</span><span class="nx">str</span><span class="p">,</span> <span class="nx">delimited</span><span class="p">...)</span> <span class="o">-></span>
|
||||
|
|
@ -286,7 +237,7 @@ interpolations within strings etc...</p> </td> <td class
|
|||
<span class="nx">i</span> <span class="o">+=</span> <span class="mi">1</span>
|
||||
<span class="k">throw</span> <span class="k">new</span> <span class="nb">Error</span> <span class="s2">"SyntaxError: Unterminated ${levels.pop()[0]} starting on line ${@line + 1}"</span> <span class="k">if</span> <span class="nx">levels</span><span class="p">.</span><span class="nx">length</span>
|
||||
<span class="k">return</span> <span class="kc">false</span> <span class="k">if</span> <span class="nx">i</span> <span class="o">is</span> <span class="mi">0</span>
|
||||
<span class="k">return</span> <span class="nx">str</span><span class="p">.</span><span class="nx">substring</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nx">i</span><span class="p">)</span></pre></div> </td> </tr> <tr id="section-42"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-42">#</a> </div> <p>Expand variables and expressions inside double-quoted strings using
|
||||
<span class="k">return</span> <span class="nx">str</span><span class="p">.</span><span class="nx">substring</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="nx">i</span><span class="p">)</span></pre></div> </td> </tr> <tr id="section-30"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-30">#</a> </div> <p>Expand variables and expressions inside double-quoted strings using
|
||||
<a href="http://wiki.ecmascript.org/doku.php?id=strawman:string_interpolation">ECMA Harmony's interpolation syntax</a>
|
||||
for substitution of bare variables as well as arbitrary expressions.</p>
|
||||
|
||||
|
|
@ -329,18 +280,70 @@ token stream.</p> </td> <td class="code">
|
|||
<span class="err">@</span><span class="nv">tokens: </span><span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">concat</span> <span class="nx">each</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">else</span>
|
||||
<span class="err">@</span><span class="nx">token</span> <span class="nx">each</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nx">each</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="err">@</span><span class="nx">token</span> <span class="s1">'+'</span><span class="p">,</span> <span class="s1">'+'</span> <span class="k">if</span> <span class="nx">i</span> <span class="o"><</span> <span class="nx">tokens</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="mi">1</span></pre></div> </td> </tr> <tr id="section-43"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-43">#</a> </div> <h2>Helpers</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-44"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-44">#</a> </div> <p>Add a token to the results, taking note of the line number.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">token: </span><span class="p">(</span><span class="nx">tag</span><span class="p">,</span> <span class="nx">value</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">push</span><span class="p">([</span><span class="nx">tag</span><span class="p">,</span> <span class="nx">value</span><span class="p">,</span> <span class="err">@</span><span class="nx">line</span><span class="p">])</span></pre></div> </td> </tr> <tr id="section-45"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-45">#</a> </div> <p>Peek at a tag in the current token stream.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">tag: </span><span class="p">(</span><span class="nx">index</span><span class="p">,</span> <span class="nx">tag</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">token</span> <span class="s1">'+'</span><span class="p">,</span> <span class="s1">'+'</span> <span class="k">if</span> <span class="nx">i</span> <span class="o"><</span> <span class="nx">tokens</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="mi">1</span></pre></div> </td> </tr> <tr id="section-31"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-31">#</a> </div> <h2>Helpers</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-32"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-32">#</a> </div> <p>Add a token to the results, taking note of the line number.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">token: </span><span class="p">(</span><span class="nx">tag</span><span class="p">,</span> <span class="nx">value</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">push</span><span class="p">([</span><span class="nx">tag</span><span class="p">,</span> <span class="nx">value</span><span class="p">,</span> <span class="err">@</span><span class="nx">line</span><span class="p">])</span></pre></div> </td> </tr> <tr id="section-33"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-33">#</a> </div> <p>Peek at a tag in the current token stream.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">tag: </span><span class="p">(</span><span class="nx">index</span><span class="p">,</span> <span class="nx">tag</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="k">return</span> <span class="nx">unless</span> <span class="nv">tok: </span><span class="err">@</span><span class="nx">prev</span><span class="p">(</span><span class="nx">index</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="nx">tok</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">:</span> <span class="nx">tag</span> <span class="k">if</span> <span class="nx">tag</span><span class="o">?</span>
|
||||
<span class="nx">tok</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-46"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-46">#</a> </div> <p>Peek at a value in the current token stream.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">value: </span><span class="p">(</span><span class="nx">index</span><span class="p">,</span> <span class="nx">val</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="nx">tok</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-34"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-34">#</a> </div> <p>Peek at a value in the current token stream.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">value: </span><span class="p">(</span><span class="nx">index</span><span class="p">,</span> <span class="nx">val</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="k">return</span> <span class="nx">unless</span> <span class="nv">tok: </span><span class="err">@</span><span class="nx">prev</span><span class="p">(</span><span class="nx">index</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="nx">tok</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="o">:</span> <span class="nx">val</span> <span class="k">if</span> <span class="nx">val</span><span class="o">?</span>
|
||||
<span class="nx">tok</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-47"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-47">#</a> </div> <p>Peek at a previous token, entire.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">prev: </span><span class="p">(</span><span class="nx">index</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">[</span><span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="p">(</span><span class="nx">index</span> <span class="o">or</span> <span class="mi">1</span><span class="p">)]</span></pre></div> </td> </tr> <tr id="section-48"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-48">#</a> </div> <p>Attempt to match a string against the current chunk, returning the indexed
|
||||
<span class="nx">tok</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-35"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-35">#</a> </div> <p>Peek at a previous token, entire.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">prev: </span><span class="p">(</span><span class="nx">index</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">[</span><span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="p">(</span><span class="nx">index</span> <span class="o">or</span> <span class="mi">1</span><span class="p">)]</span></pre></div> </td> </tr> <tr id="section-36"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-36">#</a> </div> <p>Attempt to match a string against the current chunk, returning the indexed
|
||||
match if successful, and <code>false</code> otherwise.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">match: </span><span class="p">(</span><span class="nx">regex</span><span class="p">,</span> <span class="nx">index</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="k">return</span> <span class="kc">false</span> <span class="nx">unless</span> <span class="nv">m: </span><span class="err">@</span><span class="nx">chunk</span><span class="p">.</span><span class="nx">match</span><span class="p">(</span><span class="nx">regex</span><span class="p">)</span>
|
||||
<span class="k">if</span> <span class="nx">m</span> <span class="k">then</span> <span class="nx">m</span><span class="p">[</span><span class="nx">index</span><span class="p">]</span> <span class="k">else</span> <span class="kc">false</span></pre></div> </td> </tr> <tr id="section-49"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-49">#</a> </div> <h2>Utility Functions</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-50"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-50">#</a> </div> <p>Does a list include a value?</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">include: </span><span class="p">(</span><span class="nx">list</span><span class="p">,</span> <span class="nx">value</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="k">if</span> <span class="nx">m</span> <span class="k">then</span> <span class="nx">m</span><span class="p">[</span><span class="nx">index</span><span class="p">]</span> <span class="k">else</span> <span class="kc">false</span></pre></div> </td> </tr> <tr id="section-37"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-37">#</a> </div> <h2>Constants</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-38"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-38">#</a> </div> <p>Keywords that CoffeeScript shares in common with JavaScript.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">JS_KEYWORDS: </span><span class="p">[</span>
|
||||
<span class="s2">"if"</span><span class="p">,</span> <span class="s2">"else"</span><span class="p">,</span>
|
||||
<span class="s2">"true"</span><span class="p">,</span> <span class="s2">"false"</span><span class="p">,</span>
|
||||
<span class="s2">"new"</span><span class="p">,</span> <span class="s2">"return"</span><span class="p">,</span>
|
||||
<span class="s2">"try"</span><span class="p">,</span> <span class="s2">"catch"</span><span class="p">,</span> <span class="s2">"finally"</span><span class="p">,</span> <span class="s2">"throw"</span><span class="p">,</span>
|
||||
<span class="s2">"break"</span><span class="p">,</span> <span class="s2">"continue"</span><span class="p">,</span>
|
||||
<span class="s2">"for"</span><span class="p">,</span> <span class="s2">"in"</span><span class="p">,</span> <span class="s2">"while"</span><span class="p">,</span>
|
||||
<span class="s2">"delete"</span><span class="p">,</span> <span class="s2">"instanceof"</span><span class="p">,</span> <span class="s2">"typeof"</span><span class="p">,</span>
|
||||
<span class="s2">"switch"</span><span class="p">,</span> <span class="s2">"super"</span><span class="p">,</span> <span class="s2">"extends"</span><span class="p">,</span> <span class="s2">"class"</span>
|
||||
<span class="p">]</span></pre></div> </td> </tr> <tr id="section-39"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-39">#</a> </div> <p>CoffeeScript-only keywords, which we're more relaxed about allowing. They can't
|
||||
be used standalone, but you can reference them as an attached property.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">COFFEE_KEYWORDS: </span><span class="p">[</span>
|
||||
<span class="s2">"then"</span><span class="p">,</span> <span class="s2">"unless"</span><span class="p">,</span>
|
||||
<span class="s2">"yes"</span><span class="p">,</span> <span class="s2">"no"</span><span class="p">,</span> <span class="s2">"on"</span><span class="p">,</span> <span class="s2">"off"</span><span class="p">,</span>
|
||||
<span class="s2">"and"</span><span class="p">,</span> <span class="s2">"or"</span><span class="p">,</span> <span class="s2">"is"</span><span class="p">,</span> <span class="s2">"isnt"</span><span class="p">,</span> <span class="s2">"not"</span><span class="p">,</span>
|
||||
<span class="s2">"of"</span><span class="p">,</span> <span class="s2">"by"</span><span class="p">,</span> <span class="s2">"where"</span><span class="p">,</span> <span class="s2">"when"</span>
|
||||
<span class="p">]</span></pre></div> </td> </tr> <tr id="section-40"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-40">#</a> </div> <p>The combined list of keywords is the superset that gets passed verbatim to
|
||||
the parser.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">KEYWORDS: </span><span class="nx">JS_KEYWORDS</span><span class="p">.</span><span class="nx">concat</span> <span class="nx">COFFEE_KEYWORDS</span></pre></div> </td> </tr> <tr id="section-41"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-41">#</a> </div> <p>The list of keywords that are reserved by JavaScript, but not used, or are
|
||||
used by CoffeeScript internally. We throw an error when these are encountered,
|
||||
to avoid having a JavaScript error at runtime.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">RESERVED: </span><span class="p">[</span>
|
||||
<span class="s2">"case"</span><span class="p">,</span> <span class="s2">"default"</span><span class="p">,</span> <span class="s2">"do"</span><span class="p">,</span> <span class="s2">"function"</span><span class="p">,</span> <span class="s2">"var"</span><span class="p">,</span> <span class="s2">"void"</span><span class="p">,</span> <span class="s2">"with"</span>
|
||||
<span class="s2">"const"</span><span class="p">,</span> <span class="s2">"let"</span><span class="p">,</span> <span class="s2">"debugger"</span><span class="p">,</span> <span class="s2">"enum"</span><span class="p">,</span> <span class="s2">"export"</span><span class="p">,</span> <span class="s2">"import"</span><span class="p">,</span> <span class="s2">"native"</span><span class="p">,</span>
|
||||
<span class="s2">"__extends"</span><span class="p">,</span> <span class="s2">"__hasProp"</span>
|
||||
<span class="p">]</span></pre></div> </td> </tr> <tr id="section-42"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-42">#</a> </div> <p>The superset of both JavaScript keywords and reserved words, none of which may
|
||||
be used as identifiers or properties.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">JS_FORBIDDEN: </span><span class="nx">JS_KEYWORDS</span><span class="p">.</span><span class="nx">concat</span> <span class="nx">RESERVED</span></pre></div> </td> </tr> <tr id="section-43"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-43">#</a> </div> <p>Token matching regexes.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nx">IDENTIFIER</span> <span class="o">:</span> <span class="sr">/^([a-zA-Z$_](\w|\$)*)/</span>
|
||||
<span class="nx">NUMBER</span> <span class="o">:</span> <span class="sr">/^(\b((0(x|X)[0-9a-fA-F]+)|([0-9]+(\.[0-9]+)?(e[+\-]?[0-9]+)?)))\b/i</span>
|
||||
<span class="nx">HEREDOC</span> <span class="o">:</span> <span class="sr">/^("{6}|'{6}|"{3}\n?([\s\S]*?)\n?([ \t]*)"{3}|'{3}\n?([\s\S]*?)\n?([ \t]*)'{3})/</span>
|
||||
<span class="nx">INTERPOLATION</span> <span class="o">:</span> <span class="sr">/^\$([a-zA-Z_@]\w*)/</span>
|
||||
<span class="nx">OPERATOR</span> <span class="o">:</span> <span class="sr">/^([+\*&|\/\-%=<>:!?]+)/</span>
|
||||
<span class="nx">WHITESPACE</span> <span class="o">:</span> <span class="sr">/^([ \t]+)/</span>
|
||||
<span class="nx">COMMENT</span> <span class="o">:</span> <span class="sr">/^(((\n?[ \t]*)?#[^\n]*)+)/</span>
|
||||
<span class="nx">CODE</span> <span class="o">:</span> <span class="sr">/^((-|=)>)/</span>
|
||||
<span class="nx">REGEX</span> <span class="o">:</span> <span class="sr">/^(\/(\S.*?)?([^\\]|\\\\)\/[imgy]{0,4})/</span>
|
||||
<span class="nx">MULTI_DENT</span> <span class="o">:</span> <span class="sr">/^((\n([ \t]*))+)(\.)?/</span>
|
||||
<span class="nx">LAST_DENTS</span> <span class="o">:</span> <span class="sr">/\n([ \t]*)/g</span>
|
||||
<span class="nx">LAST_DENT</span> <span class="o">:</span> <span class="sr">/\n([ \t]*)/</span>
|
||||
<span class="nx">ASSIGNMENT</span> <span class="o">:</span> <span class="sr">/^(:|=)$/</span></pre></div> </td> </tr> <tr id="section-44"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-44">#</a> </div> <p>Token cleaning regexes.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nx">JS_CLEANER</span> <span class="o">:</span> <span class="sr">/(^`|`$)/g</span>
|
||||
<span class="nx">MULTILINER</span> <span class="o">:</span> <span class="sr">/\n/g</span>
|
||||
<span class="nx">STRING_NEWLINES</span> <span class="o">:</span> <span class="sr">/\n[ \t]*/g</span>
|
||||
<span class="nx">COMMENT_CLEANER</span> <span class="o">:</span> <span class="sr">/(^[ \t]*#|\n[ \t]*$)/mg</span>
|
||||
<span class="nx">NO_NEWLINE</span> <span class="o">:</span> <span class="sr">/^([+\*&|\/\-%=<>:!.\\][<>=&|]*|and|or|is|isnt|not|delete|typeof|instanceof)$/</span>
|
||||
<span class="nx">HEREDOC_INDENT</span> <span class="o">:</span> <span class="sr">/^[ \t]+/mg</span></pre></div> </td> </tr> <tr id="section-45"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-45">#</a> </div> <p>Tokens which a regular expression will never immediately follow, but which
|
||||
a division operator might.</p>
|
||||
|
||||
<p>See: <a href='http://www.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions'>http://www.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions</a></p>
|
||||
|
||||
<p>Our list is shorter, due to sans-parentheses method calls.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">NOT_REGEX: </span><span class="p">[</span>
|
||||
<span class="s1">'NUMBER'</span><span class="p">,</span> <span class="s1">'REGEX'</span><span class="p">,</span> <span class="s1">'++'</span><span class="p">,</span> <span class="s1">'--'</span><span class="p">,</span> <span class="s1">'FALSE'</span><span class="p">,</span> <span class="s1">'NULL'</span><span class="p">,</span> <span class="s1">'TRUE'</span>
|
||||
<span class="p">]</span></pre></div> </td> </tr> <tr id="section-46"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-46">#</a> </div> <p>Tokens which could legitimately be invoked or indexed. A opening
|
||||
parentheses or bracket following these tokens will be recorded as the start
|
||||
of a function invocation or indexing operation.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">CALLABLE: </span><span class="p">[</span><span class="s1">'IDENTIFIER'</span><span class="p">,</span> <span class="s1">'SUPER'</span><span class="p">,</span> <span class="s1">')'</span><span class="p">,</span> <span class="s1">']'</span><span class="p">,</span> <span class="s1">'}'</span><span class="p">,</span> <span class="s1">'STRING'</span><span class="p">,</span> <span class="s1">'@'</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-47"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-47">#</a> </div> <p>Tokens that indicate an access -- keywords immediately following will be
|
||||
treated as identifiers.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">ACCESSORS: </span><span class="p">[</span><span class="s1">'PROPERTY_ACCESS'</span><span class="p">,</span> <span class="s1">'PROTOTYPE_ACCESS'</span><span class="p">,</span> <span class="s1">'SOAK_ACCESS'</span><span class="p">,</span> <span class="s1">'@'</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-48"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-48">#</a> </div> <p>Tokens that, when immediately preceding a <code>WHEN</code>, indicate that the <code>WHEN</code>
|
||||
occurs at the start of a line. We disambiguate these from trailing whens to
|
||||
avoid an ambiguity in the grammar.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">BEFORE_WHEN: </span><span class="p">[</span><span class="s1">'INDENT'</span><span class="p">,</span> <span class="s1">'OUTDENT'</span><span class="p">,</span> <span class="s1">'TERMINATOR'</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-49"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-49">#</a> </div> <h2>Utility Functions</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-50"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-50">#</a> </div> <p>Does a list include a value?</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">include: </span><span class="p">(</span><span class="nx">list</span><span class="p">,</span> <span class="nx">value</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="nx">list</span><span class="p">.</span><span class="nx">indexOf</span><span class="p">(</span><span class="nx">value</span><span class="p">)</span> <span class="o">>=</span> <span class="mi">0</span></pre></div> </td> </tr> <tr id="section-51"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-51">#</a> </div> <p>Peek at the beginning of a given string to see if it matches a sequence.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">starts: </span><span class="p">(</span><span class="nx">string</span><span class="p">,</span> <span class="nx">literal</span><span class="p">,</span> <span class="nx">start</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="nx">string</span><span class="p">.</span><span class="nx">substring</span><span class="p">(</span><span class="nx">start</span><span class="p">,</span> <span class="p">(</span><span class="nx">start</span> <span class="o">or</span> <span class="mi">0</span><span class="p">)</span> <span class="o">+</span> <span class="nx">literal</span><span class="p">.</span><span class="nx">length</span><span class="p">)</span> <span class="o">is</span> <span class="nx">literal</span></pre></div> </td> </tr> <tr id="section-52"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-52">#</a> </div> <p>Trim out all falsy values from an array.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">compact: </span><span class="p">(</span><span class="nx">array</span><span class="p">)</span> <span class="o">-></span> <span class="nx">item</span> <span class="k">for</span> <span class="nx">item</span> <span class="k">in</span> <span class="nx">array</span> <span class="k">when</span> <span class="nx">item</span></pre></div> </td> </tr> <tr id="section-53"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-53">#</a> </div> <p>Count the number of occurences of a character in a string.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">count: </span><span class="p">(</span><span class="nx">string</span><span class="p">,</span> <span class="nx">letter</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="nv">num: </span><span class="mi">0</span>
|
||||
|
|
|
|||
|
|
@ -1,42 +1,36 @@
|
|||
<!DOCTYPE html> <html> <head> <title>rewriter.coffee</title> <meta http-equiv="content-type" content="text/html; charset=UTF-8"> <link rel="stylesheet" media="all" href="docco.css" /> </head> <body> <div id="container"> <div id="background"></div> <div id="jump_to"> Jump To … <div id="jump_wrapper"> <div id="jump_page"> <a class="source" href="cake.html"> cake.coffee </a> <a class="source" href="coffee-script.html"> coffee-script.coffee </a> <a class="source" href="command.html"> command.coffee </a> <a class="source" href="grammar.html"> grammar.coffee </a> <a class="source" href="lexer.html"> lexer.coffee </a> <a class="source" href="nodes.html"> nodes.coffee </a> <a class="source" href="optparse.html"> optparse.coffee </a> <a class="source" href="repl.html"> repl.coffee </a> <a class="source" href="rewriter.html"> rewriter.coffee </a> <a class="source" href="scope.html"> scope.coffee </a> </div> </div> </div> <table cellpadding="0" cellspacing="0"> <thead> <tr> <th class="docs"> <h1> rewriter.coffee </h1> </th> <th class="code"> </th> </tr> </thead> <tbody> <tr id="section-1"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-1">#</a> </div> </td> <td class="code"> <div class="highlight"><pre><span class="k">this</span><span class="p">.</span><span class="nv">exports: </span><span class="k">this</span> <span class="nx">unless</span> <span class="nx">process</span><span class="o">?</span></pre></div> </td> </tr> <tr id="section-2"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-2">#</a> </div> <p>Tokens that must be balanced.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">BALANCED_PAIRS: </span><span class="p">[[</span><span class="s1">'('</span><span class="p">,</span> <span class="s1">')'</span><span class="p">],</span> <span class="p">[</span><span class="s1">'['</span><span class="p">,</span> <span class="s1">']'</span><span class="p">],</span> <span class="p">[</span><span class="s1">'{'</span><span class="p">,</span> <span class="s1">'}'</span><span class="p">],</span> <span class="p">[</span><span class="s1">'INDENT'</span><span class="p">,</span> <span class="s1">'OUTDENT'</span><span class="p">],</span>
|
||||
<span class="p">[</span><span class="s1">'PARAM_START'</span><span class="p">,</span> <span class="s1">'PARAM_END'</span><span class="p">],</span> <span class="p">[</span><span class="s1">'CALL_START'</span><span class="p">,</span> <span class="s1">'CALL_END'</span><span class="p">],</span>
|
||||
<span class="p">[</span><span class="s1">'INDEX_START'</span><span class="p">,</span> <span class="s1">'INDEX_END'</span><span class="p">],</span> <span class="p">[</span><span class="s1">'SOAKED_INDEX_START'</span><span class="p">,</span> <span class="s1">'SOAKED_INDEX_END'</span><span class="p">]]</span></pre></div> </td> </tr> <tr id="section-3"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-3">#</a> </div> <p>Tokens that signal the start of a balanced pair.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">EXPRESSION_START: </span><span class="nx">pair</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="nx">pair</span> <span class="k">in</span> <span class="nx">BALANCED_PAIRS</span></pre></div> </td> </tr> <tr id="section-4"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-4">#</a> </div> <p>Tokens that signal the end of a balanced pair.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">EXPRESSION_TAIL: </span><span class="nx">pair</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="nx">pair</span> <span class="k">in</span> <span class="nx">BALANCED_PAIRS</span></pre></div> </td> </tr> <tr id="section-5"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-5">#</a> </div> <p>Tokens that indicate the close of a clause of an expression.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">EXPRESSION_CLOSE: </span><span class="p">[</span><span class="s1">'CATCH'</span><span class="p">,</span> <span class="s1">'WHEN'</span><span class="p">,</span> <span class="s1">'ELSE'</span><span class="p">,</span> <span class="s1">'FINALLY'</span><span class="p">].</span><span class="nx">concat</span><span class="p">(</span><span class="nx">EXPRESSION_TAIL</span><span class="p">)</span></pre></div> </td> </tr> <tr id="section-6"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-6">#</a> </div> <p>Tokens pairs that, in immediate succession, indicate an implicit call.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">IMPLICIT_FUNC: </span><span class="p">[</span><span class="s1">'IDENTIFIER'</span><span class="p">,</span> <span class="s1">'SUPER'</span><span class="p">,</span> <span class="s1">')'</span><span class="p">,</span> <span class="s1">'CALL_END'</span><span class="p">,</span> <span class="s1">']'</span><span class="p">,</span> <span class="s1">'INDEX_END'</span><span class="p">]</span>
|
||||
<span class="nx">IMPLICIT_BLOCK</span><span class="o">:</span><span class="p">[</span><span class="s1">'->'</span><span class="p">,</span> <span class="s1">'=>'</span><span class="p">,</span> <span class="s1">'{'</span><span class="p">,</span> <span class="s1">'['</span><span class="p">,</span> <span class="s1">','</span><span class="p">]</span>
|
||||
<span class="nv">IMPLICIT_END: </span> <span class="p">[</span><span class="s1">'IF'</span><span class="p">,</span> <span class="s1">'UNLESS'</span><span class="p">,</span> <span class="s1">'FOR'</span><span class="p">,</span> <span class="s1">'WHILE'</span><span class="p">,</span> <span class="s1">'TERMINATOR'</span><span class="p">,</span> <span class="s1">'INDENT'</span><span class="p">,</span> <span class="s1">'OUTDENT'</span><span class="p">]</span>
|
||||
<span class="nv">IMPLICIT_CALL: </span><span class="p">[</span><span class="s1">'IDENTIFIER'</span><span class="p">,</span> <span class="s1">'NUMBER'</span><span class="p">,</span> <span class="s1">'STRING'</span><span class="p">,</span> <span class="s1">'JS'</span><span class="p">,</span> <span class="s1">'REGEX'</span><span class="p">,</span> <span class="s1">'NEW'</span><span class="p">,</span> <span class="s1">'PARAM_START'</span><span class="p">,</span>
|
||||
<span class="s1">'TRY'</span><span class="p">,</span> <span class="s1">'DELETE'</span><span class="p">,</span> <span class="s1">'TYPEOF'</span><span class="p">,</span> <span class="s1">'SWITCH'</span><span class="p">,</span>
|
||||
<span class="s1">'TRUE'</span><span class="p">,</span> <span class="s1">'FALSE'</span><span class="p">,</span> <span class="s1">'YES'</span><span class="p">,</span> <span class="s1">'NO'</span><span class="p">,</span> <span class="s1">'ON'</span><span class="p">,</span> <span class="s1">'OFF'</span><span class="p">,</span> <span class="s1">'!'</span><span class="p">,</span> <span class="s1">'!!'</span><span class="p">,</span> <span class="s1">'NOT'</span><span class="p">,</span>
|
||||
<span class="s1">'@'</span><span class="p">,</span> <span class="s1">'->'</span><span class="p">,</span> <span class="s1">'=>'</span><span class="p">,</span> <span class="s1">'['</span><span class="p">,</span> <span class="s1">'('</span><span class="p">,</span> <span class="s1">'{'</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-7"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-7">#</a> </div> <p>The inverse mappings of token pairs we're trying to fix up.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">INVERSES: </span><span class="p">{}</span>
|
||||
<span class="k">for</span> <span class="nx">pair</span> <span class="k">in</span> <span class="nx">BALANCED_PAIRS</span>
|
||||
<span class="nx">INVERSES</span><span class="p">[</span><span class="nx">pair</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span><span class="o">:</span> <span class="nx">pair</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="nx">INVERSES</span><span class="p">[</span><span class="nx">pair</span><span class="p">[</span><span class="mi">1</span><span class="p">]]</span><span class="o">:</span> <span class="nx">pair</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-8"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-8">#</a> </div> <p>Single-line flavors of block expressions that have unclosed endings.
|
||||
The grammar can't disambiguate them, so we insert the implicit indentation.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">SINGLE_LINERS: </span><span class="p">[</span><span class="s1">'ELSE'</span><span class="p">,</span> <span class="s2">"->"</span><span class="p">,</span> <span class="s2">"=>"</span><span class="p">,</span> <span class="s1">'TRY'</span><span class="p">,</span> <span class="s1">'FINALLY'</span><span class="p">,</span> <span class="s1">'THEN'</span><span class="p">]</span>
|
||||
<span class="nv">SINGLE_CLOSERS: </span><span class="p">[</span><span class="s1">'TERMINATOR'</span><span class="p">,</span> <span class="s1">'CATCH'</span><span class="p">,</span> <span class="s1">'FINALLY'</span><span class="p">,</span> <span class="s1">'ELSE'</span><span class="p">,</span> <span class="s1">'OUTDENT'</span><span class="p">,</span> <span class="s1">'LEADING_WHEN'</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-9"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-9">#</a> </div> <p>In order to keep the grammar simple, the stream of tokens that the Lexer
|
||||
emits is rewritten by the Rewriter, smoothing out ambiguities, mis-nested
|
||||
indentation, and single-line flavors of expressions.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">exports.Rewriter: </span><span class="nx">class</span> <span class="nx">Rewriter</span></pre></div> </td> </tr> <tr id="section-10"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-10">#</a> </div> <p>Rewrite the token stream in multiple passes, one logical filter at
|
||||
<!DOCTYPE html> <html> <head> <title>rewriter.coffee</title> <meta http-equiv="content-type" content="text/html; charset=UTF-8"> <link rel="stylesheet" media="all" href="docco.css" /> </head> <body> <div id="container"> <div id="background"></div> <div id="jump_to"> Jump To … <div id="jump_wrapper"> <div id="jump_page"> <a class="source" href="cake.html"> cake.coffee </a> <a class="source" href="coffee-script.html"> coffee-script.coffee </a> <a class="source" href="command.html"> command.coffee </a> <a class="source" href="grammar.html"> grammar.coffee </a> <a class="source" href="lexer.html"> lexer.coffee </a> <a class="source" href="nodes.html"> nodes.coffee </a> <a class="source" href="optparse.html"> optparse.coffee </a> <a class="source" href="repl.html"> repl.coffee </a> <a class="source" href="rewriter.html"> rewriter.coffee </a> <a class="source" href="scope.html"> scope.coffee </a> </div> </div> </div> <table cellpadding="0" cellspacing="0"> <thead> <tr> <th class="docs"> <h1> rewriter.coffee </h1> </th> <th class="code"> </th> </tr> </thead> <tbody> <tr id="section-1"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-1">#</a> </div> <p>The CoffeeScript language has a decent amount of optional syntax,
|
||||
implicit syntax, and shorthand syntax. These things can greatly complicate a
|
||||
grammar and bloat the resulting parse table. Instead of making the parser
|
||||
handle it all, we take a series of passes over the token stream,
|
||||
using this <strong>Rewriter</strong> to convert shorthand into the unambiguous long form,
|
||||
add implicit indentation and parentheses, balance incorrect nestings, and
|
||||
generally clean things up.</p> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-2"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-2">#</a> </div> <p>Set up exported variables for both Node.js and the browser.</p> </td> <td class="code"> <div class="highlight"><pre><span class="k">this</span><span class="p">.</span><span class="nv">exports: </span><span class="k">this</span> <span class="nx">unless</span> <span class="nx">process</span><span class="o">?</span></pre></div> </td> </tr> <tr id="section-3"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-3">#</a> </div> <p>The <strong>Rewriter</strong> class is used by the <a href="lexer.html">Lexer</a>, directly against
|
||||
its internal array of tokens.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">exports.Rewriter: </span><span class="nx">class</span> <span class="nx">Rewriter</span></pre></div> </td> </tr> <tr id="section-4"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-4">#</a> </div> <p>Rewrite the token stream in multiple passes, one logical filter at
|
||||
a time. This could certainly be changed into a single pass through the
|
||||
stream, with a big ol' efficient switch, but it's much nicer like this.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">rewrite: </span><span class="p">(</span><span class="nx">tokens</span><span class="p">)</span> <span class="o">-></span>
|
||||
stream, with a big ol' efficient switch, but it's much nicer to work with
|
||||
like this. The order of these passes matters -- indentation must be
|
||||
corrected before implicit parentheses can be wrapped around blocks of code.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">rewrite: </span><span class="p">(</span><span class="nx">tokens</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="err">@</span><span class="nv">tokens: </span><span class="nx">tokens</span>
|
||||
<span class="err">@</span><span class="nx">adjust_comments</span><span class="p">()</span>
|
||||
<span class="err">@</span><span class="nx">remove_leading_newlines</span><span class="p">()</span>
|
||||
<span class="err">@</span><span class="nx">remove_mid_expression_newlines</span><span class="p">()</span>
|
||||
<span class="err">@</span><span class="nx">move_commas_outside_outdents</span><span class="p">()</span>
|
||||
<span class="err">@</span><span class="nx">close_open_calls_and_indexes</span><span class="p">()</span>
|
||||
<span class="err">@</span><span class="nx">add_implicit_indentation</span><span class="p">()</span>
|
||||
<span class="err">@</span><span class="nx">add_implicit_parentheses</span><span class="p">()</span>
|
||||
<span class="err">@</span><span class="nx">ensure_balance</span><span class="p">(</span><span class="nx">BALANCED_PAIRS</span><span class="p">)</span>
|
||||
<span class="err">@</span><span class="nx">rewrite_closing_parens</span><span class="p">()</span>
|
||||
<span class="err">@</span><span class="nx">tokens</span></pre></div> </td> </tr> <tr id="section-11"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-11">#</a> </div> <p>Rewrite the token stream, looking one token ahead and behind.
|
||||
<span class="err">@</span><span class="nx">tokens</span></pre></div> </td> </tr> <tr id="section-5"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-5">#</a> </div> <p>Rewrite the token stream, looking one token ahead and behind.
|
||||
Allow the return value of the block to tell us how many tokens to move
|
||||
forwards (or backwards) in the stream, to make sure we don't miss anything
|
||||
as the stream changes length under our feet.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">scan_tokens: </span><span class="p">(</span><span class="nx">block</span><span class="p">)</span> <span class="o">-></span>
|
||||
as tokens are inserted and removed, and the stream changes length under
|
||||
our feet.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">scan_tokens: </span><span class="p">(</span><span class="nx">block</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="nv">i: </span><span class="mi">0</span>
|
||||
<span class="k">while</span> <span class="kc">true</span>
|
||||
<span class="k">break</span> <span class="nx">unless</span> <span class="err">@</span><span class="nx">tokens</span><span class="p">[</span><span class="nx">i</span><span class="p">]</span>
|
||||
<span class="nv">move: </span><span class="nx">block</span><span class="p">(</span><span class="err">@</span><span class="nx">tokens</span><span class="p">[</span><span class="nx">i</span> <span class="o">-</span> <span class="mi">1</span><span class="p">],</span> <span class="err">@</span><span class="nx">tokens</span><span class="p">[</span><span class="nx">i</span><span class="p">],</span> <span class="err">@</span><span class="nx">tokens</span><span class="p">[</span><span class="nx">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">],</span> <span class="nx">i</span><span class="p">)</span>
|
||||
<span class="nx">i</span> <span class="o">+=</span> <span class="nx">move</span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-12"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-12">#</a> </div> <p>Massage newlines and indentations so that comments don't have to be
|
||||
correctly indented, or appear on their own line.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">adjust_comments: </span><span class="o">-></span>
|
||||
<span class="kc">true</span></pre></div> </td> </tr> <tr id="section-6"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-6">#</a> </div> <p>Massage newlines and indentations so that comments don't have to be
|
||||
correctly indented, or appear on a line of their own.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">adjust_comments: </span><span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">scan_tokens</span> <span class="p">(</span><span class="nx">prev</span><span class="p">,</span> <span class="nx">token</span><span class="p">,</span> <span class="nx">post</span><span class="p">,</span> <span class="nx">i</span><span class="p">)</span> <span class="o">=></span>
|
||||
<span class="k">return</span> <span class="mi">1</span> <span class="nx">unless</span> <span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">'COMMENT'</span>
|
||||
<span class="nv">after: </span> <span class="err">@</span><span class="nx">tokens</span><span class="p">[</span><span class="nx">i</span> <span class="o">+</span> <span class="mi">2</span><span class="p">]</span>
|
||||
|
|
@ -48,19 +42,16 @@ correctly indented, or appear on their own line.</p> </td>
|
|||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">splice</span><span class="p">(</span><span class="nx">i</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="s1">'TERMINATOR'</span><span class="p">,</span> <span class="s2">"\n"</span><span class="p">,</span> <span class="nx">prev</span><span class="p">[</span><span class="mi">2</span><span class="p">]])</span>
|
||||
<span class="k">return</span> <span class="mi">2</span>
|
||||
<span class="k">else</span>
|
||||
<span class="k">return</span> <span class="mi">1</span></pre></div> </td> </tr> <tr id="section-13"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-13">#</a> </div> <p>Leading newlines would introduce an ambiguity in the grammar, so we
|
||||
<span class="k">return</span> <span class="mi">1</span></pre></div> </td> </tr> <tr id="section-7"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-7">#</a> </div> <p>Leading newlines would introduce an ambiguity in the grammar, so we
|
||||
dispatch them here.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">remove_leading_newlines: </span><span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">shift</span><span class="p">()</span> <span class="k">if</span> <span class="err">@</span><span class="nx">tokens</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">'TERMINATOR'</span></pre></div> </td> </tr> <tr id="section-14"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-14">#</a> </div> <p>Some blocks occur in the middle of expressions -- when we're expecting
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">shift</span><span class="p">()</span> <span class="k">while</span> <span class="err">@</span><span class="nx">tokens</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">'TERMINATOR'</span></pre></div> </td> </tr> <tr id="section-8"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-8">#</a> </div> <p>Some blocks occur in the middle of expressions -- when we're expecting
|
||||
this, remove their trailing newlines.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">remove_mid_expression_newlines: </span><span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">scan_tokens</span> <span class="p">(</span><span class="nx">prev</span><span class="p">,</span> <span class="nx">token</span><span class="p">,</span> <span class="nx">post</span><span class="p">,</span> <span class="nx">i</span><span class="p">)</span> <span class="o">=></span>
|
||||
<span class="k">return</span> <span class="mi">1</span> <span class="nx">unless</span> <span class="nx">post</span> <span class="o">and</span> <span class="nx">EXPRESSION_CLOSE</span><span class="p">.</span><span class="nx">indexOf</span><span class="p">(</span><span class="nx">post</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">>=</span> <span class="mi">0</span> <span class="o">and</span> <span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">'TERMINATOR'</span>
|
||||
<span class="k">return</span> <span class="mi">1</span> <span class="nx">unless</span> <span class="nx">post</span> <span class="o">and</span> <span class="nx">include</span><span class="p">(</span><span class="nx">EXPRESSION_CLOSE</span><span class="p">,</span> <span class="nx">post</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">and</span> <span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">'TERMINATOR'</span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">splice</span><span class="p">(</span><span class="nx">i</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="mi">0</span></pre></div> </td> </tr> <tr id="section-15"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-15">#</a> </div> <p>Make sure that we don't accidentally break trailing commas, which need
|
||||
to go on the outside of expression closers.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">move_commas_outside_outdents: </span><span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">scan_tokens</span> <span class="p">(</span><span class="nx">prev</span><span class="p">,</span> <span class="nx">token</span><span class="p">,</span> <span class="nx">post</span><span class="p">,</span> <span class="nx">i</span><span class="p">)</span> <span class="o">=></span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">splice</span><span class="p">(</span><span class="nx">i</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="nx">token</span><span class="p">)</span> <span class="k">if</span> <span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">'OUTDENT'</span> <span class="o">and</span> <span class="nx">prev</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">','</span>
|
||||
<span class="k">return</span> <span class="mi">1</span></pre></div> </td> </tr> <tr id="section-16"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-16">#</a> </div> <p>We've tagged the opening parenthesis of a method call, and the opening
|
||||
bracket of an indexing operation. Match them with their close.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">close_open_calls_and_indexes: </span><span class="o">-></span>
|
||||
<span class="k">return</span> <span class="mi">0</span></pre></div> </td> </tr> <tr id="section-9"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-9">#</a> </div> <p>The lexer has tagged the opening parenthesis of a method call, and the
|
||||
opening bracket of an indexing operation. Match them with their paired
|
||||
close.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">close_open_calls_and_indexes: </span><span class="o">-></span>
|
||||
<span class="nv">parens: </span> <span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="nv">brackets: </span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="err">@</span><span class="nx">scan_tokens</span> <span class="p">(</span><span class="nx">prev</span><span class="p">,</span> <span class="nx">token</span><span class="p">,</span> <span class="nx">post</span><span class="p">,</span> <span class="nx">i</span><span class="p">)</span> <span class="o">=></span>
|
||||
|
|
@ -81,7 +72,7 @@ bracket of an indexing operation. Match them with their close.</p> <
|
|||
<span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span><span class="o">:</span> <span class="s1">'INDEX_END'</span>
|
||||
<span class="k">else</span>
|
||||
<span class="nx">brackets</span><span class="p">[</span><span class="nx">brackets</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="mi">1</span><span class="p">]</span> <span class="o">-=</span> <span class="mi">1</span>
|
||||
<span class="k">return</span> <span class="mi">1</span></pre></div> </td> </tr> <tr id="section-17"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-17">#</a> </div> <p>Methods may be optionally called without parentheses, for simple cases.
|
||||
<span class="k">return</span> <span class="mi">1</span></pre></div> </td> </tr> <tr id="section-10"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-10">#</a> </div> <p>Methods may be optionally called without parentheses, for simple cases.
|
||||
Insert the implicit parentheses here, so that the parser doesn't have to
|
||||
deal with them.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">add_implicit_parentheses: </span><span class="o">-></span>
|
||||
<span class="nv">stack: </span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
|
|
@ -91,25 +82,26 @@ deal with them.</p> </td> <td class="code">
|
|||
<span class="k">if</span> <span class="nx">tag</span> <span class="o">is</span> <span class="s1">'OUTDENT'</span>
|
||||
<span class="nv">last: </span><span class="nx">stack</span><span class="p">.</span><span class="nx">pop</span><span class="p">()</span>
|
||||
<span class="nx">stack</span><span class="p">[</span><span class="nx">stack</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="mi">1</span><span class="p">]</span> <span class="o">+=</span> <span class="nx">last</span>
|
||||
<span class="k">if</span> <span class="nx">IMPLICIT_END</span><span class="p">.</span><span class="nx">indexOf</span><span class="p">(</span><span class="nx">tag</span><span class="p">)</span> <span class="o">>=</span> <span class="mi">0</span> <span class="o">or</span> <span class="o">!</span><span class="nx">post</span><span class="o">?</span>
|
||||
<span class="k">return</span> <span class="mi">1</span> <span class="k">if</span> <span class="nx">tag</span> <span class="o">is</span> <span class="s1">'INDENT'</span> <span class="o">and</span> <span class="nx">prev</span> <span class="o">and</span> <span class="nx">IMPLICIT_BLOCK</span><span class="p">.</span><span class="nx">indexOf</span><span class="p">(</span><span class="nx">prev</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">>=</span> <span class="mi">0</span>
|
||||
<span class="k">if</span> <span class="o">!</span><span class="nx">post</span><span class="o">?</span> <span class="o">or</span> <span class="nx">include</span> <span class="nx">IMPLICIT_END</span><span class="p">,</span> <span class="nx">tag</span>
|
||||
<span class="k">return</span> <span class="mi">1</span> <span class="k">if</span> <span class="nx">tag</span> <span class="o">is</span> <span class="s1">'INDENT'</span> <span class="o">and</span> <span class="nx">prev</span> <span class="o">and</span> <span class="nx">include</span> <span class="nx">IMPLICIT_BLOCK</span><span class="p">,</span> <span class="nx">prev</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="nx">stack</span><span class="p">[</span><span class="nx">stack</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="mi">1</span><span class="p">]</span> <span class="o">></span> <span class="mi">0</span> <span class="o">or</span> <span class="nx">tag</span> <span class="o">is</span> <span class="s1">'INDENT'</span>
|
||||
<span class="nv">idx: </span><span class="k">if</span> <span class="nx">tag</span> <span class="o">is</span> <span class="s1">'OUTDENT'</span> <span class="k">then</span> <span class="nx">i</span> <span class="o">+</span><span class="mi">1</span> <span class="k">else</span> <span class="nx">i</span>
|
||||
<span class="nv">idx: </span><span class="k">if</span> <span class="nx">tag</span> <span class="o">is</span> <span class="s1">'OUTDENT'</span> <span class="k">then</span> <span class="nx">i</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">else</span> <span class="nx">i</span>
|
||||
<span class="nv">stack_pointer: </span><span class="k">if</span> <span class="nx">tag</span> <span class="o">is</span> <span class="s1">'INDENT'</span> <span class="k">then</span> <span class="mi">2</span> <span class="k">else</span> <span class="mi">1</span>
|
||||
<span class="k">for</span> <span class="nx">tmp</span> <span class="k">in</span> <span class="p">[</span><span class="mi">0</span><span class="p">...</span><span class="nx">stack</span><span class="p">[</span><span class="nx">stack</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="nx">stack_pointer</span><span class="p">]]</span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">splice</span><span class="p">(</span><span class="nx">idx</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="s1">'CALL_END'</span><span class="p">,</span> <span class="s1">')'</span><span class="p">,</span> <span class="nx">token</span><span class="p">[</span><span class="mi">2</span><span class="p">]])</span>
|
||||
<span class="nv">size: </span><span class="nx">stack</span><span class="p">[</span><span class="nx">stack</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="nx">stack_pointer</span><span class="p">]</span> <span class="o">+</span> <span class="mi">1</span>
|
||||
<span class="nx">stack</span><span class="p">[</span><span class="nx">stack</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="nx">stack_pointer</span><span class="p">]</span><span class="o">:</span> <span class="mi">0</span>
|
||||
<span class="k">return</span> <span class="nx">size</span>
|
||||
<span class="k">return</span> <span class="mi">1</span> <span class="nx">unless</span> <span class="nx">prev</span> <span class="o">and</span> <span class="nx">IMPLICIT_FUNC</span><span class="p">.</span><span class="nx">indexOf</span><span class="p">(</span><span class="nx">prev</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">>=</span> <span class="mi">0</span> <span class="o">and</span> <span class="nx">IMPLICIT_CALL</span><span class="p">.</span><span class="nx">indexOf</span><span class="p">(</span><span class="nx">tag</span><span class="p">)</span> <span class="o">>=</span> <span class="mi">0</span>
|
||||
<span class="k">return</span> <span class="mi">1</span> <span class="nx">unless</span> <span class="nx">prev</span> <span class="o">and</span> <span class="nx">include</span><span class="p">(</span><span class="nx">IMPLICIT_FUNC</span><span class="p">,</span> <span class="nx">prev</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">and</span> <span class="nx">include</span> <span class="nx">IMPLICIT_CALL</span><span class="p">,</span> <span class="nx">tag</span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">splice</span><span class="p">(</span><span class="nx">i</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="s1">'CALL_START'</span><span class="p">,</span> <span class="s1">'('</span><span class="p">,</span> <span class="nx">token</span><span class="p">[</span><span class="mi">2</span><span class="p">]])</span>
|
||||
<span class="nx">stack</span><span class="p">[</span><span class="nx">stack</span><span class="p">.</span><span class="nx">length</span> <span class="o">-</span> <span class="mi">1</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
|
||||
<span class="k">return</span> <span class="mi">2</span></pre></div> </td> </tr> <tr id="section-18"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-18">#</a> </div> <p>Because our grammar is LALR(1), it can't handle some single-line
|
||||
expressions that lack ending delimiters. Use the lexer to add the implicit
|
||||
blocks, so it doesn't need to.
|
||||
')' can close a single-line block, but we need to make sure it's balanced.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">add_implicit_indentation: </span><span class="o">-></span>
|
||||
<span class="k">return</span> <span class="mi">2</span></pre></div> </td> </tr> <tr id="section-11"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-11">#</a> </div> <p>Because our grammar is LALR(1), it can't handle some single-line
|
||||
expressions that lack ending delimiters. The <strong>Rewriter</strong> adds the implicit
|
||||
blocks, so it doesn't need to. ')' can close a single-line block,
|
||||
but we need to make sure it's balanced.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">add_implicit_indentation: </span><span class="o">-></span>
|
||||
<span class="err">@</span><span class="nx">scan_tokens</span> <span class="p">(</span><span class="nx">prev</span><span class="p">,</span> <span class="nx">token</span><span class="p">,</span> <span class="nx">post</span><span class="p">,</span> <span class="nx">i</span><span class="p">)</span> <span class="o">=></span>
|
||||
<span class="k">return</span> <span class="mi">1</span> <span class="nx">unless</span> <span class="nx">SINGLE_LINERS</span><span class="p">.</span><span class="nx">indexOf</span><span class="p">(</span><span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">>=</span> <span class="mi">0</span> <span class="o">and</span> <span class="nx">post</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">isnt</span> <span class="s1">'INDENT'</span> <span class="o">and</span>
|
||||
<span class="k">return</span> <span class="mi">1</span> <span class="nx">unless</span> <span class="nx">include</span><span class="p">(</span><span class="nx">SINGLE_LINERS</span><span class="p">,</span> <span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">and</span>
|
||||
<span class="nx">post</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">isnt</span> <span class="s1">'INDENT'</span> <span class="o">and</span>
|
||||
<span class="o">not</span> <span class="p">(</span><span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">'ELSE'</span> <span class="o">and</span> <span class="nx">post</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">'IF'</span><span class="p">)</span>
|
||||
<span class="nv">starter: </span><span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">splice</span><span class="p">(</span><span class="nx">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="s1">'INDENT'</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="nx">token</span><span class="p">[</span><span class="mi">2</span><span class="p">]])</span>
|
||||
|
|
@ -120,7 +112,7 @@ blocks, so it doesn't need to.
|
|||
<span class="nv">tok: </span><span class="err">@</span><span class="nx">tokens</span><span class="p">[</span><span class="nx">idx</span><span class="p">]</span>
|
||||
<span class="nv">pre: </span><span class="err">@</span><span class="nx">tokens</span><span class="p">[</span><span class="nx">idx</span> <span class="o">-</span> <span class="mi">1</span><span class="p">]</span>
|
||||
<span class="k">if</span> <span class="p">(</span><span class="o">not</span> <span class="nx">tok</span> <span class="o">or</span>
|
||||
<span class="p">(</span><span class="nx">SINGLE_CLOSERS</span><span class="p">.</span><span class="nx">indexOf</span><span class="p">(</span><span class="nx">tok</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">>=</span> <span class="mi">0</span> <span class="o">and</span> <span class="nx">tok</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">isnt</span> <span class="s1">';'</span><span class="p">)</span> <span class="o">or</span>
|
||||
<span class="p">(</span><span class="nx">include</span><span class="p">(</span><span class="nx">SINGLE_CLOSERS</span><span class="p">,</span> <span class="nx">tok</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">and</span> <span class="nx">tok</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="o">isnt</span> <span class="s1">';'</span><span class="p">)</span> <span class="o">or</span>
|
||||
<span class="p">(</span><span class="nx">tok</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">')'</span> <span class="o">&&</span> <span class="nx">parens</span> <span class="o">is</span> <span class="mi">0</span><span class="p">))</span> <span class="o">and</span>
|
||||
<span class="o">not</span> <span class="p">(</span><span class="nx">starter</span> <span class="o">is</span> <span class="s1">'ELSE'</span> <span class="o">and</span> <span class="nx">tok</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">'ELSE'</span><span class="p">)</span>
|
||||
<span class="nv">insertion: </span><span class="k">if</span> <span class="nx">pre</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s2">","</span> <span class="k">then</span> <span class="nx">idx</span> <span class="o">-</span> <span class="mi">1</span> <span class="k">else</span> <span class="nx">idx</span>
|
||||
|
|
@ -130,7 +122,7 @@ blocks, so it doesn't need to.
|
|||
<span class="nx">parens</span> <span class="o">-=</span> <span class="mi">1</span> <span class="k">if</span> <span class="nx">tok</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">')'</span>
|
||||
<span class="k">return</span> <span class="mi">1</span> <span class="nx">unless</span> <span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">is</span> <span class="s1">'THEN'</span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">splice</span><span class="p">(</span><span class="nx">i</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="mi">0</span></pre></div> </td> </tr> <tr id="section-19"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-19">#</a> </div> <p>Ensure that all listed pairs of tokens are correctly balanced throughout
|
||||
<span class="k">return</span> <span class="mi">0</span></pre></div> </td> </tr> <tr id="section-12"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-12">#</a> </div> <p>Ensure that all listed pairs of tokens are correctly balanced throughout
|
||||
the course of the token stream.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">ensure_balance: </span><span class="p">(</span><span class="nx">pairs</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="nv">levels: </span><span class="p">{}</span>
|
||||
<span class="err">@</span><span class="nx">scan_tokens</span> <span class="p">(</span><span class="nx">prev</span><span class="p">,</span> <span class="nx">token</span><span class="p">,</span> <span class="nx">post</span><span class="p">,</span> <span class="nx">i</span><span class="p">)</span> <span class="o">=></span>
|
||||
|
|
@ -142,10 +134,13 @@ the course of the token stream.</p> </td> <td class="cod
|
|||
<span class="k">throw</span> <span class="k">new</span> <span class="nb">Error</span><span class="p">(</span><span class="s2">"too many ${token[1]} on line ${token[2] + 1}"</span><span class="p">)</span> <span class="k">if</span> <span class="nx">levels</span><span class="p">[</span><span class="nx">open</span><span class="p">]</span> <span class="o"><</span> <span class="mi">0</span>
|
||||
<span class="k">return</span> <span class="mi">1</span>
|
||||
<span class="nv">unclosed: </span><span class="nx">key</span> <span class="k">for</span> <span class="nx">key</span><span class="p">,</span> <span class="nx">value</span> <span class="k">of</span> <span class="nx">levels</span> <span class="k">when</span> <span class="nx">value</span> <span class="o">></span> <span class="mi">0</span>
|
||||
<span class="k">throw</span> <span class="k">new</span> <span class="nb">Error</span><span class="p">(</span><span class="s2">"unclosed ${unclosed[0]}"</span><span class="p">)</span> <span class="k">if</span> <span class="nx">unclosed</span><span class="p">.</span><span class="nx">length</span></pre></div> </td> </tr> <tr id="section-20"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-20">#</a> </div> <p>We'd like to support syntax like this:
|
||||
el.click((event) ->
|
||||
el.hide())
|
||||
In order to accomplish this, move outdents that follow closing parens
|
||||
<span class="k">throw</span> <span class="k">new</span> <span class="nb">Error</span><span class="p">(</span><span class="s2">"unclosed ${unclosed[0]}"</span><span class="p">)</span> <span class="k">if</span> <span class="nx">unclosed</span><span class="p">.</span><span class="nx">length</span></pre></div> </td> </tr> <tr id="section-13"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-13">#</a> </div> <p>We'd like to support syntax like this:</p>
|
||||
|
||||
<pre><code>el.click((event) ->
|
||||
el.hide())
|
||||
</code></pre>
|
||||
|
||||
<p>In order to accomplish this, move outdents that follow closing parens
|
||||
inwards, safely. The steps to accomplish this are:</p>
|
||||
|
||||
<ol>
|
||||
|
|
@ -161,20 +156,36 @@ up balanced in the end.</li>
|
|||
<span class="p">(</span><span class="nx">debt</span><span class="p">[</span><span class="nx">key</span><span class="p">]</span><span class="o">:</span> <span class="mi">0</span><span class="p">)</span> <span class="k">for</span> <span class="nx">key</span><span class="p">,</span> <span class="nx">val</span> <span class="k">of</span> <span class="nx">INVERSES</span>
|
||||
<span class="err">@</span><span class="nx">scan_tokens</span> <span class="p">(</span><span class="nx">prev</span><span class="p">,</span> <span class="nx">token</span><span class="p">,</span> <span class="nx">post</span><span class="p">,</span> <span class="nx">i</span><span class="p">)</span> <span class="o">=></span>
|
||||
<span class="nv">tag: </span><span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="nv">inv: </span><span class="nx">INVERSES</span><span class="p">[</span><span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span></pre></div> </td> </tr> <tr id="section-21"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-21">#</a> </div> <p>Push openers onto the stack.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="k">if</span> <span class="nx">EXPRESSION_START</span><span class="p">.</span><span class="nx">indexOf</span><span class="p">(</span><span class="nx">tag</span><span class="p">)</span> <span class="o">>=</span> <span class="mi">0</span>
|
||||
<span class="nx">stack</span><span class="p">.</span><span class="nx">push</span><span class="p">(</span><span class="nx">token</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="mi">1</span></pre></div> </td> </tr> <tr id="section-22"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-22">#</a> </div> <p>The end of an expression, check stack and debt for a pair.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="k">else</span> <span class="k">if</span> <span class="nx">EXPRESSION_TAIL</span><span class="p">.</span><span class="nx">indexOf</span><span class="p">(</span><span class="nx">tag</span><span class="p">)</span> <span class="o">>=</span> <span class="mi">0</span></pre></div> </td> </tr> <tr id="section-23"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-23">#</a> </div> <p>If the tag is already in our debt, swallow it.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="k">if</span> <span class="nx">debt</span><span class="p">[</span><span class="nx">inv</span><span class="p">]</span> <span class="o">></span> <span class="mi">0</span>
|
||||
<span class="nx">debt</span><span class="p">[</span><span class="nx">inv</span><span class="p">]</span> <span class="o">-=</span> <span class="mi">1</span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">splice</span><span class="p">(</span><span class="nx">i</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="mi">0</span>
|
||||
<span class="k">else</span></pre></div> </td> </tr> <tr id="section-24"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-24">#</a> </div> <p>Pop the stack of open delimiters.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nv">match: </span><span class="nx">stack</span><span class="p">.</span><span class="nx">pop</span><span class="p">()</span>
|
||||
<span class="nv">mtag: </span> <span class="nx">match</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-25"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-25">#</a> </div> <p>Continue onwards if it's the expected tag.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="k">if</span> <span class="nx">tag</span> <span class="o">is</span> <span class="nx">INVERSES</span><span class="p">[</span><span class="nx">mtag</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="mi">1</span>
|
||||
<span class="k">else</span></pre></div> </td> </tr> <tr id="section-26"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-26">#</a> </div> <p>Unexpected close, insert correct close, adding to the debt.</p> </td> <td class="code"> <div class="highlight"><pre> <span class="nx">debt</span><span class="p">[</span><span class="nx">mtag</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
|
||||
<span class="nv">val: </span><span class="k">if</span> <span class="nx">mtag</span> <span class="o">is</span> <span class="s1">'INDENT'</span> <span class="k">then</span> <span class="nx">match</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">else</span> <span class="nx">INVERSES</span><span class="p">[</span><span class="nx">mtag</span><span class="p">]</span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">splice</span><span class="p">(</span><span class="nx">i</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="nx">INVERSES</span><span class="p">[</span><span class="nx">mtag</span><span class="p">],</span> <span class="nx">val</span><span class="p">])</span>
|
||||
<span class="k">return</span> <span class="mi">1</span>
|
||||
<span class="k">else</span>
|
||||
<span class="nv">inv: </span><span class="nx">INVERSES</span><span class="p">[</span><span class="nx">token</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span>
|
||||
<span class="k">if</span> <span class="nx">include</span> <span class="nx">EXPRESSION_START</span><span class="p">,</span> <span class="nx">tag</span>
|
||||
<span class="nx">stack</span><span class="p">.</span><span class="nx">push</span> <span class="nx">token</span>
|
||||
<span class="k">return</span> <span class="mi">1</span>
|
||||
<span class="k">else</span> <span class="k">if</span> <span class="nx">include</span> <span class="nx">EXPRESSION_END</span><span class="p">,</span> <span class="nx">tag</span>
|
||||
<span class="k">if</span> <span class="nx">debt</span><span class="p">[</span><span class="nx">inv</span><span class="p">]</span> <span class="o">></span> <span class="mi">0</span>
|
||||
<span class="nx">debt</span><span class="p">[</span><span class="nx">inv</span><span class="p">]</span> <span class="o">-=</span> <span class="mi">1</span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">splice</span> <span class="nx">i</span><span class="p">,</span> <span class="mi">1</span>
|
||||
<span class="k">return</span> <span class="mi">0</span>
|
||||
<span class="k">else</span>
|
||||
<span class="nv">match: </span><span class="nx">stack</span><span class="p">.</span><span class="nx">pop</span><span class="p">()</span>
|
||||
<span class="nv">mtag: </span> <span class="nx">match</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="mi">1</span> <span class="k">if</span> <span class="nx">tag</span> <span class="o">is</span> <span class="nx">INVERSES</span><span class="p">[</span><span class="nx">mtag</span><span class="p">]</span>
|
||||
<span class="nx">debt</span><span class="p">[</span><span class="nx">mtag</span><span class="p">]</span> <span class="o">+=</span> <span class="mi">1</span>
|
||||
<span class="nv">val: </span><span class="k">if</span> <span class="nx">mtag</span> <span class="o">is</span> <span class="s1">'INDENT'</span> <span class="k">then</span> <span class="nx">match</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">else</span> <span class="nx">INVERSES</span><span class="p">[</span><span class="nx">mtag</span><span class="p">]</span>
|
||||
<span class="err">@</span><span class="nx">tokens</span><span class="p">.</span><span class="nx">splice</span> <span class="nx">i</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="p">[</span><span class="nx">INVERSES</span><span class="p">[</span><span class="nx">mtag</span><span class="p">],</span> <span class="nx">val</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="mi">1</span>
|
||||
<span class="k">else</span>
|
||||
<span class="k">return</span> <span class="mi">1</span></pre></div> </td> </tr> <tr id="section-14"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-14">#</a> </div> <h2>Constants</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-15"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-15">#</a> </div> <p>List of the token pairs that must be balanced.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">BALANCED_PAIRS: </span><span class="p">[[</span><span class="s1">'('</span><span class="p">,</span> <span class="s1">')'</span><span class="p">],</span> <span class="p">[</span><span class="s1">'['</span><span class="p">,</span> <span class="s1">']'</span><span class="p">],</span> <span class="p">[</span><span class="s1">'{'</span><span class="p">,</span> <span class="s1">'}'</span><span class="p">],</span> <span class="p">[</span><span class="s1">'INDENT'</span><span class="p">,</span> <span class="s1">'OUTDENT'</span><span class="p">],</span>
|
||||
<span class="p">[</span><span class="s1">'PARAM_START'</span><span class="p">,</span> <span class="s1">'PARAM_END'</span><span class="p">],</span> <span class="p">[</span><span class="s1">'CALL_START'</span><span class="p">,</span> <span class="s1">'CALL_END'</span><span class="p">],</span>
|
||||
<span class="p">[</span><span class="s1">'INDEX_START'</span><span class="p">,</span> <span class="s1">'INDEX_END'</span><span class="p">],</span> <span class="p">[</span><span class="s1">'SOAKED_INDEX_START'</span><span class="p">,</span> <span class="s1">'SOAKED_INDEX_END'</span><span class="p">]]</span></pre></div> </td> </tr> <tr id="section-16"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-16">#</a> </div> <p>The inverse mappings of <code>BALANCED_PAIRS</code> we're trying to fix up, so we can
|
||||
look things up from either end.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">INVERSES: </span><span class="p">{}</span>
|
||||
<span class="k">for</span> <span class="nx">pair</span> <span class="k">in</span> <span class="nx">BALANCED_PAIRS</span>
|
||||
<span class="nx">INVERSES</span><span class="p">[</span><span class="nx">pair</span><span class="p">[</span><span class="mi">0</span><span class="p">]]</span><span class="o">:</span> <span class="nx">pair</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
|
||||
<span class="nx">INVERSES</span><span class="p">[</span><span class="nx">pair</span><span class="p">[</span><span class="mi">1</span><span class="p">]]</span><span class="o">:</span> <span class="nx">pair</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-17"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-17">#</a> </div> <p>The tokens that signal the start of a balanced pair.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">EXPRESSION_START: </span><span class="nx">pair</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="nx">pair</span> <span class="k">in</span> <span class="nx">BALANCED_PAIRS</span></pre></div> </td> </tr> <tr id="section-18"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-18">#</a> </div> <p>The tokens that signal the end of a balanced pair.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">EXPRESSION_END: </span> <span class="nx">pair</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="nx">pair</span> <span class="k">in</span> <span class="nx">BALANCED_PAIRS</span></pre></div> </td> </tr> <tr id="section-19"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-19">#</a> </div> <p>Tokens that indicate the close of a clause of an expression.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">EXPRESSION_CLOSE: </span><span class="p">[</span><span class="s1">'CATCH'</span><span class="p">,</span> <span class="s1">'WHEN'</span><span class="p">,</span> <span class="s1">'ELSE'</span><span class="p">,</span> <span class="s1">'FINALLY'</span><span class="p">].</span><span class="nx">concat</span> <span class="nx">EXPRESSION_END</span></pre></div> </td> </tr> <tr id="section-20"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-20">#</a> </div> <p>Tokens that, if followed by an <code>IMPLICIT_CALL</code>, indicate a function invocation.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">IMPLICIT_FUNC: </span> <span class="p">[</span><span class="s1">'IDENTIFIER'</span><span class="p">,</span> <span class="s1">'SUPER'</span><span class="p">,</span> <span class="s1">')'</span><span class="p">,</span> <span class="s1">'CALL_END'</span><span class="p">,</span> <span class="s1">']'</span><span class="p">,</span> <span class="s1">'INDEX_END'</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-21"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-21">#</a> </div> <p>If preceded by an <code>IMPLICIT_FUNC</code>, indicates a function invocation.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">IMPLICIT_CALL: </span> <span class="p">[</span><span class="s1">'IDENTIFIER'</span><span class="p">,</span> <span class="s1">'NUMBER'</span><span class="p">,</span> <span class="s1">'STRING'</span><span class="p">,</span> <span class="s1">'JS'</span><span class="p">,</span> <span class="s1">'REGEX'</span><span class="p">,</span> <span class="s1">'NEW'</span><span class="p">,</span> <span class="s1">'PARAM_START'</span><span class="p">,</span>
|
||||
<span class="s1">'TRY'</span><span class="p">,</span> <span class="s1">'DELETE'</span><span class="p">,</span> <span class="s1">'TYPEOF'</span><span class="p">,</span> <span class="s1">'SWITCH'</span><span class="p">,</span>
|
||||
<span class="s1">'TRUE'</span><span class="p">,</span> <span class="s1">'FALSE'</span><span class="p">,</span> <span class="s1">'YES'</span><span class="p">,</span> <span class="s1">'NO'</span><span class="p">,</span> <span class="s1">'ON'</span><span class="p">,</span> <span class="s1">'OFF'</span><span class="p">,</span> <span class="s1">'!'</span><span class="p">,</span> <span class="s1">'!!'</span><span class="p">,</span> <span class="s1">'NOT'</span><span class="p">,</span>
|
||||
<span class="s1">'@'</span><span class="p">,</span> <span class="s1">'->'</span><span class="p">,</span> <span class="s1">'=>'</span><span class="p">,</span> <span class="s1">'['</span><span class="p">,</span> <span class="s1">'('</span><span class="p">,</span> <span class="s1">'{'</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-22"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-22">#</a> </div> <p>Tokens indicating that the implicit call must enclose a block of expressions.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">IMPLICIT_BLOCK: </span><span class="p">[</span><span class="s1">'->'</span><span class="p">,</span> <span class="s1">'=>'</span><span class="p">,</span> <span class="s1">'{'</span><span class="p">,</span> <span class="s1">'['</span><span class="p">,</span> <span class="s1">','</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-23"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-23">#</a> </div> <p>Tokens that always mark the end of an implicit call for single-liners.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">IMPLICIT_END: </span> <span class="p">[</span><span class="s1">'IF'</span><span class="p">,</span> <span class="s1">'UNLESS'</span><span class="p">,</span> <span class="s1">'FOR'</span><span class="p">,</span> <span class="s1">'WHILE'</span><span class="p">,</span> <span class="s1">'TERMINATOR'</span><span class="p">,</span> <span class="s1">'INDENT'</span><span class="p">,</span> <span class="s1">'OUTDENT'</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-24"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-24">#</a> </div> <p>Single-line flavors of block expressions that have unclosed endings.
|
||||
The grammar can't disambiguate them, so we insert the implicit indentation.</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">SINGLE_LINERS: </span><span class="p">[</span><span class="s1">'ELSE'</span><span class="p">,</span> <span class="s2">"->"</span><span class="p">,</span> <span class="s2">"=>"</span><span class="p">,</span> <span class="s1">'TRY'</span><span class="p">,</span> <span class="s1">'FINALLY'</span><span class="p">,</span> <span class="s1">'THEN'</span><span class="p">]</span>
|
||||
<span class="nv">SINGLE_CLOSERS: </span><span class="p">[</span><span class="s1">'TERMINATOR'</span><span class="p">,</span> <span class="s1">'CATCH'</span><span class="p">,</span> <span class="s1">'FINALLY'</span><span class="p">,</span> <span class="s1">'ELSE'</span><span class="p">,</span> <span class="s1">'OUTDENT'</span><span class="p">,</span> <span class="s1">'LEADING_WHEN'</span><span class="p">]</span></pre></div> </td> </tr> <tr id="section-25"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-25">#</a> </div> <h2>Utility Functions</h2> </td> <td class="code"> <div class="highlight"><pre></pre></div> </td> </tr> <tr id="section-26"> <td class="docs"> <div class="octowrap"> <a class="octothorpe" href="#section-26">#</a> </div> <p>Does a list include a value?</p> </td> <td class="code"> <div class="highlight"><pre><span class="nv">include: </span><span class="p">(</span><span class="nx">list</span><span class="p">,</span> <span class="nx">value</span><span class="p">)</span> <span class="o">-></span>
|
||||
<span class="nx">list</span><span class="p">.</span><span class="nx">indexOf</span><span class="p">(</span><span class="nx">value</span><span class="p">)</span> <span class="o">>=</span> <span class="mi">0</span>
|
||||
|
||||
</pre></div> </td> </tr> </tbody> </table> </div> </body> </html>
|
||||
112
lib/lexer.js
112
lib/lexer.js
|
|
@ -13,64 +13,10 @@
|
|||
this.exports = this;
|
||||
Rewriter = this.Rewriter;
|
||||
}
|
||||
// Constants
|
||||
// ---------
|
||||
// Keywords that CoffeeScript shares in common with JavaScript.
|
||||
JS_KEYWORDS = ["if", "else", "true", "false", "new", "return", "try", "catch", "finally", "throw", "break", "continue", "for", "in", "while", "delete", "instanceof", "typeof", "switch", "super", "extends", "class"];
|
||||
// CoffeeScript-only keywords, which we're more relaxed about allowing. They can't
|
||||
// be used standalone, but you can reference them as an attached property.
|
||||
COFFEE_KEYWORDS = ["then", "unless", "yes", "no", "on", "off", "and", "or", "is", "isnt", "not", "of", "by", "where", "when"];
|
||||
// The combined list of keywords is the superset that gets passed verbatim to
|
||||
// the parser.
|
||||
KEYWORDS = JS_KEYWORDS.concat(COFFEE_KEYWORDS);
|
||||
// The list of keywords that are reserved by JavaScript, but not used, or are
|
||||
// used by CoffeeScript internally. We throw an error when these are encountered,
|
||||
// to avoid having a JavaScript error at runtime.
|
||||
RESERVED = ["case", "default", "do", "function", "var", "void", "with", "const", "let", "debugger", "enum", "export", "import", "native", "__extends", "__hasProp"];
|
||||
// The superset of both JavaScript keywords and reserved words, none of which may
|
||||
// be used as identifiers or properties.
|
||||
JS_FORBIDDEN = JS_KEYWORDS.concat(RESERVED);
|
||||
// Token matching regexes.
|
||||
IDENTIFIER = /^([a-zA-Z$_](\w|\$)*)/;
|
||||
NUMBER = /^(\b((0(x|X)[0-9a-fA-F]+)|([0-9]+(\.[0-9]+)?(e[+\-]?[0-9]+)?)))\b/i;
|
||||
HEREDOC = /^("{6}|'{6}|"{3}\n?([\s\S]*?)\n?([ \t]*)"{3}|'{3}\n?([\s\S]*?)\n?([ \t]*)'{3})/;
|
||||
INTERPOLATION = /^\$([a-zA-Z_@]\w*)/;
|
||||
OPERATOR = /^([+\*&|\/\-%=<>:!?]+)/;
|
||||
WHITESPACE = /^([ \t]+)/;
|
||||
COMMENT = /^(((\n?[ \t]*)?#[^\n]*)+)/;
|
||||
CODE = /^((-|=)>)/;
|
||||
REGEX = /^(\/(\S.*?)?([^\\]|\\\\)\/[imgy]{0,4})/;
|
||||
MULTI_DENT = /^((\n([ \t]*))+)(\.)?/;
|
||||
LAST_DENTS = /\n([ \t]*)/g;
|
||||
LAST_DENT = /\n([ \t]*)/;
|
||||
ASSIGNMENT = /^(:|=)$/;
|
||||
// Token cleaning regexes.
|
||||
JS_CLEANER = /(^`|`$)/g;
|
||||
MULTILINER = /\n/g;
|
||||
STRING_NEWLINES = /\n[ \t]*/g;
|
||||
COMMENT_CLEANER = /(^[ \t]*#|\n[ \t]*$)/mg;
|
||||
NO_NEWLINE = /^([+\*&|\/\-%=<>:!.\\][<>=&|]*|and|or|is|isnt|not|delete|typeof|instanceof)$/;
|
||||
HEREDOC_INDENT = /^[ \t]+/mg;
|
||||
// Tokens which a regular expression will never immediately follow, but which
|
||||
// a division operator might.
|
||||
// See: http://www.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions
|
||||
// Our list is shorter, due to sans-parentheses method calls.
|
||||
NOT_REGEX = ['NUMBER', 'REGEX', '++', '--', 'FALSE', 'NULL', 'TRUE'];
|
||||
// Tokens which could legitimately be invoked or indexed. A opening
|
||||
// parentheses or bracket following these tokens will be recorded as the start
|
||||
// of a function invocation or indexing operation.
|
||||
CALLABLE = ['IDENTIFIER', 'SUPER', ')', ']', '}', 'STRING', '@'];
|
||||
// Tokens that indicate an access -- keywords immediately following will be
|
||||
// treated as identifiers.
|
||||
ACCESSORS = ['PROPERTY_ACCESS', 'PROTOTYPE_ACCESS', 'SOAK_ACCESS', '@'];
|
||||
// Tokens that, when immediately preceding a `WHEN`, indicate that the `WHEN`
|
||||
// occurs at the start of a line. We disambiguate these from trailing whens to
|
||||
// avoid an ambiguity in the grammar.
|
||||
BEFORE_WHEN = ['INDENT', 'OUTDENT', 'TERMINATOR'];
|
||||
// The Lexer Class
|
||||
// ---------------
|
||||
// The Lexer class reads a stream of CoffeeScript and divvys it up into tagged
|
||||
// tokens. A minor bit of the ambiguity in the grammar has been avoided by
|
||||
// tokens. Some potential ambiguity in the grammar has been avoided by
|
||||
// pushing some extra smarts into the Lexer.
|
||||
exports.Lexer = (function() {
|
||||
Lexer = function Lexer() { };
|
||||
|
|
@ -82,6 +28,8 @@
|
|||
// Each tokenizing method is responsible for incrementing `@i` by the number of
|
||||
// characters it has consumed. `@i` can be thought of as our finger on the page
|
||||
// of source.
|
||||
// Before returning the token stream, run it through the [Rewriter](rewriter.html)
|
||||
// unless explicitly asked not to.
|
||||
Lexer.prototype.tokenize = function tokenize(code, options) {
|
||||
var o;
|
||||
o = options || {};
|
||||
|
|
@ -598,6 +546,60 @@
|
|||
};
|
||||
return Lexer;
|
||||
}).call(this);
|
||||
// Constants
|
||||
// ---------
|
||||
// Keywords that CoffeeScript shares in common with JavaScript.
|
||||
JS_KEYWORDS = ["if", "else", "true", "false", "new", "return", "try", "catch", "finally", "throw", "break", "continue", "for", "in", "while", "delete", "instanceof", "typeof", "switch", "super", "extends", "class"];
|
||||
// CoffeeScript-only keywords, which we're more relaxed about allowing. They can't
|
||||
// be used standalone, but you can reference them as an attached property.
|
||||
COFFEE_KEYWORDS = ["then", "unless", "yes", "no", "on", "off", "and", "or", "is", "isnt", "not", "of", "by", "where", "when"];
|
||||
// The combined list of keywords is the superset that gets passed verbatim to
|
||||
// the parser.
|
||||
KEYWORDS = JS_KEYWORDS.concat(COFFEE_KEYWORDS);
|
||||
// The list of keywords that are reserved by JavaScript, but not used, or are
|
||||
// used by CoffeeScript internally. We throw an error when these are encountered,
|
||||
// to avoid having a JavaScript error at runtime.
|
||||
RESERVED = ["case", "default", "do", "function", "var", "void", "with", "const", "let", "debugger", "enum", "export", "import", "native", "__extends", "__hasProp"];
|
||||
// The superset of both JavaScript keywords and reserved words, none of which may
|
||||
// be used as identifiers or properties.
|
||||
JS_FORBIDDEN = JS_KEYWORDS.concat(RESERVED);
|
||||
// Token matching regexes.
|
||||
IDENTIFIER = /^([a-zA-Z$_](\w|\$)*)/;
|
||||
NUMBER = /^(\b((0(x|X)[0-9a-fA-F]+)|([0-9]+(\.[0-9]+)?(e[+\-]?[0-9]+)?)))\b/i;
|
||||
HEREDOC = /^("{6}|'{6}|"{3}\n?([\s\S]*?)\n?([ \t]*)"{3}|'{3}\n?([\s\S]*?)\n?([ \t]*)'{3})/;
|
||||
INTERPOLATION = /^\$([a-zA-Z_@]\w*)/;
|
||||
OPERATOR = /^([+\*&|\/\-%=<>:!?]+)/;
|
||||
WHITESPACE = /^([ \t]+)/;
|
||||
COMMENT = /^(((\n?[ \t]*)?#[^\n]*)+)/;
|
||||
CODE = /^((-|=)>)/;
|
||||
REGEX = /^(\/(\S.*?)?([^\\]|\\\\)\/[imgy]{0,4})/;
|
||||
MULTI_DENT = /^((\n([ \t]*))+)(\.)?/;
|
||||
LAST_DENTS = /\n([ \t]*)/g;
|
||||
LAST_DENT = /\n([ \t]*)/;
|
||||
ASSIGNMENT = /^(:|=)$/;
|
||||
// Token cleaning regexes.
|
||||
JS_CLEANER = /(^`|`$)/g;
|
||||
MULTILINER = /\n/g;
|
||||
STRING_NEWLINES = /\n[ \t]*/g;
|
||||
COMMENT_CLEANER = /(^[ \t]*#|\n[ \t]*$)/mg;
|
||||
NO_NEWLINE = /^([+\*&|\/\-%=<>:!.\\][<>=&|]*|and|or|is|isnt|not|delete|typeof|instanceof)$/;
|
||||
HEREDOC_INDENT = /^[ \t]+/mg;
|
||||
// Tokens which a regular expression will never immediately follow, but which
|
||||
// a division operator might.
|
||||
// See: http://www.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions
|
||||
// Our list is shorter, due to sans-parentheses method calls.
|
||||
NOT_REGEX = ['NUMBER', 'REGEX', '++', '--', 'FALSE', 'NULL', 'TRUE'];
|
||||
// Tokens which could legitimately be invoked or indexed. A opening
|
||||
// parentheses or bracket following these tokens will be recorded as the start
|
||||
// of a function invocation or indexing operation.
|
||||
CALLABLE = ['IDENTIFIER', 'SUPER', ')', ']', '}', 'STRING', '@'];
|
||||
// Tokens that indicate an access -- keywords immediately following will be
|
||||
// treated as identifiers.
|
||||
ACCESSORS = ['PROPERTY_ACCESS', 'PROTOTYPE_ACCESS', 'SOAK_ACCESS', '@'];
|
||||
// Tokens that, when immediately preceding a `WHEN`, indicate that the `WHEN`
|
||||
// occurs at the start of a line. We disambiguate these from trailing whens to
|
||||
// avoid an ambiguity in the grammar.
|
||||
BEFORE_WHEN = ['INDENT', 'OUTDENT', 'TERMINATOR'];
|
||||
// Utility Functions
|
||||
// -----------------
|
||||
// Does a list include a value?
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@
|
|||
var CoffeeScript, prompt, quit, run;
|
||||
// A very simple Read-Eval-Print-Loop. Compiles one line at a time to JavaScript
|
||||
// and evaluates it. Good for simple tests, or poking around the **Node.js** API.
|
||||
// Using it looks like this:
|
||||
// coffee> puts "$num bottles of beer" for num in [99..1]
|
||||
// Require the **coffee-script** module to get access to the compiler.
|
||||
CoffeeScript = require('coffee-script');
|
||||
// Our prompt.
|
||||
|
|
@ -10,7 +12,7 @@
|
|||
quit = function quit() {
|
||||
return process.exit(0);
|
||||
};
|
||||
// The main REPL function. `run` is called every time a line of code is entered.
|
||||
// The main REPL function. **run** is called every time a line of code is entered.
|
||||
// Attempt to evaluate the command. If there's an exception, print it out instead
|
||||
// of exiting.
|
||||
run = function run(code) {
|
||||
|
|
@ -28,7 +30,7 @@
|
|||
}
|
||||
return print(prompt);
|
||||
};
|
||||
// Start up the REPL.
|
||||
// Start up the REPL by opening **stdio** and listening for input.
|
||||
process.stdio.addListener('data', run);
|
||||
process.stdio.open();
|
||||
print(prompt);
|
||||
|
|
|
|||
237
lib/rewriter.js
237
lib/rewriter.js
|
|
@ -1,62 +1,31 @@
|
|||
(function(){
|
||||
var BALANCED_PAIRS, EXPRESSION_CLOSE, EXPRESSION_START, EXPRESSION_TAIL, IMPLICIT_BLOCK, IMPLICIT_CALL, IMPLICIT_END, IMPLICIT_FUNC, INVERSES, Rewriter, SINGLE_CLOSERS, SINGLE_LINERS, _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, pair;
|
||||
var BALANCED_PAIRS, EXPRESSION_CLOSE, EXPRESSION_END, EXPRESSION_START, IMPLICIT_BLOCK, IMPLICIT_CALL, IMPLICIT_END, IMPLICIT_FUNC, INVERSES, Rewriter, SINGLE_CLOSERS, SINGLE_LINERS, _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, include, pair;
|
||||
var __hasProp = Object.prototype.hasOwnProperty;
|
||||
// The CoffeeScript language has a decent amount of optional syntax,
|
||||
// implicit syntax, and shorthand syntax. These things can greatly complicate a
|
||||
// grammar and bloat the resulting parse table. Instead of making the parser
|
||||
// handle it all, we take a series of passes over the token stream,
|
||||
// using this **Rewriter** to convert shorthand into the unambiguous long form,
|
||||
// add implicit indentation and parentheses, balance incorrect nestings, and
|
||||
// generally clean things up.
|
||||
// Set up exported variables for both Node.js and the browser.
|
||||
if (!((typeof process !== "undefined" && process !== null))) {
|
||||
this.exports = this;
|
||||
}
|
||||
// Tokens that must be balanced.
|
||||
BALANCED_PAIRS = [['(', ')'], ['[', ']'], ['{', '}'], ['INDENT', 'OUTDENT'], ['PARAM_START', 'PARAM_END'], ['CALL_START', 'CALL_END'], ['INDEX_START', 'INDEX_END'], ['SOAKED_INDEX_START', 'SOAKED_INDEX_END']];
|
||||
// Tokens that signal the start of a balanced pair.
|
||||
EXPRESSION_START = (function() {
|
||||
_a = []; _b = BALANCED_PAIRS;
|
||||
for (_c = 0, _d = _b.length; _c < _d; _c++) {
|
||||
pair = _b[_c];
|
||||
_a.push(pair[0]);
|
||||
}
|
||||
return _a;
|
||||
}).call(this);
|
||||
// Tokens that signal the end of a balanced pair.
|
||||
EXPRESSION_TAIL = (function() {
|
||||
_e = []; _f = BALANCED_PAIRS;
|
||||
for (_g = 0, _h = _f.length; _g < _h; _g++) {
|
||||
pair = _f[_g];
|
||||
_e.push(pair[1]);
|
||||
}
|
||||
return _e;
|
||||
}).call(this);
|
||||
// Tokens that indicate the close of a clause of an expression.
|
||||
EXPRESSION_CLOSE = ['CATCH', 'WHEN', 'ELSE', 'FINALLY'].concat(EXPRESSION_TAIL);
|
||||
// Tokens pairs that, in immediate succession, indicate an implicit call.
|
||||
IMPLICIT_FUNC = ['IDENTIFIER', 'SUPER', ')', 'CALL_END', ']', 'INDEX_END'];
|
||||
IMPLICIT_BLOCK = ['->', '=>', '{', '[', ','];
|
||||
IMPLICIT_END = ['IF', 'UNLESS', 'FOR', 'WHILE', 'TERMINATOR', 'INDENT', 'OUTDENT'];
|
||||
IMPLICIT_CALL = ['IDENTIFIER', 'NUMBER', 'STRING', 'JS', 'REGEX', 'NEW', 'PARAM_START', 'TRY', 'DELETE', 'TYPEOF', 'SWITCH', 'TRUE', 'FALSE', 'YES', 'NO', 'ON', 'OFF', '!', '!!', 'NOT', '@', '->', '=>', '[', '(', '{'];
|
||||
// The inverse mappings of token pairs we're trying to fix up.
|
||||
INVERSES = {};
|
||||
_i = BALANCED_PAIRS;
|
||||
for (_j = 0, _k = _i.length; _j < _k; _j++) {
|
||||
pair = _i[_j];
|
||||
INVERSES[pair[0]] = pair[1];
|
||||
INVERSES[pair[1]] = pair[0];
|
||||
}
|
||||
// Single-line flavors of block expressions that have unclosed endings.
|
||||
// The grammar can't disambiguate them, so we insert the implicit indentation.
|
||||
SINGLE_LINERS = ['ELSE', "->", "=>", 'TRY', 'FINALLY', 'THEN'];
|
||||
SINGLE_CLOSERS = ['TERMINATOR', 'CATCH', 'FINALLY', 'ELSE', 'OUTDENT', 'LEADING_WHEN'];
|
||||
// In order to keep the grammar simple, the stream of tokens that the Lexer
|
||||
// emits is rewritten by the Rewriter, smoothing out ambiguities, mis-nested
|
||||
// indentation, and single-line flavors of expressions.
|
||||
// The **Rewriter** class is used by the [Lexer](lexer.html), directly against
|
||||
// its internal array of tokens.
|
||||
exports.Rewriter = (function() {
|
||||
Rewriter = function Rewriter() { };
|
||||
// Rewrite the token stream in multiple passes, one logical filter at
|
||||
// a time. This could certainly be changed into a single pass through the
|
||||
// stream, with a big ol' efficient switch, but it's much nicer like this.
|
||||
// stream, with a big ol' efficient switch, but it's much nicer to work with
|
||||
// like this. The order of these passes matters -- indentation must be
|
||||
// corrected before implicit parentheses can be wrapped around blocks of code.
|
||||
Rewriter.prototype.rewrite = function rewrite(tokens) {
|
||||
this.tokens = tokens;
|
||||
this.adjust_comments();
|
||||
this.remove_leading_newlines();
|
||||
this.remove_mid_expression_newlines();
|
||||
this.move_commas_outside_outdents();
|
||||
this.close_open_calls_and_indexes();
|
||||
this.add_implicit_indentation();
|
||||
this.add_implicit_parentheses();
|
||||
|
|
@ -67,7 +36,8 @@
|
|||
// Rewrite the token stream, looking one token ahead and behind.
|
||||
// Allow the return value of the block to tell us how many tokens to move
|
||||
// forwards (or backwards) in the stream, to make sure we don't miss anything
|
||||
// as the stream changes length under our feet.
|
||||
// as tokens are inserted and removed, and the stream changes length under
|
||||
// our feet.
|
||||
Rewriter.prototype.scan_tokens = function scan_tokens(block) {
|
||||
var i, move;
|
||||
i = 0;
|
||||
|
|
@ -81,7 +51,7 @@
|
|||
return true;
|
||||
};
|
||||
// Massage newlines and indentations so that comments don't have to be
|
||||
// correctly indented, or appear on their own line.
|
||||
// correctly indented, or appear on a line of their own.
|
||||
Rewriter.prototype.adjust_comments = function adjust_comments() {
|
||||
return this.scan_tokens((function(__this) {
|
||||
var __func = function(prev, token, post, i) {
|
||||
|
|
@ -109,16 +79,19 @@
|
|||
// Leading newlines would introduce an ambiguity in the grammar, so we
|
||||
// dispatch them here.
|
||||
Rewriter.prototype.remove_leading_newlines = function remove_leading_newlines() {
|
||||
if (this.tokens[0][0] === 'TERMINATOR') {
|
||||
return this.tokens.shift();
|
||||
var _a;
|
||||
_a = [];
|
||||
while (this.tokens[0][0] === 'TERMINATOR') {
|
||||
_a.push(this.tokens.shift());
|
||||
}
|
||||
return _a;
|
||||
};
|
||||
// Some blocks occur in the middle of expressions -- when we're expecting
|
||||
// this, remove their trailing newlines.
|
||||
Rewriter.prototype.remove_mid_expression_newlines = function remove_mid_expression_newlines() {
|
||||
return this.scan_tokens((function(__this) {
|
||||
var __func = function(prev, token, post, i) {
|
||||
if (!(post && EXPRESSION_CLOSE.indexOf(post[0]) >= 0 && token[0] === 'TERMINATOR')) {
|
||||
if (!(post && include(EXPRESSION_CLOSE, post[0]) && token[0] === 'TERMINATOR')) {
|
||||
return 1;
|
||||
}
|
||||
this.tokens.splice(i, 1);
|
||||
|
|
@ -129,46 +102,32 @@
|
|||
});
|
||||
})(this));
|
||||
};
|
||||
// Make sure that we don't accidentally break trailing commas, which need
|
||||
// to go on the outside of expression closers.
|
||||
Rewriter.prototype.move_commas_outside_outdents = function move_commas_outside_outdents() {
|
||||
return this.scan_tokens((function(__this) {
|
||||
var __func = function(prev, token, post, i) {
|
||||
if (token[0] === 'OUTDENT' && prev[0] === ',') {
|
||||
this.tokens.splice(i, 1, token);
|
||||
}
|
||||
return 1;
|
||||
};
|
||||
return (function() {
|
||||
return __func.apply(__this, arguments);
|
||||
});
|
||||
})(this));
|
||||
};
|
||||
// We've tagged the opening parenthesis of a method call, and the opening
|
||||
// bracket of an indexing operation. Match them with their close.
|
||||
// The lexer has tagged the opening parenthesis of a method call, and the
|
||||
// opening bracket of an indexing operation. Match them with their paired
|
||||
// close.
|
||||
Rewriter.prototype.close_open_calls_and_indexes = function close_open_calls_and_indexes() {
|
||||
var brackets, parens;
|
||||
parens = [0];
|
||||
brackets = [0];
|
||||
return this.scan_tokens((function(__this) {
|
||||
var __func = function(prev, token, post, i) {
|
||||
var _l;
|
||||
if ((_l = token[0]) === 'CALL_START') {
|
||||
var _a;
|
||||
if ((_a = token[0]) === 'CALL_START') {
|
||||
parens.push(0);
|
||||
} else if (_l === 'INDEX_START') {
|
||||
} else if (_a === 'INDEX_START') {
|
||||
brackets.push(0);
|
||||
} else if (_l === '(') {
|
||||
} else if (_a === '(') {
|
||||
parens[parens.length - 1] += 1;
|
||||
} else if (_l === '[') {
|
||||
} else if (_a === '[') {
|
||||
brackets[brackets.length - 1] += 1;
|
||||
} else if (_l === ')') {
|
||||
} else if (_a === ')') {
|
||||
if (parens[parens.length - 1] === 0) {
|
||||
parens.pop();
|
||||
token[0] = 'CALL_END';
|
||||
} else {
|
||||
parens[parens.length - 1] -= 1;
|
||||
}
|
||||
} else if (_l === ']') {
|
||||
} else if (_a === ']') {
|
||||
if (brackets[brackets.length - 1] === 0) {
|
||||
brackets.pop();
|
||||
token[0] = 'INDEX_END';
|
||||
|
|
@ -191,7 +150,7 @@
|
|||
stack = [0];
|
||||
return this.scan_tokens((function(__this) {
|
||||
var __func = function(prev, token, post, i) {
|
||||
var _l, _m, _n, _o, idx, last, size, stack_pointer, tag, tmp;
|
||||
var _a, _b, _c, _d, idx, last, size, stack_pointer, tag, tmp;
|
||||
tag = token[0];
|
||||
if (tag === 'INDENT') {
|
||||
stack.push(0);
|
||||
|
|
@ -200,15 +159,15 @@
|
|||
last = stack.pop();
|
||||
stack[stack.length - 1] += last;
|
||||
}
|
||||
if (IMPLICIT_END.indexOf(tag) >= 0 || !(typeof post !== "undefined" && post !== null)) {
|
||||
if (tag === 'INDENT' && prev && IMPLICIT_BLOCK.indexOf(prev[0]) >= 0) {
|
||||
if (!(typeof post !== "undefined" && post !== null) || include(IMPLICIT_END, tag)) {
|
||||
if (tag === 'INDENT' && prev && include(IMPLICIT_BLOCK, prev[0])) {
|
||||
return 1;
|
||||
}
|
||||
if (stack[stack.length - 1] > 0 || tag === 'INDENT') {
|
||||
idx = tag === 'OUTDENT' ? i + 1 : i;
|
||||
stack_pointer = tag === 'INDENT' ? 2 : 1;
|
||||
_n = 0; _o = stack[stack.length - stack_pointer];
|
||||
for (_m = 0, tmp = _n; (_n <= _o ? tmp < _o : tmp > _o); (_n <= _o ? tmp += 1 : tmp -= 1), _m++) {
|
||||
_c = 0; _d = stack[stack.length - stack_pointer];
|
||||
for (_b = 0, tmp = _c; (_c <= _d ? tmp < _d : tmp > _d); (_c <= _d ? tmp += 1 : tmp -= 1), _b++) {
|
||||
this.tokens.splice(idx, 0, ['CALL_END', ')', token[2]]);
|
||||
}
|
||||
size = stack[stack.length - stack_pointer] + 1;
|
||||
|
|
@ -216,7 +175,7 @@
|
|||
return size;
|
||||
}
|
||||
}
|
||||
if (!(prev && IMPLICIT_FUNC.indexOf(prev[0]) >= 0 && IMPLICIT_CALL.indexOf(tag) >= 0)) {
|
||||
if (!(prev && include(IMPLICIT_FUNC, prev[0]) && include(IMPLICIT_CALL, tag))) {
|
||||
return 1;
|
||||
}
|
||||
this.tokens.splice(i, 0, ['CALL_START', '(', token[2]]);
|
||||
|
|
@ -229,14 +188,14 @@
|
|||
})(this));
|
||||
};
|
||||
// Because our grammar is LALR(1), it can't handle some single-line
|
||||
// expressions that lack ending delimiters. Use the lexer to add the implicit
|
||||
// blocks, so it doesn't need to.
|
||||
// ')' can close a single-line block, but we need to make sure it's balanced.
|
||||
// expressions that lack ending delimiters. The **Rewriter** adds the implicit
|
||||
// blocks, so it doesn't need to. ')' can close a single-line block,
|
||||
// but we need to make sure it's balanced.
|
||||
Rewriter.prototype.add_implicit_indentation = function add_implicit_indentation() {
|
||||
return this.scan_tokens((function(__this) {
|
||||
var __func = function(prev, token, post, i) {
|
||||
var idx, insertion, parens, pre, starter, tok;
|
||||
if (!(SINGLE_LINERS.indexOf(token[0]) >= 0 && post[0] !== 'INDENT' && !(token[0] === 'ELSE' && post[0] === 'IF'))) {
|
||||
if (!(include(SINGLE_LINERS, token[0]) && post[0] !== 'INDENT' && !(token[0] === 'ELSE' && post[0] === 'IF'))) {
|
||||
return 1;
|
||||
}
|
||||
starter = token[0];
|
||||
|
|
@ -247,7 +206,7 @@
|
|||
idx += 1;
|
||||
tok = this.tokens[idx];
|
||||
pre = this.tokens[idx - 1];
|
||||
if ((!tok || (SINGLE_CLOSERS.indexOf(tok[0]) >= 0 && tok[1] !== ';') || (tok[0] === ')' && parens === 0)) && !(starter === 'ELSE' && tok[0] === 'ELSE')) {
|
||||
if ((!tok || (include(SINGLE_CLOSERS, tok[0]) && tok[1] !== ';') || (tok[0] === ')' && parens === 0)) && !(starter === 'ELSE' && tok[0] === 'ELSE')) {
|
||||
insertion = pre[0] === "," ? idx - 1 : idx;
|
||||
this.tokens.splice(insertion, 0, ['OUTDENT', 2, token[2]]);
|
||||
break;
|
||||
|
|
@ -273,17 +232,17 @@
|
|||
// Ensure that all listed pairs of tokens are correctly balanced throughout
|
||||
// the course of the token stream.
|
||||
Rewriter.prototype.ensure_balance = function ensure_balance(pairs) {
|
||||
var _l, _m, key, levels, unclosed, value;
|
||||
var _a, _b, key, levels, unclosed, value;
|
||||
levels = {};
|
||||
this.scan_tokens((function(__this) {
|
||||
var __func = function(prev, token, post, i) {
|
||||
var _l, _m, _n, _o, close, open;
|
||||
_l = pairs;
|
||||
for (_m = 0, _n = _l.length; _m < _n; _m++) {
|
||||
pair = _l[_m];
|
||||
_o = pair;
|
||||
open = _o[0];
|
||||
close = _o[1];
|
||||
var _a, _b, _c, _d, close, open, pair;
|
||||
_a = pairs;
|
||||
for (_b = 0, _c = _a.length; _b < _c; _b++) {
|
||||
pair = _a[_b];
|
||||
_d = pair;
|
||||
open = _d[0];
|
||||
close = _d[1];
|
||||
levels[open] = levels[open] || 0;
|
||||
if (token[0] === open) {
|
||||
levels[open] += 1;
|
||||
|
|
@ -302,22 +261,22 @@
|
|||
});
|
||||
})(this));
|
||||
unclosed = (function() {
|
||||
_l = []; _m = levels;
|
||||
for (key in _m) { if (__hasProp.call(_m, key)) {
|
||||
value = _m[key];
|
||||
_a = []; _b = levels;
|
||||
for (key in _b) { if (__hasProp.call(_b, key)) {
|
||||
value = _b[key];
|
||||
if (value > 0) {
|
||||
_l.push(key);
|
||||
_a.push(key);
|
||||
}
|
||||
}}
|
||||
return _l;
|
||||
return _a;
|
||||
}).call(this);
|
||||
if (unclosed.length) {
|
||||
throw new Error("unclosed " + (unclosed[0]));
|
||||
}
|
||||
};
|
||||
// We'd like to support syntax like this:
|
||||
// el.click((event) ->
|
||||
// el.hide())
|
||||
// el.click((event) ->
|
||||
// el.hide())
|
||||
// In order to accomplish this, move outdents that follow closing parens
|
||||
// inwards, safely. The steps to accomplish this are:
|
||||
// 1. Check that all paired tokens are balanced and in order.
|
||||
|
|
@ -327,12 +286,12 @@
|
|||
// 3. Keep track of "debt" for tokens that we fake, to make sure we end
|
||||
// up balanced in the end.
|
||||
Rewriter.prototype.rewrite_closing_parens = function rewrite_closing_parens() {
|
||||
var _l, debt, key, stack, val;
|
||||
var _a, debt, key, stack, val;
|
||||
stack = [];
|
||||
debt = {};
|
||||
_l = INVERSES;
|
||||
for (key in _l) { if (__hasProp.call(_l, key)) {
|
||||
val = _l[key];
|
||||
_a = INVERSES;
|
||||
for (key in _a) { if (__hasProp.call(_a, key)) {
|
||||
val = _a[key];
|
||||
((debt[key] = 0));
|
||||
}}
|
||||
return this.scan_tokens((function(__this) {
|
||||
|
|
@ -340,31 +299,24 @@
|
|||
var inv, match, mtag, tag;
|
||||
tag = token[0];
|
||||
inv = INVERSES[token[0]];
|
||||
// Push openers onto the stack.
|
||||
if (EXPRESSION_START.indexOf(tag) >= 0) {
|
||||
if (include(EXPRESSION_START, tag)) {
|
||||
stack.push(token);
|
||||
return 1;
|
||||
// The end of an expression, check stack and debt for a pair.
|
||||
} else if (EXPRESSION_TAIL.indexOf(tag) >= 0) {
|
||||
// If the tag is already in our debt, swallow it.
|
||||
} else if (include(EXPRESSION_END, tag)) {
|
||||
if (debt[inv] > 0) {
|
||||
debt[inv] -= 1;
|
||||
this.tokens.splice(i, 1);
|
||||
return 0;
|
||||
} else {
|
||||
// Pop the stack of open delimiters.
|
||||
match = stack.pop();
|
||||
mtag = match[0];
|
||||
// Continue onwards if it's the expected tag.
|
||||
if (tag === INVERSES[mtag]) {
|
||||
return 1;
|
||||
} else {
|
||||
// Unexpected close, insert correct close, adding to the debt.
|
||||
debt[mtag] += 1;
|
||||
val = mtag === 'INDENT' ? match[1] : INVERSES[mtag];
|
||||
this.tokens.splice(i, 0, [INVERSES[mtag], val]);
|
||||
return 1;
|
||||
}
|
||||
debt[mtag] += 1;
|
||||
val = mtag === 'INDENT' ? match[1] : INVERSES[mtag];
|
||||
this.tokens.splice(i, 0, [INVERSES[mtag], val]);
|
||||
return 1;
|
||||
}
|
||||
} else {
|
||||
return 1;
|
||||
|
|
@ -377,4 +329,55 @@
|
|||
};
|
||||
return Rewriter;
|
||||
}).call(this);
|
||||
// Constants
|
||||
// ---------
|
||||
// List of the token pairs that must be balanced.
|
||||
BALANCED_PAIRS = [['(', ')'], ['[', ']'], ['{', '}'], ['INDENT', 'OUTDENT'], ['PARAM_START', 'PARAM_END'], ['CALL_START', 'CALL_END'], ['INDEX_START', 'INDEX_END'], ['SOAKED_INDEX_START', 'SOAKED_INDEX_END']];
|
||||
// The inverse mappings of `BALANCED_PAIRS` we're trying to fix up, so we can
|
||||
// look things up from either end.
|
||||
INVERSES = {};
|
||||
_a = BALANCED_PAIRS;
|
||||
for (_b = 0, _c = _a.length; _b < _c; _b++) {
|
||||
pair = _a[_b];
|
||||
INVERSES[pair[0]] = pair[1];
|
||||
INVERSES[pair[1]] = pair[0];
|
||||
}
|
||||
// The tokens that signal the start of a balanced pair.
|
||||
EXPRESSION_START = (function() {
|
||||
_d = []; _e = BALANCED_PAIRS;
|
||||
for (_f = 0, _g = _e.length; _f < _g; _f++) {
|
||||
pair = _e[_f];
|
||||
_d.push(pair[0]);
|
||||
}
|
||||
return _d;
|
||||
}).call(this);
|
||||
// The tokens that signal the end of a balanced pair.
|
||||
EXPRESSION_END = (function() {
|
||||
_h = []; _i = BALANCED_PAIRS;
|
||||
for (_j = 0, _k = _i.length; _j < _k; _j++) {
|
||||
pair = _i[_j];
|
||||
_h.push(pair[1]);
|
||||
}
|
||||
return _h;
|
||||
}).call(this);
|
||||
// Tokens that indicate the close of a clause of an expression.
|
||||
EXPRESSION_CLOSE = ['CATCH', 'WHEN', 'ELSE', 'FINALLY'].concat(EXPRESSION_END);
|
||||
// Tokens that, if followed by an `IMPLICIT_CALL`, indicate a function invocation.
|
||||
IMPLICIT_FUNC = ['IDENTIFIER', 'SUPER', ')', 'CALL_END', ']', 'INDEX_END'];
|
||||
// If preceded by an `IMPLICIT_FUNC`, indicates a function invocation.
|
||||
IMPLICIT_CALL = ['IDENTIFIER', 'NUMBER', 'STRING', 'JS', 'REGEX', 'NEW', 'PARAM_START', 'TRY', 'DELETE', 'TYPEOF', 'SWITCH', 'TRUE', 'FALSE', 'YES', 'NO', 'ON', 'OFF', '!', '!!', 'NOT', '@', '->', '=>', '[', '(', '{'];
|
||||
// Tokens indicating that the implicit call must enclose a block of expressions.
|
||||
IMPLICIT_BLOCK = ['->', '=>', '{', '[', ','];
|
||||
// Tokens that always mark the end of an implicit call for single-liners.
|
||||
IMPLICIT_END = ['IF', 'UNLESS', 'FOR', 'WHILE', 'TERMINATOR', 'INDENT', 'OUTDENT'];
|
||||
// Single-line flavors of block expressions that have unclosed endings.
|
||||
// The grammar can't disambiguate them, so we insert the implicit indentation.
|
||||
SINGLE_LINERS = ['ELSE', "->", "=>", 'TRY', 'FINALLY', 'THEN'];
|
||||
SINGLE_CLOSERS = ['TERMINATOR', 'CATCH', 'FINALLY', 'ELSE', 'OUTDENT', 'LEADING_WHEN'];
|
||||
// Utility Functions
|
||||
// -----------------
|
||||
// Does a list include a value?
|
||||
include = function include(list, value) {
|
||||
return list.indexOf(value) >= 0;
|
||||
};
|
||||
})();
|
||||
|
|
|
|||
181
src/lexer.coffee
181
src/lexer.coffee
|
|
@ -14,99 +14,11 @@ else
|
|||
this.exports: this
|
||||
Rewriter: this.Rewriter
|
||||
|
||||
# Constants
|
||||
# ---------
|
||||
|
||||
# Keywords that CoffeeScript shares in common with JavaScript.
|
||||
JS_KEYWORDS: [
|
||||
"if", "else",
|
||||
"true", "false",
|
||||
"new", "return",
|
||||
"try", "catch", "finally", "throw",
|
||||
"break", "continue",
|
||||
"for", "in", "while",
|
||||
"delete", "instanceof", "typeof",
|
||||
"switch", "super", "extends", "class"
|
||||
]
|
||||
|
||||
# CoffeeScript-only keywords, which we're more relaxed about allowing. They can't
|
||||
# be used standalone, but you can reference them as an attached property.
|
||||
COFFEE_KEYWORDS: [
|
||||
"then", "unless",
|
||||
"yes", "no", "on", "off",
|
||||
"and", "or", "is", "isnt", "not",
|
||||
"of", "by", "where", "when"
|
||||
]
|
||||
|
||||
# The combined list of keywords is the superset that gets passed verbatim to
|
||||
# the parser.
|
||||
KEYWORDS: JS_KEYWORDS.concat COFFEE_KEYWORDS
|
||||
|
||||
# The list of keywords that are reserved by JavaScript, but not used, or are
|
||||
# used by CoffeeScript internally. We throw an error when these are encountered,
|
||||
# to avoid having a JavaScript error at runtime.
|
||||
RESERVED: [
|
||||
"case", "default", "do", "function", "var", "void", "with"
|
||||
"const", "let", "debugger", "enum", "export", "import", "native",
|
||||
"__extends", "__hasProp"
|
||||
]
|
||||
|
||||
# The superset of both JavaScript keywords and reserved words, none of which may
|
||||
# be used as identifiers or properties.
|
||||
JS_FORBIDDEN: JS_KEYWORDS.concat RESERVED
|
||||
|
||||
# Token matching regexes.
|
||||
IDENTIFIER : /^([a-zA-Z$_](\w|\$)*)/
|
||||
NUMBER : /^(\b((0(x|X)[0-9a-fA-F]+)|([0-9]+(\.[0-9]+)?(e[+\-]?[0-9]+)?)))\b/i
|
||||
HEREDOC : /^("{6}|'{6}|"{3}\n?([\s\S]*?)\n?([ \t]*)"{3}|'{3}\n?([\s\S]*?)\n?([ \t]*)'{3})/
|
||||
INTERPOLATION : /^\$([a-zA-Z_@]\w*)/
|
||||
OPERATOR : /^([+\*&|\/\-%=<>:!?]+)/
|
||||
WHITESPACE : /^([ \t]+)/
|
||||
COMMENT : /^(((\n?[ \t]*)?#[^\n]*)+)/
|
||||
CODE : /^((-|=)>)/
|
||||
REGEX : /^(\/(\S.*?)?([^\\]|\\\\)\/[imgy]{0,4})/
|
||||
MULTI_DENT : /^((\n([ \t]*))+)(\.)?/
|
||||
LAST_DENTS : /\n([ \t]*)/g
|
||||
LAST_DENT : /\n([ \t]*)/
|
||||
ASSIGNMENT : /^(:|=)$/
|
||||
|
||||
# Token cleaning regexes.
|
||||
JS_CLEANER : /(^`|`$)/g
|
||||
MULTILINER : /\n/g
|
||||
STRING_NEWLINES : /\n[ \t]*/g
|
||||
COMMENT_CLEANER : /(^[ \t]*#|\n[ \t]*$)/mg
|
||||
NO_NEWLINE : /^([+\*&|\/\-%=<>:!.\\][<>=&|]*|and|or|is|isnt|not|delete|typeof|instanceof)$/
|
||||
HEREDOC_INDENT : /^[ \t]+/mg
|
||||
|
||||
# Tokens which a regular expression will never immediately follow, but which
|
||||
# a division operator might.
|
||||
#
|
||||
# See: http://www.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions
|
||||
#
|
||||
# Our list is shorter, due to sans-parentheses method calls.
|
||||
NOT_REGEX: [
|
||||
'NUMBER', 'REGEX', '++', '--', 'FALSE', 'NULL', 'TRUE'
|
||||
]
|
||||
|
||||
# Tokens which could legitimately be invoked or indexed. A opening
|
||||
# parentheses or bracket following these tokens will be recorded as the start
|
||||
# of a function invocation or indexing operation.
|
||||
CALLABLE: ['IDENTIFIER', 'SUPER', ')', ']', '}', 'STRING', '@']
|
||||
|
||||
# Tokens that indicate an access -- keywords immediately following will be
|
||||
# treated as identifiers.
|
||||
ACCESSORS: ['PROPERTY_ACCESS', 'PROTOTYPE_ACCESS', 'SOAK_ACCESS', '@']
|
||||
|
||||
# Tokens that, when immediately preceding a `WHEN`, indicate that the `WHEN`
|
||||
# occurs at the start of a line. We disambiguate these from trailing whens to
|
||||
# avoid an ambiguity in the grammar.
|
||||
BEFORE_WHEN: ['INDENT', 'OUTDENT', 'TERMINATOR']
|
||||
|
||||
# The Lexer Class
|
||||
# ---------------
|
||||
|
||||
# The Lexer class reads a stream of CoffeeScript and divvys it up into tagged
|
||||
# tokens. A minor bit of the ambiguity in the grammar has been avoided by
|
||||
# tokens. Some potential ambiguity in the grammar has been avoided by
|
||||
# pushing some extra smarts into the Lexer.
|
||||
exports.Lexer: class Lexer
|
||||
|
||||
|
|
@ -119,6 +31,9 @@ exports.Lexer: class Lexer
|
|||
# Each tokenizing method is responsible for incrementing `@i` by the number of
|
||||
# characters it has consumed. `@i` can be thought of as our finger on the page
|
||||
# of source.
|
||||
#
|
||||
# Before returning the token stream, run it through the [Rewriter](rewriter.html)
|
||||
# unless explicitly asked not to.
|
||||
tokenize: (code, options) ->
|
||||
o : options or {}
|
||||
@code : code # The remainder of the source code.
|
||||
|
|
@ -480,6 +395,94 @@ exports.Lexer: class Lexer
|
|||
return false unless m: @chunk.match(regex)
|
||||
if m then m[index] else false
|
||||
|
||||
# Constants
|
||||
# ---------
|
||||
|
||||
# Keywords that CoffeeScript shares in common with JavaScript.
|
||||
JS_KEYWORDS: [
|
||||
"if", "else",
|
||||
"true", "false",
|
||||
"new", "return",
|
||||
"try", "catch", "finally", "throw",
|
||||
"break", "continue",
|
||||
"for", "in", "while",
|
||||
"delete", "instanceof", "typeof",
|
||||
"switch", "super", "extends", "class"
|
||||
]
|
||||
|
||||
# CoffeeScript-only keywords, which we're more relaxed about allowing. They can't
|
||||
# be used standalone, but you can reference them as an attached property.
|
||||
COFFEE_KEYWORDS: [
|
||||
"then", "unless",
|
||||
"yes", "no", "on", "off",
|
||||
"and", "or", "is", "isnt", "not",
|
||||
"of", "by", "where", "when"
|
||||
]
|
||||
|
||||
# The combined list of keywords is the superset that gets passed verbatim to
|
||||
# the parser.
|
||||
KEYWORDS: JS_KEYWORDS.concat COFFEE_KEYWORDS
|
||||
|
||||
# The list of keywords that are reserved by JavaScript, but not used, or are
|
||||
# used by CoffeeScript internally. We throw an error when these are encountered,
|
||||
# to avoid having a JavaScript error at runtime.
|
||||
RESERVED: [
|
||||
"case", "default", "do", "function", "var", "void", "with"
|
||||
"const", "let", "debugger", "enum", "export", "import", "native",
|
||||
"__extends", "__hasProp"
|
||||
]
|
||||
|
||||
# The superset of both JavaScript keywords and reserved words, none of which may
|
||||
# be used as identifiers or properties.
|
||||
JS_FORBIDDEN: JS_KEYWORDS.concat RESERVED
|
||||
|
||||
# Token matching regexes.
|
||||
IDENTIFIER : /^([a-zA-Z$_](\w|\$)*)/
|
||||
NUMBER : /^(\b((0(x|X)[0-9a-fA-F]+)|([0-9]+(\.[0-9]+)?(e[+\-]?[0-9]+)?)))\b/i
|
||||
HEREDOC : /^("{6}|'{6}|"{3}\n?([\s\S]*?)\n?([ \t]*)"{3}|'{3}\n?([\s\S]*?)\n?([ \t]*)'{3})/
|
||||
INTERPOLATION : /^\$([a-zA-Z_@]\w*)/
|
||||
OPERATOR : /^([+\*&|\/\-%=<>:!?]+)/
|
||||
WHITESPACE : /^([ \t]+)/
|
||||
COMMENT : /^(((\n?[ \t]*)?#[^\n]*)+)/
|
||||
CODE : /^((-|=)>)/
|
||||
REGEX : /^(\/(\S.*?)?([^\\]|\\\\)\/[imgy]{0,4})/
|
||||
MULTI_DENT : /^((\n([ \t]*))+)(\.)?/
|
||||
LAST_DENTS : /\n([ \t]*)/g
|
||||
LAST_DENT : /\n([ \t]*)/
|
||||
ASSIGNMENT : /^(:|=)$/
|
||||
|
||||
# Token cleaning regexes.
|
||||
JS_CLEANER : /(^`|`$)/g
|
||||
MULTILINER : /\n/g
|
||||
STRING_NEWLINES : /\n[ \t]*/g
|
||||
COMMENT_CLEANER : /(^[ \t]*#|\n[ \t]*$)/mg
|
||||
NO_NEWLINE : /^([+\*&|\/\-%=<>:!.\\][<>=&|]*|and|or|is|isnt|not|delete|typeof|instanceof)$/
|
||||
HEREDOC_INDENT : /^[ \t]+/mg
|
||||
|
||||
# Tokens which a regular expression will never immediately follow, but which
|
||||
# a division operator might.
|
||||
#
|
||||
# See: http://www.mozilla.org/js/language/js20-2002-04/rationale/syntax.html#regular-expressions
|
||||
#
|
||||
# Our list is shorter, due to sans-parentheses method calls.
|
||||
NOT_REGEX: [
|
||||
'NUMBER', 'REGEX', '++', '--', 'FALSE', 'NULL', 'TRUE'
|
||||
]
|
||||
|
||||
# Tokens which could legitimately be invoked or indexed. A opening
|
||||
# parentheses or bracket following these tokens will be recorded as the start
|
||||
# of a function invocation or indexing operation.
|
||||
CALLABLE: ['IDENTIFIER', 'SUPER', ')', ']', '}', 'STRING', '@']
|
||||
|
||||
# Tokens that indicate an access -- keywords immediately following will be
|
||||
# treated as identifiers.
|
||||
ACCESSORS: ['PROPERTY_ACCESS', 'PROTOTYPE_ACCESS', 'SOAK_ACCESS', '@']
|
||||
|
||||
# Tokens that, when immediately preceding a `WHEN`, indicate that the `WHEN`
|
||||
# occurs at the start of a line. We disambiguate these from trailing whens to
|
||||
# avoid an ambiguity in the grammar.
|
||||
BEFORE_WHEN: ['INDENT', 'OUTDENT', 'TERMINATOR']
|
||||
|
||||
# Utility Functions
|
||||
# -----------------
|
||||
|
||||
|
|
|
|||
|
|
@ -1,53 +1,28 @@
|
|||
# The CoffeeScript language has a decent amount of optional syntax,
|
||||
# implicit syntax, and shorthand syntax. These things can greatly complicate a
|
||||
# grammar and bloat the resulting parse table. Instead of making the parser
|
||||
# handle it all, we take a series of passes over the token stream,
|
||||
# using this **Rewriter** to convert shorthand into the unambiguous long form,
|
||||
# add implicit indentation and parentheses, balance incorrect nestings, and
|
||||
# generally clean things up.
|
||||
|
||||
# Set up exported variables for both Node.js and the browser.
|
||||
this.exports: this unless process?
|
||||
|
||||
# Tokens that must be balanced.
|
||||
BALANCED_PAIRS: [['(', ')'], ['[', ']'], ['{', '}'], ['INDENT', 'OUTDENT'],
|
||||
['PARAM_START', 'PARAM_END'], ['CALL_START', 'CALL_END'],
|
||||
['INDEX_START', 'INDEX_END'], ['SOAKED_INDEX_START', 'SOAKED_INDEX_END']]
|
||||
|
||||
# Tokens that signal the start of a balanced pair.
|
||||
EXPRESSION_START: pair[0] for pair in BALANCED_PAIRS
|
||||
|
||||
# Tokens that signal the end of a balanced pair.
|
||||
EXPRESSION_TAIL: pair[1] for pair in BALANCED_PAIRS
|
||||
|
||||
# Tokens that indicate the close of a clause of an expression.
|
||||
EXPRESSION_CLOSE: ['CATCH', 'WHEN', 'ELSE', 'FINALLY'].concat(EXPRESSION_TAIL)
|
||||
|
||||
# Tokens pairs that, in immediate succession, indicate an implicit call.
|
||||
IMPLICIT_FUNC: ['IDENTIFIER', 'SUPER', ')', 'CALL_END', ']', 'INDEX_END']
|
||||
IMPLICIT_BLOCK:['->', '=>', '{', '[', ',']
|
||||
IMPLICIT_END: ['IF', 'UNLESS', 'FOR', 'WHILE', 'TERMINATOR', 'INDENT', 'OUTDENT']
|
||||
IMPLICIT_CALL: ['IDENTIFIER', 'NUMBER', 'STRING', 'JS', 'REGEX', 'NEW', 'PARAM_START',
|
||||
'TRY', 'DELETE', 'TYPEOF', 'SWITCH',
|
||||
'TRUE', 'FALSE', 'YES', 'NO', 'ON', 'OFF', '!', '!!', 'NOT',
|
||||
'@', '->', '=>', '[', '(', '{']
|
||||
|
||||
# The inverse mappings of token pairs we're trying to fix up.
|
||||
INVERSES: {}
|
||||
for pair in BALANCED_PAIRS
|
||||
INVERSES[pair[0]]: pair[1]
|
||||
INVERSES[pair[1]]: pair[0]
|
||||
|
||||
# Single-line flavors of block expressions that have unclosed endings.
|
||||
# The grammar can't disambiguate them, so we insert the implicit indentation.
|
||||
SINGLE_LINERS: ['ELSE', "->", "=>", 'TRY', 'FINALLY', 'THEN']
|
||||
SINGLE_CLOSERS: ['TERMINATOR', 'CATCH', 'FINALLY', 'ELSE', 'OUTDENT', 'LEADING_WHEN']
|
||||
|
||||
# In order to keep the grammar simple, the stream of tokens that the Lexer
|
||||
# emits is rewritten by the Rewriter, smoothing out ambiguities, mis-nested
|
||||
# indentation, and single-line flavors of expressions.
|
||||
# The **Rewriter** class is used by the [Lexer](lexer.html), directly against
|
||||
# its internal array of tokens.
|
||||
exports.Rewriter: class Rewriter
|
||||
|
||||
# Rewrite the token stream in multiple passes, one logical filter at
|
||||
# a time. This could certainly be changed into a single pass through the
|
||||
# stream, with a big ol' efficient switch, but it's much nicer like this.
|
||||
# stream, with a big ol' efficient switch, but it's much nicer to work with
|
||||
# like this. The order of these passes matters -- indentation must be
|
||||
# corrected before implicit parentheses can be wrapped around blocks of code.
|
||||
rewrite: (tokens) ->
|
||||
@tokens: tokens
|
||||
@adjust_comments()
|
||||
@remove_leading_newlines()
|
||||
@remove_mid_expression_newlines()
|
||||
@move_commas_outside_outdents()
|
||||
@close_open_calls_and_indexes()
|
||||
@add_implicit_indentation()
|
||||
@add_implicit_parentheses()
|
||||
|
|
@ -58,7 +33,8 @@ exports.Rewriter: class Rewriter
|
|||
# Rewrite the token stream, looking one token ahead and behind.
|
||||
# Allow the return value of the block to tell us how many tokens to move
|
||||
# forwards (or backwards) in the stream, to make sure we don't miss anything
|
||||
# as the stream changes length under our feet.
|
||||
# as tokens are inserted and removed, and the stream changes length under
|
||||
# our feet.
|
||||
scan_tokens: (block) ->
|
||||
i: 0
|
||||
while true
|
||||
|
|
@ -68,7 +44,7 @@ exports.Rewriter: class Rewriter
|
|||
true
|
||||
|
||||
# Massage newlines and indentations so that comments don't have to be
|
||||
# correctly indented, or appear on their own line.
|
||||
# correctly indented, or appear on a line of their own.
|
||||
adjust_comments: ->
|
||||
@scan_tokens (prev, token, post, i) =>
|
||||
return 1 unless token[0] is 'COMMENT'
|
||||
|
|
@ -86,25 +62,19 @@ exports.Rewriter: class Rewriter
|
|||
# Leading newlines would introduce an ambiguity in the grammar, so we
|
||||
# dispatch them here.
|
||||
remove_leading_newlines: ->
|
||||
@tokens.shift() if @tokens[0][0] is 'TERMINATOR'
|
||||
@tokens.shift() while @tokens[0][0] is 'TERMINATOR'
|
||||
|
||||
# Some blocks occur in the middle of expressions -- when we're expecting
|
||||
# this, remove their trailing newlines.
|
||||
remove_mid_expression_newlines: ->
|
||||
@scan_tokens (prev, token, post, i) =>
|
||||
return 1 unless post and EXPRESSION_CLOSE.indexOf(post[0]) >= 0 and token[0] is 'TERMINATOR'
|
||||
return 1 unless post and include(EXPRESSION_CLOSE, post[0]) and token[0] is 'TERMINATOR'
|
||||
@tokens.splice(i, 1)
|
||||
return 0
|
||||
|
||||
# Make sure that we don't accidentally break trailing commas, which need
|
||||
# to go on the outside of expression closers.
|
||||
move_commas_outside_outdents: ->
|
||||
@scan_tokens (prev, token, post, i) =>
|
||||
@tokens.splice(i, 1, token) if token[0] is 'OUTDENT' and prev[0] is ','
|
||||
return 1
|
||||
|
||||
# We've tagged the opening parenthesis of a method call, and the opening
|
||||
# bracket of an indexing operation. Match them with their close.
|
||||
# The lexer has tagged the opening parenthesis of a method call, and the
|
||||
# opening bracket of an indexing operation. Match them with their paired
|
||||
# close.
|
||||
close_open_calls_and_indexes: ->
|
||||
parens: [0]
|
||||
brackets: [0]
|
||||
|
|
@ -139,28 +109,29 @@ exports.Rewriter: class Rewriter
|
|||
if tag is 'OUTDENT'
|
||||
last: stack.pop()
|
||||
stack[stack.length - 1] += last
|
||||
if IMPLICIT_END.indexOf(tag) >= 0 or !post?
|
||||
return 1 if tag is 'INDENT' and prev and IMPLICIT_BLOCK.indexOf(prev[0]) >= 0
|
||||
if !post? or include IMPLICIT_END, tag
|
||||
return 1 if tag is 'INDENT' and prev and include IMPLICIT_BLOCK, prev[0]
|
||||
if stack[stack.length - 1] > 0 or tag is 'INDENT'
|
||||
idx: if tag is 'OUTDENT' then i +1 else i
|
||||
idx: if tag is 'OUTDENT' then i + 1 else i
|
||||
stack_pointer: if tag is 'INDENT' then 2 else 1
|
||||
for tmp in [0...stack[stack.length - stack_pointer]]
|
||||
@tokens.splice(idx, 0, ['CALL_END', ')', token[2]])
|
||||
size: stack[stack.length - stack_pointer] + 1
|
||||
stack[stack.length - stack_pointer]: 0
|
||||
return size
|
||||
return 1 unless prev and IMPLICIT_FUNC.indexOf(prev[0]) >= 0 and IMPLICIT_CALL.indexOf(tag) >= 0
|
||||
return 1 unless prev and include(IMPLICIT_FUNC, prev[0]) and include IMPLICIT_CALL, tag
|
||||
@tokens.splice(i, 0, ['CALL_START', '(', token[2]])
|
||||
stack[stack.length - 1] += 1
|
||||
return 2
|
||||
|
||||
# Because our grammar is LALR(1), it can't handle some single-line
|
||||
# expressions that lack ending delimiters. Use the lexer to add the implicit
|
||||
# blocks, so it doesn't need to.
|
||||
# ')' can close a single-line block, but we need to make sure it's balanced.
|
||||
# expressions that lack ending delimiters. The **Rewriter** adds the implicit
|
||||
# blocks, so it doesn't need to. ')' can close a single-line block,
|
||||
# but we need to make sure it's balanced.
|
||||
add_implicit_indentation: ->
|
||||
@scan_tokens (prev, token, post, i) =>
|
||||
return 1 unless SINGLE_LINERS.indexOf(token[0]) >= 0 and post[0] isnt 'INDENT' and
|
||||
return 1 unless include(SINGLE_LINERS, token[0]) and
|
||||
post[0] isnt 'INDENT' and
|
||||
not (token[0] is 'ELSE' and post[0] is 'IF')
|
||||
starter: token[0]
|
||||
@tokens.splice(i + 1, 0, ['INDENT', 2, token[2]])
|
||||
|
|
@ -171,7 +142,7 @@ exports.Rewriter: class Rewriter
|
|||
tok: @tokens[idx]
|
||||
pre: @tokens[idx - 1]
|
||||
if (not tok or
|
||||
(SINGLE_CLOSERS.indexOf(tok[0]) >= 0 and tok[1] isnt ';') or
|
||||
(include(SINGLE_CLOSERS, tok[0]) and tok[1] isnt ';') or
|
||||
(tok[0] is ')' && parens is 0)) and
|
||||
not (starter is 'ELSE' and tok[0] is 'ELSE')
|
||||
insertion: if pre[0] is "," then idx - 1 else idx
|
||||
|
|
@ -199,8 +170,10 @@ exports.Rewriter: class Rewriter
|
|||
throw new Error("unclosed ${unclosed[0]}") if unclosed.length
|
||||
|
||||
# We'd like to support syntax like this:
|
||||
# el.click((event) ->
|
||||
# el.hide())
|
||||
#
|
||||
# el.click((event) ->
|
||||
# el.hide())
|
||||
#
|
||||
# In order to accomplish this, move outdents that follow closing parens
|
||||
# inwards, safely. The steps to accomplish this are:
|
||||
#
|
||||
|
|
@ -218,29 +191,72 @@ exports.Rewriter: class Rewriter
|
|||
@scan_tokens (prev, token, post, i) =>
|
||||
tag: token[0]
|
||||
inv: INVERSES[token[0]]
|
||||
# Push openers onto the stack.
|
||||
if EXPRESSION_START.indexOf(tag) >= 0
|
||||
stack.push(token)
|
||||
if include EXPRESSION_START, tag
|
||||
stack.push token
|
||||
return 1
|
||||
# The end of an expression, check stack and debt for a pair.
|
||||
else if EXPRESSION_TAIL.indexOf(tag) >= 0
|
||||
# If the tag is already in our debt, swallow it.
|
||||
else if include EXPRESSION_END, tag
|
||||
if debt[inv] > 0
|
||||
debt[inv] -= 1
|
||||
@tokens.splice(i, 1)
|
||||
@tokens.splice i, 1
|
||||
return 0
|
||||
else
|
||||
# Pop the stack of open delimiters.
|
||||
match: stack.pop()
|
||||
mtag: match[0]
|
||||
# Continue onwards if it's the expected tag.
|
||||
if tag is INVERSES[mtag]
|
||||
return 1
|
||||
else
|
||||
# Unexpected close, insert correct close, adding to the debt.
|
||||
debt[mtag] += 1
|
||||
val: if mtag is 'INDENT' then match[1] else INVERSES[mtag]
|
||||
@tokens.splice(i, 0, [INVERSES[mtag], val])
|
||||
return 1
|
||||
return 1 if tag is INVERSES[mtag]
|
||||
debt[mtag] += 1
|
||||
val: if mtag is 'INDENT' then match[1] else INVERSES[mtag]
|
||||
@tokens.splice i, 0, [INVERSES[mtag], val]
|
||||
return 1
|
||||
else
|
||||
return 1
|
||||
|
||||
# Constants
|
||||
# ---------
|
||||
|
||||
# List of the token pairs that must be balanced.
|
||||
BALANCED_PAIRS: [['(', ')'], ['[', ']'], ['{', '}'], ['INDENT', 'OUTDENT'],
|
||||
['PARAM_START', 'PARAM_END'], ['CALL_START', 'CALL_END'],
|
||||
['INDEX_START', 'INDEX_END'], ['SOAKED_INDEX_START', 'SOAKED_INDEX_END']]
|
||||
|
||||
# The inverse mappings of `BALANCED_PAIRS` we're trying to fix up, so we can
|
||||
# look things up from either end.
|
||||
INVERSES: {}
|
||||
for pair in BALANCED_PAIRS
|
||||
INVERSES[pair[0]]: pair[1]
|
||||
INVERSES[pair[1]]: pair[0]
|
||||
|
||||
# The tokens that signal the start of a balanced pair.
|
||||
EXPRESSION_START: pair[0] for pair in BALANCED_PAIRS
|
||||
|
||||
# The tokens that signal the end of a balanced pair.
|
||||
EXPRESSION_END: pair[1] for pair in BALANCED_PAIRS
|
||||
|
||||
# Tokens that indicate the close of a clause of an expression.
|
||||
EXPRESSION_CLOSE: ['CATCH', 'WHEN', 'ELSE', 'FINALLY'].concat EXPRESSION_END
|
||||
|
||||
# Tokens that, if followed by an `IMPLICIT_CALL`, indicate a function invocation.
|
||||
IMPLICIT_FUNC: ['IDENTIFIER', 'SUPER', ')', 'CALL_END', ']', 'INDEX_END']
|
||||
|
||||
# If preceded by an `IMPLICIT_FUNC`, indicates a function invocation.
|
||||
IMPLICIT_CALL: ['IDENTIFIER', 'NUMBER', 'STRING', 'JS', 'REGEX', 'NEW', 'PARAM_START',
|
||||
'TRY', 'DELETE', 'TYPEOF', 'SWITCH',
|
||||
'TRUE', 'FALSE', 'YES', 'NO', 'ON', 'OFF', '!', '!!', 'NOT',
|
||||
'@', '->', '=>', '[', '(', '{']
|
||||
|
||||
# Tokens indicating that the implicit call must enclose a block of expressions.
|
||||
IMPLICIT_BLOCK: ['->', '=>', '{', '[', ',']
|
||||
|
||||
# Tokens that always mark the end of an implicit call for single-liners.
|
||||
IMPLICIT_END: ['IF', 'UNLESS', 'FOR', 'WHILE', 'TERMINATOR', 'INDENT', 'OUTDENT']
|
||||
|
||||
# Single-line flavors of block expressions that have unclosed endings.
|
||||
# The grammar can't disambiguate them, so we insert the implicit indentation.
|
||||
SINGLE_LINERS: ['ELSE', "->", "=>", 'TRY', 'FINALLY', 'THEN']
|
||||
SINGLE_CLOSERS: ['TERMINATOR', 'CATCH', 'FINALLY', 'ELSE', 'OUTDENT', 'LEADING_WHEN']
|
||||
|
||||
# Utility Functions
|
||||
# -----------------
|
||||
|
||||
# Does a list include a value?
|
||||
include: (list, value) ->
|
||||
list.indexOf(value) >= 0
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
|
||||
|
||||
func: ->
|
||||
a: 3
|
||||
b: []
|
||||
|
|
|
|||
Loading…
Reference in New Issue