Numeric literal separators (#5215)

* implement numeric literal separators

* add tests

* Revert changes to package-lock.json

* small regex adjustment

* split tests

* add comment

* Add Node versions to CI

* Fix #5103: Add support for BigInt literals (#5104)

* Fix #5103: Add support for BigInt literals

* Fix typos found in testing

* Support binary, octal and hex BigInt literals

* Make decimal BigInt test consistent other bases

* Correct test BigInt test names

* Add Node versions to CI

* Update output

* Fix style

* support bigint literal with separators

* un-disallow property access on number literal

* Update output

* Refactor numeric literal separator tests to be more like the rest of the tests

* Add test for numeric property with underscore

Co-authored-by: Geoffrey Booth <GeoffreyBooth@users.noreply.github.com>
Co-authored-by: Robert de Forest <guitar.robot@gmail.com>

Cakefile

@@ -472,6 +472,7 @@ runTests = (CoffeeScript) ->
   skipUnless 'var a = 2 ** 2; a **= 3', ['exponentiation.coffee']
   skipUnless 'var {...a} = {}', ['object_rest_spread.coffee']
   skipUnless '/foo.bar/s.test("foo\tbar")', ['regex_dotall.coffee']
+  skipUnless '1_2_3', ['numeric_literal_separators.coffee']
   skipUnless '1n', ['numbers_bigint.coffee']
   files = fs.readdirSync('test').filter (filename) ->
     filename not in testFilesToSkip

lib/coffeescript/lexer.js

@@ -1570,7 +1570,9 @@
   // Is this an attribute with a value?
 }(?:\\s*:\\s*${CSX_IDENTIFIER_PART})?)([^\\S]*=(?!=))?`);
 
-  NUMBER = /^0b[01]+n?|^0o[0-7]+n?|^0x[\da-f]+n?|^\d+n|^\d*\.?\d+(?:e[+-]?\d+)?/i; // binary
+  // decimal without support for numeric literal separators for reference:
+  // \d*\.?\d+ (?:e[+-]?\d+)?
+  NUMBER = /^0b[01](?:_?[01])*n?|^0o[0-7](?:_?[0-7])*n?|^0x[\da-f](?:_?[\da-f])*n?|^\d+n|^(?:\d(?:_?\d)*)?\.?(?:\d(?:_?\d)*)+(?:e[+-]?(?:\d(?:_?\d)*)+)?/i; // binary
   // octal
   // hex
   // decimal bigint

src/lexer.coffee

@@ -1212,11 +1212,14 @@ CSX_ATTRIBUTE = /// ^
 ///
 
 NUMBER     = ///
-  ^ 0b[01]+n?    |            # binary
-  ^ 0o[0-7]+n?   |            # octal
-  ^ 0x[\da-f]+n? |            # hex
-  ^ \d+n         |            # decimal bigint
-  ^ \d*\.?\d+ (?:e[+-]?\d+)?  # decimal
+  ^ 0b[01](?:_?[01])*n?                         | # binary
+  ^ 0o[0-7](?:_?[0-7])*n?                       | # octal
+  ^ 0x[\da-f](?:_?[\da-f])*n?                   | # hex
+  ^ \d+n                                        | # decimal bigint
+  ^ (?:\d(?:_?\d)*)?    \.?   (?:\d(?:_?\d)*)+    # decimal
+                    (?:e[+-]? (?:\d(?:_?\d)*)+ )?
+  # decimal without support for numeric literal separators for reference:
+  # \d*\.?\d+ (?:e[+-]?\d+)?
 ///i
 
 OPERATOR   = /// ^ (

test/numeric_literal_separators.coffee

@@ -0,0 +1,55 @@
+# Numeric Literal Separators
+# --------------------------
+
+test 'integer literals with separators', ->
+  eq 123_456, 123456
+  eq 12_34_56, 123456
+
+test 'decimal literals with separators', ->
+  eq 1_2.34_5, 12.345
+  eq 1_0e1_0, 10e10
+  eq 1_2.34_5e6_7, 12.345e67
+
+test 'hexadecimal literals with separators', ->
+  eq 0x1_2_3_4, 0x1234
+
+test 'binary literals with separators', ->
+  eq 0b10_10, 0b1010
+
+test 'octal literals with separators', ->
+  eq 0o7_7_7, 0o777
+
+test 'property access on a number', ->
+  # Somehow, `3..toFixed()` is valid JavaScript; though just `3.toFixed()`
+  # is not. CoffeeScript has long allowed code like `3.toFixed()` to compile
+  # into `3..toFixed()`.
+  eq 3.toFixed(), '3'
+  # Where this can conflict with numeric literal separators is when the
+  # property name contains an underscore.
+  Number::_23 = _23 = 'x'
+  eq 1._23, 'x'
+  ok 1._34 is undefined
+  delete Number::_23
+
+test 'invalid decimal literal separators do not compile', ->
+  # `1._23` is a valid property access (see previous test)
+  throws -> CoffeeScript.compile '1_.23'
+  throws -> CoffeeScript.compile '1e_2'
+  throws -> CoffeeScript.compile '1e2_'
+  throws -> CoffeeScript.compile '1_'
+  throws -> CoffeeScript.compile '1__2'
+
+test 'invalid hexadecimal literal separators do not compile', ->
+  throws -> CoffeeScript.compile '0x_1234'
+  throws -> CoffeeScript.compile '0x1234_'
+  throws -> CoffeeScript.compile '0x1__34'
+
+test 'invalid binary literal separators do not compile', ->
+  throws -> CoffeeScript.compile '0b_100'
+  throws -> CoffeeScript.compile '0b100_'
+  throws -> CoffeeScript.compile '0b1__1'
+
+test 'invalid octal literal separators do not compile', ->
+  throws -> CoffeeScript.compile '0o_777'
+  throws -> CoffeeScript.compile '0o777_'
+  throws -> CoffeeScript.compile '0o6__6'