#1059: Add support for floating numbers separator

NOTICE

@@ -19,6 +19,7 @@ under the licensing terms detailed in LICENSE:
 * jhwgh1968 <jhwgh1968@protonmail.com>
 * Jeffrey Charles <jeffreycharles@gmail.com>
 * Vladimir Tikhonov <reg@tikhonov.by>
+* Nikolai Mavrenkov <koluch@koluch.ru>
 
 Portions of this software are derived from third-party works licensed under
 the following terms:
@@ -44,6 +45,6 @@ the following terms:
   The 3-Clause BSD License (https://opensource.org/licenses/BSD-3-Clause)
 
 * Arm Optimized Routines: https://github.com/ARM-software/optimized-routines
-  
+
   Copyright (c) Arm Limited
   The MIT License (https://opensource.org/licenses/MIT)

src/tokenizer.ts

@@ -26,7 +26,8 @@ import {
   isIdentifierPart,
   isDecimalDigit,
   isOctalDigit,
-  isKeywordCharacter
+  isKeywordCharacter,
+  isDecimalSeparator,
 } from "./util";
 
 /** Named token types. */
@@ -1499,16 +1500,20 @@ export class Tokenizer extends DiagnosticEmitter {
   }
 
   readDecimalFloat(): f64 {
-    // TODO: numeric separators (parseFloat can't handle these)
     var start = this.pos;
     var end = this.end;
     var text = this.source.text;
-    while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) {
+
+    while (this.pos < end) {
+      let ch = text.charCodeAt(this.pos);
+      if (!(isDecimalDigit(ch) || isDecimalSeparator(ch))) break;
       ++this.pos;
     }
     if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) {
       ++this.pos;
-      while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) {
+      while (this.pos < end) {
+        let ch = text.charCodeAt(this.pos);
+        if (!(isDecimalDigit(ch) || isDecimalSeparator(ch))) break;
         ++this.pos;
       }
     }
@@ -1522,12 +1527,14 @@ export class Tokenizer extends DiagnosticEmitter {
         ) {
           ++this.pos;
         }
-        while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) {
+        while (this.pos < end) {
+          let ch = text.charCodeAt(this.pos);
+          if (!(isDecimalDigit(ch) || isDecimalSeparator(ch))) break;
           ++this.pos;
         }
       }
     }
-    return parseFloat(text.substring(start, this.pos));
+    return parseFloat(text.substring(start, this.pos).replace(/_/g, ''));
   }
 
   readHexFloat(): f64 {

src/util/charcode.ts

@@ -174,7 +174,11 @@ export function isWhiteSpace(c: i32): bool {
 
 /** Tests if the specified character code is a valid decimal digit. */
 export function isDecimalDigit(c: i32): bool {
-  return c >= CharCode._0 && c <= CharCode._9;
+  return c >= CharCode._0 && c <= CharCode._9 ;
+}
+
+export function isDecimalSeparator(c: i32): bool {
+  return c === CharCode._;
 }
 
 /** Tests if the specified character code is a valid octal digit. */

tests/parser/literals.ts

@@ -60,3 +60,4 @@
 "1\"23";
 "1\"2\\3";
 "\0\n\\n\r";
+1000_000.1234_1234;

tests/parser/literals.ts.fixture.ts

@@ -60,3 +60,4 @@
 "1\"23";
 "1\"2\\3";
 "\0\n\\n\r";
+1000000.12341234;