From ea4f64c926e52bbfa7f8a924b7d7f18675dcb579 Mon Sep 17 00:00:00 2001 From: Nikolai Mavrenkov Date: Wed, 5 Feb 2020 20:06:47 +0300 Subject: [PATCH 1/2] Add support for floating numbers separator --- src/tokenizer.ts | 19 +++++++++++++------ src/util/charcode.ts | 6 +++++- tests/parser/literals.ts | 1 + tests/parser/literals.ts.fixture.ts | 1 + 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index d160bffcde..33e6ba1b1a 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -26,7 +26,8 @@ import { isIdentifierPart, isDecimalDigit, isOctalDigit, - isKeywordCharacter + isKeywordCharacter, + isDecimalSeparator, } from "./util"; /** Named token types. */ @@ -1499,16 +1500,20 @@ export class Tokenizer extends DiagnosticEmitter { } readDecimalFloat(): f64 { - // TODO: numeric separators (parseFloat can't handle these) var start = this.pos; var end = this.end; var text = this.source.text; - while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) { + + while (this.pos < end) { + let ch = text.charCodeAt(this.pos); + if (!(isDecimalDigit(ch) || isDecimalSeparator(ch))) break; ++this.pos; } if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) { ++this.pos; - while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) { + while (this.pos < end) { + let ch = text.charCodeAt(this.pos); + if (!(isDecimalDigit(ch) || isDecimalSeparator(ch))) break; ++this.pos; } } @@ -1522,12 +1527,14 @@ export class Tokenizer extends DiagnosticEmitter { ) { ++this.pos; } - while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) { + while (this.pos < end) { + let ch = text.charCodeAt(this.pos); + if (!(isDecimalDigit(ch) || isDecimalSeparator(ch))) break; ++this.pos; } } } - return parseFloat(text.substring(start, this.pos)); + return parseFloat(text.substring(start, this.pos).replace(/_/g, '')); } readHexFloat(): f64 { diff --git a/src/util/charcode.ts b/src/util/charcode.ts index 318b571430..bfe181e099 100644 --- a/src/util/charcode.ts +++ b/src/util/charcode.ts @@ -174,7 +174,11 @@ export function isWhiteSpace(c: i32): bool { /** Tests if the specified character code is a valid decimal digit. */ export function isDecimalDigit(c: i32): bool { - return c >= CharCode._0 && c <= CharCode._9; + return c >= CharCode._0 && c <= CharCode._9 ; +} + +export function isDecimalSeparator(c: i32): bool { + return c === CharCode._; } /** Tests if the specified character code is a valid octal digit. */ diff --git a/tests/parser/literals.ts b/tests/parser/literals.ts index 1ea260c480..6b943a9b05 100644 --- a/tests/parser/literals.ts +++ b/tests/parser/literals.ts @@ -60,3 +60,4 @@ "1\"23"; "1\"2\\3"; "\0\n\\n\r"; +1000_000.1234_1234; diff --git a/tests/parser/literals.ts.fixture.ts b/tests/parser/literals.ts.fixture.ts index dde9856d2d..6f3b361c40 100644 --- a/tests/parser/literals.ts.fixture.ts +++ b/tests/parser/literals.ts.fixture.ts @@ -60,3 +60,4 @@ "1\"23"; "1\"2\\3"; "\0\n\\n\r"; +1000000.12341234; From e7fc6f9371029667d60d514729bb60d95d95e2b4 Mon Sep 17 00:00:00 2001 From: Nikolai Mavrenkov Date: Thu, 6 Feb 2020 16:35:22 +0300 Subject: [PATCH 2/2] Add Nikolai Mavrenkov to the contributors list --- NOTICE | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NOTICE b/NOTICE index dbba237d5e..13a25d3acf 100644 --- a/NOTICE +++ b/NOTICE @@ -19,6 +19,7 @@ under the licensing terms detailed in LICENSE: * jhwgh1968 * Jeffrey Charles * Vladimir Tikhonov +* Nikolai Mavrenkov Portions of this software are derived from third-party works licensed under the following terms: @@ -44,6 +45,6 @@ the following terms: The 3-Clause BSD License (https://opensource.org/licenses/BSD-3-Clause) * Arm Optimized Routines: https://github.com/ARM-software/optimized-routines - + Copyright (c) Arm Limited The MIT License (https://opensource.org/licenses/MIT)