Skip to content
This repository has been archived by the owner on Dec 15, 2022. It is now read-only.

Commit

Permalink
fix: add numeric separators
Browse files Browse the repository at this point in the history
  • Loading branch information
@UziTech
UziTech committed Nov 20, 2020
1 parent 16b9f3e commit 7a1e4f0
Show file tree
Hide file tree
Showing 2 changed files with 162 additions and 21 deletions.
67 changes: 46 additions & 21 deletions grammars/javascript.cson
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1245,45 +1245,70 @@
'numbers':
'patterns': [
{
'match': '\\b(?<!\\$)0(x|X)[0-9a-fA-F]+n?\\b(?!\\$)'
'match': '\\b(?<!\\$)0(x|X)[0-9a-fA-F]+(?:_[0-9a-fA-F]+)*n?\\b(?!\\$)'
'name': 'constant.numeric.hex.js'
'captures':
'0':
'patterns': [
{
'include': '#numeric_separators'
}
]
}
{
'match': '\\b(?<!\\$)0(b|B)[01]+n?\\b(?!\\$)'
'match': '\\b(?<!\\$)0(b|B)[01]+(?:_[01]+)*n?\\b(?!\\$)'
'name': 'constant.numeric.binary.js'
'captures':
'0':
'patterns': [
{
'include': '#numeric_separators'
}
]
}
{
'match': '\\b(?<!\\$)0(o|O)?[0-7]+n?\\b(?!\\$)'
'match': '\\b(?<!\\$)0(o|O)?[0-7]+(?:_[0-7]+)*n?\\b(?!\\$)'
'name': 'constant.numeric.octal.js'
'captures':
'0':
'patterns': [
{
'include': '#numeric_separators'
}
]
}
{
'match': '''(?x)
(?<!\\$)(?:
(?:\\b[0-9]+(\\.)[0-9]+[eE][+-]?[0-9]+\\b)| # 1.1E+3
(?:\\b[0-9]+(\\.)[eE][+-]?[0-9]+\\b)| # 1.E+3
(?:\\B(\\.)[0-9]+[eE][+-]?[0-9]+\\b)| # .1E+3
(?:\\b[0-9]+[eE][+-]?[0-9]+\\b)| # 1E+3
(?:\\b[0-9]+(\\.)[0-9]+\\b)| # 1.1
(?:\\b[0-9]+(\\.)\\B)| # 1.
(?:\\B(\\.)[0-9]+\\b)| # .1
(?:\\b[0-9]+n?\\b(?!\\.)) # 1n
(?:\\b[0-9]+(?:_[0-9]+)*\\.[0-9]+(?:_[0-9]+)*[eE][+-]?[0-9]+(?:_[0-9]+)*\\b)| # 1.1E+3
(?:\\b[0-9]+(?:_[0-9]+)*\\.[eE][+-]?[0-9]+(?:_[0-9]+)*\\b)| # 1.E+3
(?:\\B\\.[0-9]+(?:_[0-9]+)*[eE][+-]?[0-9]+(?:_[0-9]+)*\\b)| # .1E+3
(?:\\b[0-9]+(?:_[0-9]+)*[eE][+-]?[0-9]+(?:_[0-9]+)*\\b)| # 1E+3
(?:\\b[0-9]+(?:_[0-9]+)*\\.[0-9]+(?:_[0-9]+)*\\b)| # 1.1
(?:\\b[0-9]+(?:_[0-9]+)*\\.\\B)| # 1.
(?:\\B\\.[0-9]+(?:_[0-9]+)*\\b)| # .1
(?:\\b[0-9]+(?:_[0-9]+)*n?\\b(?!\\.)) # 1n
)(?!\\$)
'''
'name': 'constant.numeric.decimal.js'
'captures':
'0':
'name': 'constant.numeric.decimal.js'
'patterns': [
{
'include': '#numeric_separators'
}
]
}
]
'numeric_separators':
'patterns': [
{
'match': '(_)|(\\.)'
'captures':
'1':
'name': 'meta.delimiter.decimal.period.js'
'name': 'meta.delimiter.numeric.separator.js'
'2':
'name': 'meta.delimiter.decimal.period.js'
'3':
'name': 'meta.delimiter.decimal.period.js'
'4':
'name': 'meta.delimiter.decimal.period.js'
'5':
'name': 'meta.delimiter.decimal.period.js'
'6':
'name': 'meta.delimiter.decimal.period.js'
}
]
'operators':
Expand Down
116 changes: 116 additions & 0 deletions spec/javascript-spec.coffee
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,11 @@ describe "JavaScript grammar", ->
{tokens} = grammar.tokenizeLine('0X1D306n')
expect(tokens[0]).toEqual value: '0X1D306n', scopes: ['source.js', 'constant.numeric.hex.js']

{tokens} = grammar.tokenizeLine('0X1D30_69A3')
expect(tokens[0]).toEqual value: '0X1D30', scopes: ['source.js', 'constant.numeric.hex.js']
expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.hex.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[2]).toEqual value: '69A3', scopes: ['source.js', 'constant.numeric.hex.js']

it "tokenizes binary literals", ->
{tokens} = grammar.tokenizeLine('0b011101110111010001100110')
expect(tokens[0]).toEqual value: '0b011101110111010001100110', scopes: ['source.js', 'constant.numeric.binary.js']
Expand All @@ -336,6 +341,19 @@ describe "JavaScript grammar", ->
{tokens} = grammar.tokenizeLine('0B011101110111010001100110n')
expect(tokens[0]).toEqual value: '0B011101110111010001100110n', scopes: ['source.js', 'constant.numeric.binary.js']

{tokens} = grammar.tokenizeLine('0B0111_0111_0111_0100_0110_0110')
expect(tokens[0]).toEqual value: '0B0111', scopes: ['source.js', 'constant.numeric.binary.js']
expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[2]).toEqual value: '0111', scopes: ['source.js', 'constant.numeric.binary.js']
expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[4]).toEqual value: '0111', scopes: ['source.js', 'constant.numeric.binary.js']
expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[6]).toEqual value: '0100', scopes: ['source.js', 'constant.numeric.binary.js']
expect(tokens[7]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[8]).toEqual value: '0110', scopes: ['source.js', 'constant.numeric.binary.js']
expect(tokens[9]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.binary.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[10]).toEqual value: '0110', scopes: ['source.js', 'constant.numeric.binary.js']

it "tokenizes octal literals", ->
{tokens} = grammar.tokenizeLine('0o1411')
expect(tokens[0]).toEqual value: '0o1411', scopes: ['source.js', 'constant.numeric.octal.js']
Expand All @@ -352,13 +370,32 @@ describe "JavaScript grammar", ->
{tokens} = grammar.tokenizeLine('0010')
expect(tokens[0]).toEqual value: '0010', scopes: ['source.js', 'constant.numeric.octal.js']

{tokens} = grammar.tokenizeLine('0010_7201_5112')
expect(tokens[0]).toEqual value: '0010', scopes: ['source.js', 'constant.numeric.octal.js']
expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.octal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[2]).toEqual value: '7201', scopes: ['source.js', 'constant.numeric.octal.js']
expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.octal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[4]).toEqual value: '5112', scopes: ['source.js', 'constant.numeric.octal.js']

{tokens} = grammar.tokenizeLine('0O1411_1236')
expect(tokens[0]).toEqual value: '0O1411', scopes: ['source.js', 'constant.numeric.octal.js']
expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.octal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[2]).toEqual value: '1236', scopes: ['source.js', 'constant.numeric.octal.js']

it "tokenizes decimals", ->
{tokens} = grammar.tokenizeLine('1234')
expect(tokens[0]).toEqual value: '1234', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('123456789n')
expect(tokens[0]).toEqual value: '123456789n', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('123_456_789n')
expect(tokens[0]).toEqual value: '123', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[2]).toEqual value: '456', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[4]).toEqual value: '789n', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('5e-10')
expect(tokens[0]).toEqual value: '5e-10', scopes: ['source.js', 'constant.numeric.decimal.js']

Expand All @@ -369,31 +406,110 @@ describe "JavaScript grammar", ->
expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']

{tokens} = grammar.tokenizeLine('9_9.')
expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']

{tokens} = grammar.tokenizeLine('.9')
expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('.9_9')
expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[1]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[2]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[3]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('9.9')
expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('9_9.9_9')
expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[4]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[6]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('.1e-23')
expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[1]).toEqual value: '1e-23', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('.1_1E+1_1')
expect(tokens[0]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[1]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[2]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[3]).toEqual value: '1E+1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[4]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[5]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('1.E3')
expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[2]).toEqual value: 'E3', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('1_1.E-1_1')
expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[4]).toEqual value: 'E-1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[6]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('1_1.1_1E1_1')
expect(tokens[0]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[2]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[3]).toEqual value: '.', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.decimal.period.js']
expect(tokens[4]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[5]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[6]).toEqual value: '1E1', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[7]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[8]).toEqual value: '1', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('9_9')
expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('9_9_9')
expect(tokens[0]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[2]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[4]).toEqual value: '9', scopes: ['source.js', 'constant.numeric.decimal.js']

{tokens} = grammar.tokenizeLine('999_999_999')
expect(tokens[0]).toEqual value: '999', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[1]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[2]).toEqual value: '999', scopes: ['source.js', 'constant.numeric.decimal.js']
expect(tokens[3]).toEqual value: '_', scopes: ['source.js', 'constant.numeric.decimal.js', 'meta.delimiter.numeric.separator.js']
expect(tokens[4]).toEqual value: '999', scopes: ['source.js', 'constant.numeric.decimal.js']

it "does not tokenize numbers that are part of a variable", ->
{tokens} = grammar.tokenizeLine('hi$1')
expect(tokens[0]).toEqual value: 'hi$1', scopes: ['source.js']

{tokens} = grammar.tokenizeLine('hi_1')
expect(tokens[0]).toEqual value: 'hi_1', scopes: ['source.js']

{tokens} = grammar.tokenizeLine('_1')
expect(tokens[0]).toEqual value: '_1', scopes: ['source.js', 'constant.other.js']

{tokens} = grammar.tokenizeLine('1_')
expect(tokens[0]).toEqual value: '1_', scopes: ['source.js', 'invalid.illegal.identifier.js']

{tokens} = grammar.tokenizeLine('1_._1')
expect(tokens[0]).toEqual value: '1_', scopes: ['source.js', 'invalid.illegal.identifier.js']
expect(tokens[1]).toEqual value: '.', scopes: ['source.js', 'meta.delimiter.property.period.js']
expect(tokens[2]).toEqual value: '_1', scopes: ['source.js', 'variable.other.property.js']

describe "operators", ->
it "tokenizes them", ->
operators = ["delete", "in", "of", "instanceof", "new", "typeof", "void"]
Expand Down

0 comments on commit 7a1e4f0

Please sign in to comment.