Skip to content

Commit

Permalink
refactor: move raw string parsing to external scanner
Browse files Browse the repository at this point in the history
  • Loading branch information
dataphract authored and amaanq committed Feb 1, 2024
1 parent e180e05 commit 9edee9b
Show file tree
Hide file tree
Showing 7 changed files with 7,151 additions and 7,913 deletions.
23 changes: 5 additions & 18 deletions grammar.js
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,11 @@ module.exports = grammar({
[$.node_children],
],

externals: $ => [$._eof, $.multi_line_comment],
externals: $ => [
$._eof,
$.multi_line_comment,
$._raw_string,
],

extras: $ => [$.multi_line_comment],

Expand Down Expand Up @@ -182,23 +186,6 @@ module.exports = grammar({
// hex-digit := [0-9a-fA-F]
_hex_digit: _ => /[0-9a-fA-F]/,

// // raw-string := 'r' raw-string-hash
// raw_string: $ => seq('r', $._raw_string_hash),
// // raw-string-hash := '#' raw-string-hash '#' | raw-string-quotes
// _raw_string_hash: $ => choice(seq('#', $._raw_string_hash, '#'), $._raw_string_quotes),
// // raw-string-quotes := '"' .* '"'
// _raw_string_quotes: _ => seq('"', /.*/, '"'),
_raw_string: _ =>
seq(
choice(
// raw-string-hash := '#' raw-string-hash '#' | raw-string-quotes
// yes this isn't perfect but it works, ideally this should be handled in an external scanner
seq(token.immediate(seq('r', repeat1('#'))), /[^#]*/, repeat1('#')),
// raw-string-quotes := '"' . '"'
seq(token.immediate(seq('r', '"')), /[^"]*/, '"'),
),
),

// number := decimal | hex | octal | binary
number: $ => choice($._decimal, $._hex, $._octal, $._binary),

Expand Down
78 changes: 4 additions & 74 deletions src/grammar.json
Original file line number Diff line number Diff line change
Expand Up @@ -786,80 +786,6 @@
"type": "PATTERN",
"value": "[0-9a-fA-F]"
},
"_raw_string": {
"type": "SEQ",
"members": [
{
"type": "CHOICE",
"members": [
{
"type": "SEQ",
"members": [
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "r"
},
{
"type": "REPEAT1",
"content": {
"type": "STRING",
"value": "#"
}
}
]
}
},
{
"type": "PATTERN",
"value": "[^#]*"
},
{
"type": "REPEAT1",
"content": {
"type": "STRING",
"value": "#"
}
}
]
},
{
"type": "SEQ",
"members": [
{
"type": "IMMEDIATE_TOKEN",
"content": {
"type": "SEQ",
"members": [
{
"type": "STRING",
"value": "r"
},
{
"type": "STRING",
"value": "\""
}
]
}
},
{
"type": "PATTERN",
"value": "[^\"]*"
},
{
"type": "STRING",
"value": "\""
}
]
}
]
}
]
},
"number": {
"type": "CHOICE",
"members": [
Expand Down Expand Up @@ -1330,6 +1256,10 @@
{
"type": "SYMBOL",
"name": "multi_line_comment"
},
{
"type": "SYMBOL",
"name": "_raw_string"
}
],
"inline": [],
Expand Down
4 changes: 0 additions & 4 deletions src/node-types.json
Original file line number Diff line number Diff line change
Expand Up @@ -337,10 +337,6 @@
"type": "\"",
"named": false
},
{
"type": "#",
"named": false
},
{
"type": "(",
"named": false
Expand Down
Loading

0 comments on commit 9edee9b

Please sign in to comment.