refactor: move raw string parsing to external scanner

tree-sitter-grammars · Feb 1, 2024 · be685dc · be685dc
1 parent e180e05
commit be685dc
Show file tree

Hide file tree

Showing 7 changed files with 7,151 additions and 7,913 deletions.
diff --git a/grammar.js b/grammar.js
@@ -62,7 +62,11 @@ module.exports = grammar({
     [$.node_children],
   ],
 
-  externals: $ => [$._eof, $.multi_line_comment],
+  externals: $ => [
+    $._eof,
+    $.multi_line_comment,
+    $._raw_string,
+  ],
 
   extras: $ => [$.multi_line_comment],
 
@@ -182,23 +186,6 @@ module.exports = grammar({
     // hex-digit := [0-9a-fA-F]
     _hex_digit: _ => /[0-9a-fA-F]/,
 
-    // // raw-string := 'r' raw-string-hash
-    // raw_string: $ => seq('r', $._raw_string_hash),
-    // // raw-string-hash := '#' raw-string-hash '#' | raw-string-quotes
-    // _raw_string_hash: $ => choice(seq('#', $._raw_string_hash, '#'), $._raw_string_quotes),
-    // // raw-string-quotes := '"' .* '"'
-    // _raw_string_quotes: _ => seq('"', /.*/, '"'),
-    _raw_string: _ =>
-      seq(
-        choice(
-          // raw-string-hash := '#' raw-string-hash '#' | raw-string-quotes
-          // yes this isn't perfect but it works, ideally this should be handled in an external scanner
-          seq(token.immediate(seq('r', repeat1('#'))), /[^#]*/, repeat1('#')),
-          // raw-string-quotes := '"' . '"'
-          seq(token.immediate(seq('r', '"')), /[^"]*/, '"'),
-        ),
-      ),
-
     // number := decimal | hex | octal | binary
     number: $ => choice($._decimal, $._hex, $._octal, $._binary),
 

diff --git a/src/grammar.json b/src/grammar.json
@@ -786,80 +786,6 @@
       "type": "PATTERN",
       "value": "[0-9a-fA-F]"
     },
-    "_raw_string": {
-      "type": "SEQ",
-      "members": [
-        {
-          "type": "CHOICE",
-          "members": [
-            {
-              "type": "SEQ",
-              "members": [
-                {
-                  "type": "IMMEDIATE_TOKEN",
-                  "content": {
-                    "type": "SEQ",
-                    "members": [
-                      {
-                        "type": "STRING",
-                        "value": "r"
-                      },
-                      {
-                        "type": "REPEAT1",
-                        "content": {
-                          "type": "STRING",
-                          "value": "#"
-                        }
-                      }
-                    ]
-                  }
-                },
-                {
-                  "type": "PATTERN",
-                  "value": "[^#]*"
-                },
-                {
-                  "type": "REPEAT1",
-                  "content": {
-                    "type": "STRING",
-                    "value": "#"
-                  }
-                }
-              ]
-            },
-            {
-              "type": "SEQ",
-              "members": [
-                {
-                  "type": "IMMEDIATE_TOKEN",
-                  "content": {
-                    "type": "SEQ",
-                    "members": [
-                      {
-                        "type": "STRING",
-                        "value": "r"
-                      },
-                      {
-                        "type": "STRING",
-                        "value": "\""
-                      }
-                    ]
-                  }
-                },
-                {
-                  "type": "PATTERN",
-                  "value": "[^\"]*"
-                },
-                {
-                  "type": "STRING",
-                  "value": "\""
-                }
-              ]
-            }
-          ]
-        }
-      ]
-    },
     "number": {
       "type": "CHOICE",
       "members": [
@@ -1330,6 +1256,10 @@
     {
       "type": "SYMBOL",
       "name": "multi_line_comment"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_raw_string"
     }
   ],
   "inline": [],

diff --git a/src/node-types.json b/src/node-types.json
@@ -337,10 +337,6 @@
     "type": "\"",
     "named": false
   },
-  {
-    "type": "#",
-    "named": false
-  },
   {
     "type": "(",
     "named": false