Fix an incompatibility with the parser translator

The offset cache contains an entry for each byte so it can't be accessed via the string length. Adds tests for all variants except for this: ``` "fo o" "ba ’" ``` For some reason, this still has the wrong offset.
ruby · Dec 21, 2024 · 72909b4 · 72909b4
1 parent e1e5b55
commit 72909b4
Show file tree

Hide file tree

Showing 9 changed files with 57 additions and 29 deletions.
diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb
@@ -1692,7 +1692,7 @@ def visit_string_node(node)
                 start_offset = node.content_loc.start_offset
 
                 [content_lines, unescaped_lines].transpose.map do |content_line, unescaped_line|
-                  end_offset = start_offset + content_line.length
+                  end_offset = start_offset + content_line.bytesize
                   offsets = srange_offsets(start_offset, end_offset)
                   start_offset = end_offset
 
@@ -1747,7 +1747,7 @@ def visit_symbol_node(node)
               start_offset = node.value_loc.start_offset
 
               node.value.lines.map do |line|
-                end_offset = start_offset + line.length
+                end_offset = start_offset + line.bytesize
                 offsets = srange_offsets(start_offset, end_offset)
                 start_offset = end_offset
 
@@ -1890,7 +1890,7 @@ def visit_x_string_node(node)
               start_offset = node.content_loc.start_offset
 
               node.unescaped.lines.map do |line|
-                end_offset = start_offset + line.length
+                end_offset = start_offset + line.bytesize
                 offsets = srange_offsets(start_offset, end_offset)
                 start_offset = end_offset
 

diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb
@@ -343,7 +343,7 @@ def to_a
                     adjustment = 0
                   end
 
-                  end_offset = start_offset + adjusted_line.length + adjustment
+                  end_offset = start_offset + adjusted_line.bytesize + adjustment
                   tokens << [:tSTRING_CONTENT, [adjusted_line, Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset])]]
                   start_offset = end_offset
                 end

diff --git a/test/prism/fixtures/dstring.txt b/test/prism/fixtures/dstring.txt
@@ -27,3 +27,6 @@ foo\\\\
 "
 foo\\\\\
 "
+
+"
+’"
diff --git a/test/prism/fixtures/dsym_str.txt b/test/prism/fixtures/dsym_str.txt
@@ -1,2 +1,5 @@
 :"foo
   bar"
+
+:"
+’"
diff --git a/test/prism/fixtures/xstring.txt b/test/prism/fixtures/xstring.txt
@@ -11,3 +11,6 @@
 ``
 
 %x{}
+
+`
+’`
diff --git a/test/prism/ruby/ruby_parser_test.rb b/test/prism/ruby/ruby_parser_test.rb
@@ -46,6 +46,7 @@ class RubyParserTest < TestCase
     # https://github.com/seattlerb/ruby_parser/issues/344
     failures = [
       "alias.txt",
+      "dsym_str.txt",
       "dos_endings.txt",
       "heredocs_with_ignored_newlines.txt",
       "method_calls.txt",

diff --git a/test/prism/snapshots/dstring.txt b/test/prism/snapshots/dstring.txt
diff --git a/test/prism/snapshots/dsym_str.txt b/test/prism/snapshots/dsym_str.txt
diff --git a/test/prism/snapshots/xstring.txt b/test/prism/snapshots/xstring.txt
-Original file line number
+Diff line change
@@ Expand Up / @@ -27,3 +27,6 @@ foo\\\\ @@
     "
     foo\\\\\
     "
+    "
+    ’"