Merge pull request #3338 from Earlopain/parser-translator-offset-cache

ruby · Dec 21, 2024 · 79873f2 · 79873f2
2 parents a08cde8 + a651126
commit 79873f2
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 30 deletions.
diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb
@@ -1091,7 +1091,7 @@ def visit_interpolated_string_node(node)
                 start_offset = node.content_loc.start_offset
 
                 node.unescaped.lines.map do |line|
-                  end_offset = start_offset + line.length
+                  end_offset = start_offset + line.bytesize
                   offsets = srange_offsets(start_offset, end_offset)
                   start_offset = end_offset
 
@@ -1692,7 +1692,7 @@ def visit_string_node(node)
                 start_offset = node.content_loc.start_offset
 
                 [content_lines, unescaped_lines].transpose.map do |content_line, unescaped_line|
-                  end_offset = start_offset + content_line.length
+                  end_offset = start_offset + content_line.bytesize
                   offsets = srange_offsets(start_offset, end_offset)
                   start_offset = end_offset
 
@@ -1747,7 +1747,7 @@ def visit_symbol_node(node)
               start_offset = node.value_loc.start_offset
 
               node.value.lines.map do |line|
-                end_offset = start_offset + line.length
+                end_offset = start_offset + line.bytesize
                 offsets = srange_offsets(start_offset, end_offset)
                 start_offset = end_offset
 
@@ -1890,7 +1890,7 @@ def visit_x_string_node(node)
               start_offset = node.content_loc.start_offset
 
               node.unescaped.lines.map do |line|
-                end_offset = start_offset + line.length
+                end_offset = start_offset + line.bytesize
                 offsets = srange_offsets(start_offset, end_offset)
                 start_offset = end_offset
 

diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb
@@ -343,7 +343,7 @@ def to_a
                     adjustment = 0
                   end
 
-                  end_offset = start_offset + adjusted_line.length + adjustment
+                  end_offset = start_offset + adjusted_line.bytesize + adjustment
                   tokens << [:tSTRING_CONTENT, [adjusted_line, Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset])]]
                   start_offset = end_offset
                 end

diff --git a/test/prism/fixtures/dstring.txt b/test/prism/fixtures/dstring.txt
@@ -27,3 +27,6 @@ foo\\\\
 "
 foo\\\\\
 "
+
+"
+’"
diff --git a/test/prism/fixtures/dsym_str.txt b/test/prism/fixtures/dsym_str.txt
@@ -1,2 +1,5 @@
 :"foo
   bar"
+
+:"
+’"
diff --git a/test/prism/fixtures/xstring.txt b/test/prism/fixtures/xstring.txt
@@ -11,3 +11,6 @@
 ``
 
 %x{}
+
+`
+’`
diff --git a/test/prism/ruby/ruby_parser_test.rb b/test/prism/ruby/ruby_parser_test.rb
@@ -46,6 +46,7 @@ class RubyParserTest < TestCase
     # https://github.com/seattlerb/ruby_parser/issues/344
     failures = [
       "alias.txt",
+      "dsym_str.txt",
       "dos_endings.txt",
       "heredocs_with_ignored_newlines.txt",
       "method_calls.txt",

diff --git a/test/prism/snapshots/dstring.txt b/test/prism/snapshots/dstring.txt
diff --git a/test/prism/snapshots/dsym_str.txt b/test/prism/snapshots/dsym_str.txt
diff --git a/test/prism/snapshots/xstring.txt b/test/prism/snapshots/xstring.txt
-Original file line number
+Diff line change
@@ Expand Up / @@ -27,3 +27,6 @@ foo\\\\ @@
     "
     foo\\\\\
     "
+    "
+    ’"