diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb index d66e553fa3..4a3cba3de0 100644 --- a/lib/prism/translation/parser/compiler.rb +++ b/lib/prism/translation/parser/compiler.rb @@ -1091,7 +1091,7 @@ def visit_interpolated_string_node(node) start_offset = node.content_loc.start_offset node.unescaped.lines.map do |line| - end_offset = start_offset + line.length + end_offset = start_offset + line.bytesize offsets = srange_offsets(start_offset, end_offset) start_offset = end_offset @@ -1692,7 +1692,7 @@ def visit_string_node(node) start_offset = node.content_loc.start_offset [content_lines, unescaped_lines].transpose.map do |content_line, unescaped_line| - end_offset = start_offset + content_line.length + end_offset = start_offset + content_line.bytesize offsets = srange_offsets(start_offset, end_offset) start_offset = end_offset @@ -1747,7 +1747,7 @@ def visit_symbol_node(node) start_offset = node.value_loc.start_offset node.value.lines.map do |line| - end_offset = start_offset + line.length + end_offset = start_offset + line.bytesize offsets = srange_offsets(start_offset, end_offset) start_offset = end_offset @@ -1890,7 +1890,7 @@ def visit_x_string_node(node) start_offset = node.content_loc.start_offset node.unescaped.lines.map do |line| - end_offset = start_offset + line.length + end_offset = start_offset + line.bytesize offsets = srange_offsets(start_offset, end_offset) start_offset = end_offset diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb index db7dbb1c87..61e22159a1 100644 --- a/lib/prism/translation/parser/lexer.rb +++ b/lib/prism/translation/parser/lexer.rb @@ -343,7 +343,7 @@ def to_a adjustment = 0 end - end_offset = start_offset + adjusted_line.length + adjustment + end_offset = start_offset + adjusted_line.bytesize + adjustment tokens << [:tSTRING_CONTENT, [adjusted_line, Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset])]] start_offset = end_offset end diff --git a/test/prism/fixtures/dstring.txt b/test/prism/fixtures/dstring.txt index 085e0c6852..69019662c1 100644 --- a/test/prism/fixtures/dstring.txt +++ b/test/prism/fixtures/dstring.txt @@ -27,3 +27,6 @@ foo\\\\ " foo\\\\\ " + +" +’" diff --git a/test/prism/fixtures/dsym_str.txt b/test/prism/fixtures/dsym_str.txt index ee68dde88d..0af0a8ddaf 100644 --- a/test/prism/fixtures/dsym_str.txt +++ b/test/prism/fixtures/dsym_str.txt @@ -1,2 +1,5 @@ :"foo bar" + +:" +’" diff --git a/test/prism/fixtures/xstring.txt b/test/prism/fixtures/xstring.txt index 7ec09468d8..bb3d1c6ee1 100644 --- a/test/prism/fixtures/xstring.txt +++ b/test/prism/fixtures/xstring.txt @@ -11,3 +11,6 @@ `` %x{} + +` +’` diff --git a/test/prism/ruby/ruby_parser_test.rb b/test/prism/ruby/ruby_parser_test.rb index a13daeeb84..fe410c8144 100644 --- a/test/prism/ruby/ruby_parser_test.rb +++ b/test/prism/ruby/ruby_parser_test.rb @@ -46,6 +46,7 @@ class RubyParserTest < TestCase # https://github.com/seattlerb/ruby_parser/issues/344 failures = [ "alias.txt", + "dsym_str.txt", "dos_endings.txt", "heredocs_with_ignored_newlines.txt", "method_calls.txt", diff --git a/test/prism/snapshots/dstring.txt b/test/prism/snapshots/dstring.txt index b3ece40513..7c9692fc18 100644 --- a/test/prism/snapshots/dstring.txt +++ b/test/prism/snapshots/dstring.txt @@ -1,10 +1,10 @@ -@ ProgramNode (location: (1,0)-(29,1)) +@ ProgramNode (location: (1,0)-(32,4)) ├── flags: ∅ ├── locals: [] └── statements: - @ StatementsNode (location: (1,0)-(29,1)) + @ StatementsNode (location: (1,0)-(32,4)) ├── flags: ∅ - └── body: (length: 8) + └── body: (length: 9) ├── @ StringNode (location: (1,0)-(2,6)) │ ├── flags: newline │ ├── opening_loc: (1,0)-(1,1) = "\"" @@ -81,9 +81,15 @@ │ ├── content_loc: (23,1)-(25,0) = "\nfoo\\\\\\\\\n" │ ├── closing_loc: (25,0)-(25,1) = "\"" │ └── unescaped: "\nfoo\\\\\n" - └── @ StringNode (location: (27,0)-(29,1)) + ├── @ StringNode (location: (27,0)-(29,1)) + │ ├── flags: newline + │ ├── opening_loc: (27,0)-(27,1) = "\"" + │ ├── content_loc: (27,1)-(29,0) = "\nfoo\\\\\\\\\\\n" + │ ├── closing_loc: (29,0)-(29,1) = "\"" + │ └── unescaped: "\nfoo\\\\" + └── @ StringNode (location: (31,0)-(32,4)) ├── flags: newline - ├── opening_loc: (27,0)-(27,1) = "\"" - ├── content_loc: (27,1)-(29,0) = "\nfoo\\\\\\\\\\\n" - ├── closing_loc: (29,0)-(29,1) = "\"" - └── unescaped: "\nfoo\\\\" + ├── opening_loc: (31,0)-(31,1) = "\"" + ├── content_loc: (31,1)-(32,3) = "\n’" + ├── closing_loc: (32,3)-(32,4) = "\"" + └── unescaped: "\n’" diff --git a/test/prism/snapshots/dsym_str.txt b/test/prism/snapshots/dsym_str.txt index 835cbbdc8a..629fa9e969 100644 --- a/test/prism/snapshots/dsym_str.txt +++ b/test/prism/snapshots/dsym_str.txt @@ -1,13 +1,19 @@ -@ ProgramNode (location: (1,0)-(2,6)) +@ ProgramNode (location: (1,0)-(5,4)) ├── flags: ∅ ├── locals: [] └── statements: - @ StatementsNode (location: (1,0)-(2,6)) + @ StatementsNode (location: (1,0)-(5,4)) ├── flags: ∅ - └── body: (length: 1) - └── @ SymbolNode (location: (1,0)-(2,6)) - ├── flags: newline, static_literal, forced_us_ascii_encoding - ├── opening_loc: (1,0)-(1,2) = ":\"" - ├── value_loc: (1,2)-(2,5) = "foo\n bar" - ├── closing_loc: (2,5)-(2,6) = "\"" - └── unescaped: "foo\n bar" + └── body: (length: 2) + ├── @ SymbolNode (location: (1,0)-(2,6)) + │ ├── flags: newline, static_literal, forced_us_ascii_encoding + │ ├── opening_loc: (1,0)-(1,2) = ":\"" + │ ├── value_loc: (1,2)-(2,5) = "foo\n bar" + │ ├── closing_loc: (2,5)-(2,6) = "\"" + │ └── unescaped: "foo\n bar" + └── @ SymbolNode (location: (4,0)-(5,4)) + ├── flags: newline, static_literal + ├── opening_loc: (4,0)-(4,2) = ":\"" + ├── value_loc: (4,2)-(5,3) = "\n’" + ├── closing_loc: (5,3)-(5,4) = "\"" + └── unescaped: "\n’" diff --git a/test/prism/snapshots/xstring.txt b/test/prism/snapshots/xstring.txt index 837cd581a4..b3a3cb801b 100644 --- a/test/prism/snapshots/xstring.txt +++ b/test/prism/snapshots/xstring.txt @@ -1,10 +1,10 @@ -@ ProgramNode (location: (1,0)-(13,4)) +@ ProgramNode (location: (1,0)-(16,4)) ├── flags: ∅ ├── locals: [] └── statements: - @ StatementsNode (location: (1,0)-(13,4)) + @ StatementsNode (location: (1,0)-(16,4)) ├── flags: ∅ - └── body: (length: 6) + └── body: (length: 7) ├── @ XStringNode (location: (1,0)-(1,7)) │ ├── flags: newline │ ├── opening_loc: (1,0)-(1,3) = "%x[" @@ -64,9 +64,15 @@ │ ├── content_loc: (11,1)-(11,1) = "" │ ├── closing_loc: (11,1)-(11,2) = "`" │ └── unescaped: "" - └── @ XStringNode (location: (13,0)-(13,4)) + ├── @ XStringNode (location: (13,0)-(13,4)) + │ ├── flags: newline + │ ├── opening_loc: (13,0)-(13,3) = "%x{" + │ ├── content_loc: (13,3)-(13,3) = "" + │ ├── closing_loc: (13,3)-(13,4) = "}" + │ └── unescaped: "" + └── @ XStringNode (location: (15,0)-(16,4)) ├── flags: newline - ├── opening_loc: (13,0)-(13,3) = "%x{" - ├── content_loc: (13,3)-(13,3) = "" - ├── closing_loc: (13,3)-(13,4) = "}" - └── unescaped: "" + ├── opening_loc: (15,0)-(15,1) = "`" + ├── content_loc: (15,1)-(16,3) = "\n’" + ├── closing_loc: (16,3)-(16,4) = "`" + └── unescaped: "\n’"