diff --git a/lib/irb/color.rb b/lib/irb/color.rb index d2b9674a7..d325c8ded 100644 --- a/lib/irb/color.rb +++ b/lib/irb/color.rb @@ -155,27 +155,29 @@ def scan(code, allow_last_error:) pos = [1, 0] verbose, $VERBOSE = $VERBOSE, nil - lexer = Ripper::Lexer.new(code) - if lexer.respond_to?(:scan) # Ruby 2.7+ - lexer.scan.each do |elem| - str = elem.tok - next if allow_last_error and /meets end of file|unexpected end-of-input/ =~ elem.message - next if ([elem.pos[0], elem.pos[1] + str.bytesize] <=> pos) <= 0 - - str.each_line do |line| - if line.end_with?("\n") - pos[0] += 1 - pos[1] = 0 - else - pos[1] += line.bytesize + RubyLex.compile_with_errors_suppressed(code) do |inner_code| + lexer = Ripper::Lexer.new(inner_code) + if lexer.respond_to?(:scan) # Ruby 2.7+ + lexer.scan.each do |elem| + str = elem.tok + next if allow_last_error and /meets end of file|unexpected end-of-input/ =~ elem.message + next if ([elem.pos[0], elem.pos[1] + str.bytesize] <=> pos) <= 0 + + str.each_line do |line| + if line.end_with?("\n") + pos[0] += 1 + pos[1] = 0 + else + pos[1] += line.bytesize + end end - end - yield(elem.event, str, elem.state) - end - else - lexer.parse.each do |elem| - yield(elem.event, elem.tok, elem.state) + yield(elem.event, str, elem.state) + end + else + lexer.parse.each do |elem| + yield(elem.event, elem.tok, elem.state) + end end end $VERBOSE = verbose diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb index 0c71d5ec0..02bc54880 100644 --- a/lib/irb/ruby-lex.rb +++ b/lib/irb/ruby-lex.rb @@ -30,6 +30,19 @@ def initialize @prompt = nil end + def self.compile_with_errors_suppressed(code) + begin + result = yield code + rescue ArgumentError => e + magic_comment_regexp = /\A(?#.*\n)?#\s*(?:encoding|coding)\s*:.*(?\n)?/ + if e.message.match?(/unknown encoding name/) && code.match?(magic_comment_regexp) + code = code.gsub(magic_comment_regexp, "\\k#\\k") + retry + end + end + result + end + # io functions def set_input(io, p = nil, &block) @io = io @@ -76,7 +89,10 @@ def set_prompt(p = nil, &block) def ripper_lex_without_warning(code) verbose, $VERBOSE = $VERBOSE, nil - tokens = Ripper.lex(code) + tokens = nil + self.class.compile_with_errors_suppressed(code) do |inner_code| + tokens = Ripper.lex(inner_code) + end $VERBOSE = verbose tokens end @@ -210,7 +226,9 @@ def check_code_block(code) when 'jruby' JRuby.compile_ir(code) else - RubyVM::InstructionSequence.compile(code) + self.class.compile_with_errors_suppressed(code) do |inner_code| + RubyVM::InstructionSequence.compile(inner_code) + end end rescue EncodingError # This is for a hash with invalid encoding symbol, {"\xAE": 1} diff --git a/test/irb/test_ruby_lex.rb b/test/irb/test_ruby_lex.rb index dd5a1f7ca..bc5b4feaf 100644 --- a/test/irb/test_ruby_lex.rb +++ b/test/irb/test_ruby_lex.rb @@ -126,5 +126,31 @@ def test_a_closed_brace_and_not_closed_brace_in_a_line assert_indenting(lines, row.new_line_spaces, true) end end + + def test_incomplete_coding_magic_comment + input_with_correct_indents = [ + Row.new(%q(#coding:u), nil, 0), + ] + + lines = [] + input_with_correct_indents.each do |row| + lines << row.content + assert_indenting(lines, row.current_line_spaces, false) + assert_indenting(lines, row.new_line_spaces, true) + end + end + + def test_incomplete_encoding_magic_comment + input_with_correct_indents = [ + Row.new(%q(#encoding:u), nil, 0), + ] + + lines = [] + input_with_correct_indents.each do |row| + lines << row.content + assert_indenting(lines, row.current_line_spaces, false) + assert_indenting(lines, row.new_line_spaces, true) + end + end end end