mirror of
https://github.com/ruby/ruby.git
synced 2022-11-09 12:17:21 -05:00
[ruby/irb] Use Ripper::Lexer#scan to take broken tokens
ref. https://github.com/ruby/reline/pull/242 https://github.com/ruby/irb/commit/54f90cb6c9
This commit is contained in:
parent
505e01fe12
commit
5a1866caff
2 changed files with 30 additions and 1 deletions
|
@ -66,6 +66,7 @@ class RubyLex
|
|||
unprocessed_tokens = []
|
||||
line_num_offset = 0
|
||||
tokens.each do |t|
|
||||
next if t[1] == :on_parse_error || t[1] == :compile_error
|
||||
partial_tokens << t
|
||||
unprocessed_tokens << t
|
||||
if t[2].include?("\n")
|
||||
|
@ -110,7 +111,12 @@ class RubyLex
|
|||
verbose, $VERBOSE = $VERBOSE, nil
|
||||
tokens = nil
|
||||
self.class.compile_with_errors_suppressed(code) do |inner_code, line_no|
|
||||
tokens = Ripper.lex(inner_code, '-', line_no)
|
||||
lexer = Ripper::Lexer.new(inner_code, '-', line_no)
|
||||
if lexer.respond_to?(:scan) # Ruby 2.7+
|
||||
tokens = lexer.scan
|
||||
else
|
||||
tokens = lexer.parse
|
||||
end
|
||||
end
|
||||
$VERBOSE = verbose
|
||||
tokens
|
||||
|
@ -122,6 +128,7 @@ class RubyLex
|
|||
prev_spaces = md.nil? ? 0 : md[1].count(' ')
|
||||
line_count = 0
|
||||
@tokens.each_with_index do |t, i|
|
||||
next if t[1] == :on_parse_error || t[1] == :compile_error
|
||||
if t[2].include?("\n")
|
||||
line_count += t[2].count("\n")
|
||||
if line_count >= line_index
|
||||
|
@ -350,6 +357,7 @@ class RubyLex
|
|||
indent = 0
|
||||
in_oneliner_def = nil
|
||||
tokens.each_with_index { |t, index|
|
||||
next if t[1] == :on_parse_error || t[1] == :compile_error
|
||||
# detecting one-liner method definition
|
||||
if in_oneliner_def.nil?
|
||||
if t[3].allbits?(Ripper::EXPR_ENDFN)
|
||||
|
@ -435,6 +443,7 @@ class RubyLex
|
|||
open_brace_on_line = 0
|
||||
in_oneliner_def = nil
|
||||
@tokens.each_with_index do |t, index|
|
||||
next if t[1] == :on_parse_error || t[1] == :compile_error
|
||||
# detecting one-liner method definition
|
||||
if in_oneliner_def.nil?
|
||||
if t[3].allbits?(Ripper::EXPR_ENDFN)
|
||||
|
@ -504,6 +513,7 @@ class RubyLex
|
|||
open_brace_on_line = 0
|
||||
in_oneliner_def = nil
|
||||
@tokens.each_with_index do |t, index|
|
||||
next if t[1] == :on_parse_error || t[1] == :compile_error
|
||||
# detecting one-liner method definition
|
||||
if in_oneliner_def.nil?
|
||||
if t[3].allbits?(Ripper::EXPR_ENDFN)
|
||||
|
|
|
@ -367,6 +367,25 @@ module TestIRB
|
|||
end
|
||||
end
|
||||
|
||||
def test_broken_heredoc
|
||||
if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
|
||||
skip 'This test needs Ripper::Lexer#scan to take broken tokens'
|
||||
end
|
||||
input_with_correct_indents = [
|
||||
Row.new(%q(def foo), nil, 2, 1),
|
||||
Row.new(%q( <<~Q), nil, 2, 1),
|
||||
Row.new(%q( Qend), nil, 2, 1),
|
||||
]
|
||||
|
||||
lines = []
|
||||
input_with_correct_indents.each do |row|
|
||||
lines << row.content
|
||||
assert_indenting(lines, row.current_line_spaces, false)
|
||||
assert_indenting(lines, row.new_line_spaces, true)
|
||||
assert_nesting_level(lines, row.nesting_level)
|
||||
end
|
||||
end
|
||||
|
||||
PromptRow = Struct.new(:prompt, :content)
|
||||
|
||||
class MockIO_DynamicPrompt
|
||||
|
|
Loading…
Add table
Reference in a new issue