kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

[ruby/irb] Use Ripper::Lexer#scan to take broken tokens

Browse source 
ref. https://github.com/ruby/reline/pull/242

54f90cb6c9
This commit is contained in:
aycabta 2021-01-03 03:25:47 +09:00
parent 505e01fe12
commit 5a1866caff
2 changed files with 30 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

12
lib/irb/ruby-lex.rb
View file

@ -66,6 +66,7 @@ class RubyLex
unprocessed_tokens = [] unprocessed_tokens = []
line_num_offset = 0 line_num_offset = 0
tokens.each do |t| tokens.each do |t|
next if t[1] == :on_parse_error || t[1] == :compile_error
partial_tokens << t partial_tokens << t
unprocessed_tokens << t unprocessed_tokens << t
if t[2].include?("\n") if t[2].include?("\n")
@ -110,7 +111,12 @@ class RubyLex
verbose, $VERBOSE = $VERBOSE, nil verbose, $VERBOSE = $VERBOSE, nil
tokens = nil tokens = nil
self.class.compile_with_errors_suppressed(code) do |inner_code, line_no| self.class.compile_with_errors_suppressed(code) do |inner_code, line_no|
tokens = Ripper.lex(inner_code, '-', line_no) lexer = Ripper::Lexer.new(inner_code, '-', line_no)
if lexer.respond_to?(:scan) # Ruby 2.7+
tokens = lexer.scan
else
tokens = lexer.parse
end
end end
$VERBOSE = verbose $VERBOSE = verbose
tokens tokens
@ -122,6 +128,7 @@ class RubyLex
prev_spaces = md.nil? ? 0 : md[1].count(' ') prev_spaces = md.nil? ? 0 : md[1].count(' ')
line_count = 0 line_count = 0
@tokens.each_with_index do |t, i| @tokens.each_with_index do |t, i|
next if t[1] == :on_parse_error || t[1] == :compile_error
if t[2].include?("\n") if t[2].include?("\n")
line_count += t[2].count("\n") line_count += t[2].count("\n")
if line_count >= line_index if line_count >= line_index
@ -350,6 +357,7 @@ class RubyLex
indent = 0 indent = 0
in_oneliner_def = nil in_oneliner_def = nil
tokens.each_with_index { |t, index| tokens.each_with_index { |t, index|
next if t[1] == :on_parse_error || t[1] == :compile_error
# detecting one-liner method definition # detecting one-liner method definition
if in_oneliner_def.nil? if in_oneliner_def.nil?
if t[3].allbits?(Ripper::EXPR_ENDFN) if t[3].allbits?(Ripper::EXPR_ENDFN)
@ -435,6 +443,7 @@ class RubyLex
open_brace_on_line = 0 open_brace_on_line = 0
in_oneliner_def = nil in_oneliner_def = nil
@tokens.each_with_index do |t, index| @tokens.each_with_index do |t, index|
next if t[1] == :on_parse_error || t[1] == :compile_error
# detecting one-liner method definition # detecting one-liner method definition
if in_oneliner_def.nil? if in_oneliner_def.nil?
if t[3].allbits?(Ripper::EXPR_ENDFN) if t[3].allbits?(Ripper::EXPR_ENDFN)
@ -504,6 +513,7 @@ class RubyLex
open_brace_on_line = 0 open_brace_on_line = 0
in_oneliner_def = nil in_oneliner_def = nil
@tokens.each_with_index do |t, index| @tokens.each_with_index do |t, index|
next if t[1] == :on_parse_error || t[1] == :compile_error
# detecting one-liner method definition # detecting one-liner method definition
if in_oneliner_def.nil? if in_oneliner_def.nil?
if t[3].allbits?(Ripper::EXPR_ENDFN) if t[3].allbits?(Ripper::EXPR_ENDFN)

19
test/irb/test_ruby_lex.rb
View file

@ -367,6 +367,25 @@ module TestIRB
end end
end end
def test_broken_heredoc
if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
skip 'This test needs Ripper::Lexer#scan to take broken tokens'
end
input_with_correct_indents = [
Row.new(%q(def foo), nil, 2, 1),
Row.new(%q( <<~Q), nil, 2, 1),
Row.new(%q( Qend), nil, 2, 1),
]
lines = []
input_with_correct_indents.each do |row|
lines << row.content
assert_indenting(lines, row.current_line_spaces, false)
assert_indenting(lines, row.new_line_spaces, true)
assert_nesting_level(lines, row.nesting_level)
end
end
PromptRow = Struct.new(:prompt, :content) PromptRow = Struct.new(:prompt, :content)
class MockIO_DynamicPrompt class MockIO_DynamicPrompt