mirror of
https://github.com/ruby/ruby.git
synced 2022-11-09 12:17:21 -05:00
Compatibility with IRB
Instead of accessing the struct as an array, access it via methods. There are other places inside of this file already using this API (for example e0a5c3d2b7/lib/irb/ruby-lex.rb (L829-L830)).
This commit moves all struct array-ish calls to use their method calls instead. It is also ~1.23 faster accessing values via a method instead of as an array according to this microbenchmark:
```ruby
Elem = Struct.new(:pos, :event, :tok, :state, :message) do
def initialize(pos, event, tok, state, message = nil)
super(pos, event, tok, State.new(state), message)
end
# ...
def to_a
a = super
a.pop unless a.empty?
a
end
end
class ElemClass
attr_accessor :pos, :event, :tok, :state, :message
def initialize(pos, event, tok, state, message = nil)
@pos = pos
@event = event
@tok = tok
@state = State.new(state)
@message = message
end
def to_a
if @message
[@pos, @event, @tok, @state, @message]
else
[@pos, @event, @tok, @state]
end
end
end
# stub state class creation for now
class State; def initialize(val); end; end
```
```ruby
Benchmark.ips do |x|
x.report("struct") { struct[1] }
x.report("class ") { from_class.event }
x.compare!
end; nil
```
```
Warming up --------------------------------------
struct 1.624M i/100ms
class 1.958M i/100ms
Calculating -------------------------------------
struct 17.139M (± 2.6%) i/s - 86.077M in 5.025801s
class 21.104M (± 3.4%) i/s - 105.709M in 5.015193s
Comparison:
class : 21103826.3 i/s
struct: 17139201.5 i/s - 1.23x (± 0.00) slower
```
This commit is contained in:
schneems
Yusuke Endoh
parent
3685b5af95
commit
2b22c93533
Notes:
git
2021-12-02 15:56:03 +09:00
3 changed files with 75 additions and 67 deletions
|
|
@ -64,7 +64,7 @@ module IRB
|
|||
prev_tokens = []
|
||||
|
||||
# chunk with line number
|
||||
tokens.chunk { |tok| tok[0][0] }.each do |lnum, chunk|
|
||||
tokens.chunk { |tok| tok.pos[0] }.each do |lnum, chunk|
|
||||
code = lines[0..lnum].join
|
||||
prev_tokens.concat chunk
|
||||
continue = lex.process_continue(prev_tokens)
|
||||
|
|
|
|||
|
|
@ -87,8 +87,8 @@ class RubyLex
|
|||
tokens.each do |t|
|
||||
partial_tokens << t
|
||||
unprocessed_tokens << t
|
||||
if t[2].include?("\n")
|
||||
t_str = t[2]
|
||||
if t.tok.include?("\n")
|
||||
t_str = t.tok
|
||||
t_str.each_line("\n") do |s|
|
||||
code << s << "\n"
|
||||
ltype, indent, continue, code_block_open = check_state(code, partial_tokens, context: context)
|
||||
|
|
@ -97,9 +97,10 @@ class RubyLex
|
|||
end
|
||||
unprocessed_tokens = []
|
||||
else
|
||||
code << t[2]
|
||||
code << t.tok
|
||||
end
|
||||
end
|
||||
|
||||
unless unprocessed_tokens.empty?
|
||||
ltype, indent, continue, code_block_open = check_state(code, unprocessed_tokens, context: context)
|
||||
result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset)
|
||||
|
|
@ -107,6 +108,7 @@ class RubyLex
|
|||
result
|
||||
end
|
||||
end
|
||||
|
||||
if p.respond_to?(:call)
|
||||
@input = p
|
||||
elsif block_given?
|
||||
|
|
@ -153,14 +155,14 @@ class RubyLex
|
|||
pos_to_index = {}
|
||||
lexer.scan.each do |t|
|
||||
next if t.pos.first == 0
|
||||
if pos_to_index.has_key?(t[0])
|
||||
index = pos_to_index[t[0]]
|
||||
if pos_to_index.has_key?(t.pos)
|
||||
index = pos_to_index[t.pos]
|
||||
found_tk = tokens[index]
|
||||
if ERROR_TOKENS.include?(found_tk[1]) && !ERROR_TOKENS.include?(t[1])
|
||||
if ERROR_TOKENS.include?(found_tk.event) && !ERROR_TOKENS.include?(t.event)
|
||||
tokens[index] = t
|
||||
end
|
||||
else
|
||||
pos_to_index[t[0]] = tokens.size
|
||||
pos_to_index[t.pos] = tokens.size
|
||||
tokens << t
|
||||
end
|
||||
end
|
||||
|
|
@ -175,17 +177,17 @@ class RubyLex
|
|||
|
||||
def find_prev_spaces(line_index)
|
||||
return 0 if @tokens.size == 0
|
||||
md = @tokens[0][2].match(/(\A +)/)
|
||||
md = @tokens[0].tok.match(/(\A +)/)
|
||||
prev_spaces = md.nil? ? 0 : md[1].count(' ')
|
||||
line_count = 0
|
||||
@tokens.each_with_index do |t, i|
|
||||
if t[2].include?("\n")
|
||||
line_count += t[2].count("\n")
|
||||
if t.tok.include?("\n")
|
||||
line_count += t.tok.count("\n")
|
||||
if line_count >= line_index
|
||||
return prev_spaces
|
||||
end
|
||||
if (@tokens.size - 1) > i
|
||||
md = @tokens[i + 1][2].match(/(\A +)/)
|
||||
md = @tokens[i + 1].tok.match(/(\A +)/)
|
||||
prev_spaces = md.nil? ? 0 : md[1].count(' ')
|
||||
end
|
||||
end
|
||||
|
|
@ -295,18 +297,18 @@ class RubyLex
|
|||
|
||||
def process_continue(tokens = @tokens)
|
||||
# last token is always newline
|
||||
if tokens.size >= 2 and tokens[-2][1] == :on_regexp_end
|
||||
if tokens.size >= 2 and tokens[-2].event == :on_regexp_end
|
||||
# end of regexp literal
|
||||
return false
|
||||
elsif tokens.size >= 2 and tokens[-2][1] == :on_semicolon
|
||||
elsif tokens.size >= 2 and tokens[-2].event == :on_semicolon
|
||||
return false
|
||||
elsif tokens.size >= 2 and tokens[-2][1] == :on_kw and ['begin', 'else', 'ensure'].include?(tokens[-2][2])
|
||||
elsif tokens.size >= 2 and tokens[-2].event == :on_kw and ['begin', 'else', 'ensure'].include?(tokens[-2].tok)
|
||||
return false
|
||||
elsif !tokens.empty? and tokens.last[2] == "\\\n"
|
||||
elsif !tokens.empty? and tokens.last.tok == "\\\n"
|
||||
return true
|
||||
elsif tokens.size >= 1 and tokens[-1][1] == :on_heredoc_end # "EOH\n"
|
||||
elsif tokens.size >= 1 and tokens[-1].event == :on_heredoc_end # "EOH\n"
|
||||
return false
|
||||
elsif tokens.size >= 2 and defined?(Ripper::EXPR_BEG) and tokens[-2][3].anybits?(Ripper::EXPR_BEG | Ripper::EXPR_FNAME) and tokens[-2][2] !~ /\A\.\.\.?\z/
|
||||
elsif tokens.size >= 2 and defined?(Ripper::EXPR_BEG) and tokens[-2].state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_FNAME) and tokens[-2].tok !~ /\A\.\.\.?\z/
|
||||
# end of literal except for regexp
|
||||
# endless range at end of line is not a continue
|
||||
return true
|
||||
|
|
@ -316,7 +318,7 @@ class RubyLex
|
|||
|
||||
def check_code_block(code, tokens = @tokens)
|
||||
return true if tokens.empty?
|
||||
if tokens.last[1] == :on_heredoc_beg
|
||||
if tokens.last.event == :on_heredoc_beg
|
||||
return true
|
||||
end
|
||||
|
||||
|
|
@ -388,7 +390,7 @@ class RubyLex
|
|||
end
|
||||
|
||||
if defined?(Ripper::EXPR_BEG)
|
||||
last_lex_state = tokens.last[3]
|
||||
last_lex_state = tokens.last.state
|
||||
if last_lex_state.allbits?(Ripper::EXPR_BEG)
|
||||
return false
|
||||
elsif last_lex_state.allbits?(Ripper::EXPR_DOT)
|
||||
|
|
@ -413,14 +415,14 @@ class RubyLex
|
|||
tokens.each_with_index { |t, index|
|
||||
# detecting one-liner method definition
|
||||
if in_oneliner_def.nil?
|
||||
if t[3].allbits?(Ripper::EXPR_ENDFN)
|
||||
if t.state.allbits?(Ripper::EXPR_ENDFN)
|
||||
in_oneliner_def = :ENDFN
|
||||
end
|
||||
else
|
||||
if t[3].allbits?(Ripper::EXPR_ENDFN)
|
||||
if t.state.allbits?(Ripper::EXPR_ENDFN)
|
||||
# continuing
|
||||
elsif t[3].allbits?(Ripper::EXPR_BEG)
|
||||
if t[2] == '='
|
||||
elsif t.state.allbits?(Ripper::EXPR_BEG)
|
||||
if t.tok == '='
|
||||
in_oneliner_def = :BODY
|
||||
end
|
||||
else
|
||||
|
|
@ -432,14 +434,14 @@ class RubyLex
|
|||
end
|
||||
end
|
||||
|
||||
case t[1]
|
||||
case t.event
|
||||
when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg
|
||||
indent += 1
|
||||
when :on_rbracket, :on_rbrace, :on_rparen
|
||||
indent -= 1
|
||||
when :on_kw
|
||||
next if index > 0 and tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME)
|
||||
case t[2]
|
||||
next if index > 0 and tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
|
||||
case t.tok
|
||||
when 'do'
|
||||
syntax_of_do = take_corresponding_syntax_to_kw_do(tokens, index)
|
||||
indent += 1 if syntax_of_do == :method_calling
|
||||
|
|
@ -447,7 +449,7 @@ class RubyLex
|
|||
indent += 1
|
||||
when 'if', 'unless', 'while', 'until'
|
||||
# postfix if/unless/while/until must be Ripper::EXPR_LABEL
|
||||
indent += 1 unless t[3].allbits?(Ripper::EXPR_LABEL)
|
||||
indent += 1 unless t.state.allbits?(Ripper::EXPR_LABEL)
|
||||
when 'end'
|
||||
indent -= 1
|
||||
end
|
||||
|
|
@ -459,14 +461,14 @@ class RubyLex
|
|||
|
||||
def is_method_calling?(tokens, index)
|
||||
tk = tokens[index]
|
||||
if tk[3].anybits?(Ripper::EXPR_CMDARG) and tk[1] == :on_ident
|
||||
if tk.state.anybits?(Ripper::EXPR_CMDARG) and tk.event == :on_ident
|
||||
# The target method call to pass the block with "do".
|
||||
return true
|
||||
elsif tk[3].anybits?(Ripper::EXPR_ARG) and tk[1] == :on_ident
|
||||
non_sp_index = tokens[0..(index - 1)].rindex{ |t| t[1] != :on_sp }
|
||||
elsif tk.state.anybits?(Ripper::EXPR_ARG) and tk.event == :on_ident
|
||||
non_sp_index = tokens[0..(index - 1)].rindex{ |t| t.event != :on_sp }
|
||||
if non_sp_index
|
||||
prev_tk = tokens[non_sp_index]
|
||||
if prev_tk[3].anybits?(Ripper::EXPR_DOT) and prev_tk[1] == :on_period
|
||||
if prev_tk.state.anybits?(Ripper::EXPR_DOT) and prev_tk.event == :on_period
|
||||
# The target method call with receiver to pass the block with "do".
|
||||
return true
|
||||
end
|
||||
|
|
@ -481,17 +483,17 @@ class RubyLex
|
|||
index.downto(0) do |i|
|
||||
tk = tokens[i]
|
||||
# In "continue", the token isn't the corresponding syntax to "do".
|
||||
non_sp_index = tokens[0..(i - 1)].rindex{ |t| t[1] != :on_sp }
|
||||
non_sp_index = tokens[0..(i - 1)].rindex{ |t| t.event != :on_sp }
|
||||
first_in_fomula = false
|
||||
if non_sp_index.nil?
|
||||
first_in_fomula = true
|
||||
elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index][1])
|
||||
elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index].event)
|
||||
first_in_fomula = true
|
||||
end
|
||||
if is_method_calling?(tokens, i)
|
||||
syntax_of_do = :method_calling
|
||||
break if first_in_fomula
|
||||
elsif tk[1] == :on_kw && %w{while until for}.include?(tk[2])
|
||||
elsif tk.event == :on_kw && %w{while until for}.include?(tk.tok)
|
||||
# A loop syntax in front of "do" found.
|
||||
#
|
||||
# while cond do # also "until" or "for"
|
||||
|
|
@ -512,14 +514,14 @@ class RubyLex
|
|||
index.downto(0) do |i|
|
||||
tk = tokens[i]
|
||||
# In "continue", the token isn't the corresponding syntax to "do".
|
||||
non_sp_index = tokens[0..(i - 1)].rindex{ |t| t[1] != :on_sp }
|
||||
non_sp_index = tokens[0..(i - 1)].rindex{ |t| t.event != :on_sp }
|
||||
first_in_fomula = false
|
||||
if non_sp_index.nil?
|
||||
first_in_fomula = true
|
||||
elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index][1])
|
||||
elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index].event)
|
||||
first_in_fomula = true
|
||||
end
|
||||
if tk[1] == :on_kw && tk[2] == 'for'
|
||||
if tk.event == :on_kw && tk.tok == 'for'
|
||||
# A loop syntax in front of "do" found.
|
||||
#
|
||||
# while cond do # also "until" or "for"
|
||||
|
|
@ -541,14 +543,14 @@ class RubyLex
|
|||
@tokens.each_with_index do |t, index|
|
||||
# detecting one-liner method definition
|
||||
if in_oneliner_def.nil?
|
||||
if t[3].allbits?(Ripper::EXPR_ENDFN)
|
||||
if t.state.allbits?(Ripper::EXPR_ENDFN)
|
||||
in_oneliner_def = :ENDFN
|
||||
end
|
||||
else
|
||||
if t[3].allbits?(Ripper::EXPR_ENDFN)
|
||||
if t.state.allbits?(Ripper::EXPR_ENDFN)
|
||||
# continuing
|
||||
elsif t[3].allbits?(Ripper::EXPR_BEG)
|
||||
if t[2] == '='
|
||||
elsif t.state.allbits?(Ripper::EXPR_BEG)
|
||||
if t.tok == '='
|
||||
in_oneliner_def = :BODY
|
||||
end
|
||||
else
|
||||
|
|
@ -560,7 +562,7 @@ class RubyLex
|
|||
end
|
||||
end
|
||||
|
||||
case t[1]
|
||||
case t.event
|
||||
when :on_ignored_nl, :on_nl, :on_comment
|
||||
if index != (@tokens.size - 1) and in_oneliner_def != :BODY
|
||||
depth_difference = 0
|
||||
|
|
@ -570,15 +572,16 @@ class RubyLex
|
|||
when :on_sp
|
||||
next
|
||||
end
|
||||
case t[1]
|
||||
|
||||
case t.event
|
||||
when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg
|
||||
depth_difference += 1
|
||||
open_brace_on_line += 1
|
||||
when :on_rbracket, :on_rbrace, :on_rparen
|
||||
depth_difference -= 1 if open_brace_on_line > 0
|
||||
when :on_kw
|
||||
next if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME)
|
||||
case t[2]
|
||||
next if index > 0 and @tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
|
||||
case t.tok
|
||||
when 'do'
|
||||
syntax_of_do = take_corresponding_syntax_to_kw_do(@tokens, index)
|
||||
depth_difference += 1 if syntax_of_do == :method_calling
|
||||
|
|
@ -586,7 +589,7 @@ class RubyLex
|
|||
depth_difference += 1
|
||||
when 'if', 'unless', 'while', 'until', 'rescue'
|
||||
# postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL
|
||||
unless t[3].allbits?(Ripper::EXPR_LABEL)
|
||||
unless t.state.allbits?(Ripper::EXPR_LABEL)
|
||||
depth_difference += 1
|
||||
end
|
||||
when 'else', 'elsif', 'ensure', 'when'
|
||||
|
|
@ -619,14 +622,14 @@ class RubyLex
|
|||
@tokens.each_with_index do |t, index|
|
||||
# detecting one-liner method definition
|
||||
if in_oneliner_def.nil?
|
||||
if t[3].allbits?(Ripper::EXPR_ENDFN)
|
||||
if t.state.allbits?(Ripper::EXPR_ENDFN)
|
||||
in_oneliner_def = :ENDFN
|
||||
end
|
||||
else
|
||||
if t[3].allbits?(Ripper::EXPR_ENDFN)
|
||||
if t.state.allbits?(Ripper::EXPR_ENDFN)
|
||||
# continuing
|
||||
elsif t[3].allbits?(Ripper::EXPR_BEG)
|
||||
if t[2] == '='
|
||||
elsif t.state.allbits?(Ripper::EXPR_BEG)
|
||||
if t.tok == '='
|
||||
in_oneliner_def = :BODY
|
||||
end
|
||||
else
|
||||
|
|
@ -643,7 +646,7 @@ class RubyLex
|
|||
end
|
||||
end
|
||||
|
||||
case t[1]
|
||||
case t.event
|
||||
when :on_ignored_nl, :on_nl, :on_comment
|
||||
if in_oneliner_def != :BODY
|
||||
corresponding_token_depth = nil
|
||||
|
|
@ -654,11 +657,12 @@ class RubyLex
|
|||
end
|
||||
next
|
||||
when :on_sp
|
||||
spaces_at_line_head = t[2].count(' ') if is_first_spaces_of_line
|
||||
spaces_at_line_head = t.tok.count(' ') if is_first_spaces_of_line
|
||||
is_first_spaces_of_line = false
|
||||
next
|
||||
end
|
||||
case t[1]
|
||||
|
||||
case t.event
|
||||
when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg
|
||||
spaces_of_nest.push(spaces_at_line_head + open_brace_on_line * 2)
|
||||
open_brace_on_line += 1
|
||||
|
|
@ -671,8 +675,8 @@ class RubyLex
|
|||
end
|
||||
open_brace_on_line -= 1
|
||||
when :on_kw
|
||||
next if index > 0 and @tokens[index - 1][3].allbits?(Ripper::EXPR_FNAME)
|
||||
case t[2]
|
||||
next if index > 0 and @tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME)
|
||||
case t.tok
|
||||
when 'do'
|
||||
syntax_of_do = take_corresponding_syntax_to_kw_do(@tokens, index)
|
||||
if syntax_of_do == :method_calling
|
||||
|
|
@ -681,12 +685,12 @@ class RubyLex
|
|||
when 'def', 'case', 'for', 'begin', 'class', 'module'
|
||||
spaces_of_nest.push(spaces_at_line_head)
|
||||
when 'rescue'
|
||||
unless t[3].allbits?(Ripper::EXPR_LABEL)
|
||||
unless t.state.allbits?(Ripper::EXPR_LABEL)
|
||||
corresponding_token_depth = spaces_of_nest.last
|
||||
end
|
||||
when 'if', 'unless', 'while', 'until'
|
||||
# postfix if/unless/while/until must be Ripper::EXPR_LABEL
|
||||
unless t[3].allbits?(Ripper::EXPR_LABEL)
|
||||
unless t.state.allbits?(Ripper::EXPR_LABEL)
|
||||
spaces_of_nest.push(spaces_at_line_head)
|
||||
end
|
||||
when 'else', 'elsif', 'ensure', 'when', 'in'
|
||||
|
|
@ -712,7 +716,7 @@ class RubyLex
|
|||
end_type = []
|
||||
while i < tokens.size
|
||||
t = tokens[i]
|
||||
case t[1]
|
||||
case t.event
|
||||
when *end_type.last
|
||||
start_token.pop
|
||||
end_type.pop
|
||||
|
|
@ -725,7 +729,7 @@ class RubyLex
|
|||
when :on_symbeg
|
||||
acceptable_single_tokens = %i{on_ident on_const on_op on_cvar on_ivar on_gvar on_kw on_int on_backtick}
|
||||
if (i + 1) < tokens.size
|
||||
if acceptable_single_tokens.all?{ |st| tokens[i + 1][1] != st }
|
||||
if acceptable_single_tokens.all?{ |st| tokens[i + 1].event != st }
|
||||
start_token << t
|
||||
end_type << :on_tstring_end
|
||||
else
|
||||
|
|
@ -749,9 +753,11 @@ class RubyLex
|
|||
|
||||
def process_literal_type(tokens = @tokens)
|
||||
start_token = check_string_literal(tokens)
|
||||
case start_token[1]
|
||||
return nil if start_token == ""
|
||||
|
||||
case start_token.event
|
||||
when :on_tstring_beg
|
||||
case start_token[2]
|
||||
case start_token.tok
|
||||
when ?" then ?"
|
||||
when /^%.$/ then ?"
|
||||
when /^%Q.$/ then ?"
|
||||
|
|
@ -766,7 +772,7 @@ class RubyLex
|
|||
when :on_qsymbols_beg then ?]
|
||||
when :on_symbols_beg then ?]
|
||||
when :on_heredoc_beg
|
||||
start_token[2] =~ /<<[-~]?(['"`])[_a-zA-Z0-9]+\1/
|
||||
start_token.tok =~ /<<[-~]?(['"`])[_a-zA-Z0-9]+\1/
|
||||
case $1
|
||||
when ?" then ?"
|
||||
when ?' then ?'
|
||||
|
|
@ -794,6 +800,7 @@ class RubyLex
|
|||
false
|
||||
end
|
||||
end
|
||||
|
||||
if index
|
||||
first_token = nil
|
||||
last_line_tokens = tokens[(index + 1)..(tokens.size - 1)]
|
||||
|
|
@ -803,6 +810,7 @@ class RubyLex
|
|||
break
|
||||
end
|
||||
end
|
||||
|
||||
if first_token.nil?
|
||||
return false
|
||||
elsif first_token && first_token.state == Ripper::EXPR_DOT
|
||||
|
|
|
|||
|
|
@ -581,8 +581,8 @@ module TestIRB
|
|||
tokens = RubyLex.ripper_lex_without_warning('%wwww')
|
||||
pos_to_index = {}
|
||||
tokens.each_with_index { |t, i|
|
||||
assert_nil(pos_to_index[t[0]], "There is already another token in the position of #{t.inspect}.")
|
||||
pos_to_index[t[0]] = i
|
||||
assert_nil(pos_to_index[t.pos], "There is already another token in the position of #{t.inspect}.")
|
||||
pos_to_index[t.pos] = i
|
||||
}
|
||||
end
|
||||
|
||||
|
|
@ -598,8 +598,8 @@ module TestIRB
|
|||
EOC
|
||||
pos_to_index = {}
|
||||
tokens.each_with_index { |t, i|
|
||||
assert_nil(pos_to_index[t[0]], "There is already another token in the position of #{t.inspect}.")
|
||||
pos_to_index[t[0]] = i
|
||||
assert_nil(pos_to_index[t.pos], "There is already another token in the position of #{t.inspect}.")
|
||||
pos_to_index[t.pos] = i
|
||||
}
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in a new issue