kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

Remove the scraper

Browse source 
Now that we're using the jit function entry point, we don't need the
scraper.  Thank you for your service, scraper. ❤️
This commit is contained in:
Aaron Patterson 2021-07-15 14:30:43 -07:00 committed by Alan Wu
parent 84a1e04e58
commit 41f405c486
12 changed files with 5 additions and 364 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitignore vendored
View file

@ -228,6 +228,3 @@ lcov*.info
/rb_mjit_header.h
/mjit_config.h
/include/ruby-*/*/rb_mjit_min_header-*.h
# YJIT
/yjit_hooks.inc

4
common.mk
View file

@ -1111,8 +1111,6 @@ incs: $(INSNS) {$(VPATH)}node_name.inc {$(VPATH)}known_errors.inc \
insns: $(INSNS)
yjit_hooks.inc: vm.$(OBJEXT)
id.h: $(tooldir)/generic_erb.rb $(srcdir)/template/id.h.tmpl $(srcdir)/defs/id.def
$(ECHO) generating $@
$(Q) $(BASERUBY) $(tooldir)/generic_erb.rb --output=$@ \
@ -17106,7 +17104,6 @@ yjit_compile.$(OBJEXT): {$(VPATH)}vm_sync.h
yjit_compile.$(OBJEXT): {$(VPATH)}yjit.h
yjit_compile.$(OBJEXT): {$(VPATH)}yjit_asm.h
yjit_compile.$(OBJEXT): {$(VPATH)}yjit_compile.c
yjit_compile.$(OBJEXT): {$(VPATH)}yjit_hooks.inc
yjit_compile.$(OBJEXT): {$(VPATH)}yjit_utils.h
yjit_core.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h
yjit_core.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h
@ -17503,7 +17500,6 @@ yjit_iface.$(OBJEXT): {$(VPATH)}yjit.rbinc
yjit_iface.$(OBJEXT): {$(VPATH)}yjit_asm.h
yjit_iface.$(OBJEXT): {$(VPATH)}yjit_codegen.h
yjit_iface.$(OBJEXT): {$(VPATH)}yjit_core.h
yjit_iface.$(OBJEXT): {$(VPATH)}yjit_hooks.inc
yjit_iface.$(OBJEXT): {$(VPATH)}yjit_iface.c
yjit_iface.$(OBJEXT): {$(VPATH)}yjit_iface.h
yjit_utils.$(OBJEXT): {$(VPATH)}yjit_asm.h

4
template/Makefile.in
View file

@ -590,7 +590,7 @@ update-known-errors:
$(IFCHANGE) $(srcdir)/defs/known_errors.def -
INSNS = opt_sc.inc optinsn.inc optunifs.inc insns.inc insns_info.inc \
vmtc.inc vm.inc mjit_compile.inc yjit_hooks.inc
vmtc.inc vm.inc mjit_compile.inc
$(INSNS): $(srcdir)/insns.def vm_opts.h \
$(srcdir)/defs/opt_operand.def $(srcdir)/defs/opt_insn_unif.def \
@ -610,8 +610,6 @@ $(INSNS): $(srcdir)/insns.def vm_opts.h \
$(tooldir)/ruby_vm/models/instructions_unifications.rb \
$(tooldir)/ruby_vm/models/operands_unifications.rb \
$(tooldir)/ruby_vm/models/trace_instructions.rb \
$(tooldir)/ruby_vm/models/yjit.rb \
$(tooldir)/ruby_vm/models/yjit/example_instructions.rb \
$(tooldir)/ruby_vm/models/typemap.rb \
$(tooldir)/ruby_vm/scripts/converter.rb \
$(tooldir)/ruby_vm/scripts/insns2vm.rb \

4
tool/ruby_vm/models/instructions.rb
View file

@ -13,12 +13,10 @@
require_relative 'bare_instructions'
require_relative 'operands_unifications'
require_relative 'instructions_unifications'
require_relative 'yjit'
RubyVM::Instructions = RubyVM::BareInstructions.to_a + \
RubyVM::OperandsUnifications.to_a + \
RubyVM::InstructionsUnifications.to_a + \
RubyVM::YJIT::ExampleInstructions.to_a
RubyVM::InstructionsUnifications.to_a
require_relative 'trace_instructions'

221
tool/ruby_vm/models/yjit.rb
View file

@ -1,221 +0,0 @@
#! /your/favourite/path/to/ruby
# -*- Ruby -*-
# -*- frozen_string_literal: true; -*-
# -*- warn_indent: true; -*-
#
# Copyright (c) 2020 Wu, Alan. All rights reserved.
#
# This file is a part of the programming language Ruby. Permission is hereby
# granted, to either redistribute and/or modify this file, provided that the
# conditions mentioned in the file COPYING are met. Consult the file for
# details.
module RubyVM::YJIT
ScrapeResult = Struct.new(:pre_call_bytes, :post_call_bytes, :disassembly_lines)
class << self
def target_platform
# Note, checking RUBY_PLATRFORM doesn't work when cross compiling
@platform ||= if RUBY_PLATFORM.include?('darwin')
:darwin
elsif RUBY_PLATFORM.include?('linux')
:linux
else
:unknown
end
end
def get_fileoff
# use the load command to figure out the offset to the start of the content of vm.o
`otool -l vm.o`.each_line do |line|
if (fileoff = line[/fileoff (\d+)/, 1])
p [__method__, line] if $DEBUG
return fileoff.to_i
end
end
raise
end
def get_symbol_offset(symbol)
`nm vm.o`.each_line do |line|
if (offset = line[Regexp.compile('(\h+).+' + Regexp.escape(symbol) + '\Z'), 1])
p [__method__, line] if $DEBUG
return Integer(offset, 16)
end
end
raise
end
def readint8b(offset)
bytes = IO.binread('vm.o', 8, offset)
bytes.unpack('q').first # this is native endian but we want little endian. it's fine if the host moachine is x86
end
def get_symbol_section_and_offset(name)
`objdump -w -t vm.o`.each_line do |line|
split_line = line.split
next unless split_line.size >= 6
# the table should go into a data section
if split_line[5].include?('insns_address_table') && split_line[3].include?('data')
p line if $DEBUG
return [split_line[3], Integer(split_line[0], 16)]
end
end
raise 'Failed to find section and offset for the instruction address table'
end
def get_handler_offset(table_section, table_offset, insn_id)
target_offset = insn_id * 8 + table_offset
reloc_start_message = "RELOCATION RECORDS FOR [#{table_section}]:"
`objdump -w -r vm.o`.each_line do |line|
line.strip!
if (line == reloc_start_message)...(line.empty?)
split_line = line.split
next if split_line.first == 'RELOCATION'
next if split_line == ['OFFSET', 'TYPE', 'VALUE']
if Integer(split_line.first, 16) == target_offset
section, offset = split_line[2].split('+')
p line if $DEBUG
return section, Integer(offset, 16)
end
end
end
raise 'Failed to find relocation info for the target instruction'
end
def objdump_disassemble_command(offset)
case target_platform
when :darwin
"objdump --x86-asm-syntax=intel --start-address=#{offset} --stop-address=#{offset+50} -d vm.o"
when :linux
"objdump -M intel --start-address=#{offset} --stop-address=#{offset+50} -d vm.o"
else
raise "unkown platform"
end
end
def disassemble(offset)
command = objdump_disassemble_command(offset)
puts "Running: #{command}"
disassembly = `#{command}`
instructions = []
puts disassembly if $DEBUG
disassembly.each_line do |line|
line = line.strip
match_data = /\s*\h+:\s*((?:\h\h\s)+)\s+(\w+)/.match(line)
if match_data
bytes = match_data[1]
mnemonic = match_data[2]
instructions << [bytes, mnemonic, line]
break if mnemonic == 'jmp'
elsif !instructions.empty?
p line
raise "expected a continuous sequence of disassembly lines"
end
end
jmp_idx = instructions.find_index { |_, mnemonic, _| mnemonic == 'jmp' }
raise 'failed to find jmp' unless jmp_idx
raise 'generated code for example too long' unless jmp_idx < 10
handler_instructions = instructions[(0..jmp_idx)]
disassembly_lines = handler_instructions.map {|_, _, line| line}
puts "Disassembly for the example handler:"
puts disassembly_lines
raise 'rip reference in example makes copying unsafe' if handler_instructions.any? { |_, _, full_line| full_line.downcase.include?('rip') }
acceptable_mnemonics = %w(mov jmp lea call endbr64)
unrecognized = nil
handler_instructions.each { |i| unrecognized = i unless acceptable_mnemonics.include?(i[1]) }
raise "found an unrecognized \"#{unrecognized[1]}\" instruction in the example. List of recognized instructions: #{acceptable_mnemonics.join(', ')}" if unrecognized
raise 'found multiple jmp instructions' if handler_instructions.count { |_, mnemonic, _| mnemonic == 'jmp' } > 1
raise "the jmp instruction seems to be relative which isn't copiable" if instructions[jmp_idx][0].split.size > 4
raise 'no call instructions found' if handler_instructions.count { |_, mnemonic, _| mnemonic == 'call' } == 0
raise 'found multiple call instructions' if handler_instructions.count { |_, mnemonic, _| mnemonic == 'call' } > 1
call_idx = handler_instructions.find_index { |_, mnemonic, _| mnemonic == 'call' }
pre_call_bytes = []
post_call_bytes = []
handler_instructions.take(call_idx).each do |bytes, mnemonic, _|
pre_call_bytes += bytes.split
end
handler_instructions[call_idx + 1, handler_instructions.size].each do |bytes, _, _|
post_call_bytes += bytes.split
end
ScrapeResult.new(
comma_separated_hex_string(pre_call_bytes),
comma_separated_hex_string(post_call_bytes),
disassembly_lines
)
end
def darwin_scrape(instruction_id)
fileoff = get_fileoff
tc_table_offset = get_symbol_offset('vm_exec_core.insns_address_table')
vm_exec_core_offset = get_symbol_offset('vm_exec_core')
p instruction_id if $DEBUG
p fileoff if $DEBUG
p tc_table_offset.to_s(16) if $DEBUG
offset_to_insn_in_tc_table = fileoff + tc_table_offset + 8 * instruction_id
p offset_to_insn_in_tc_table if $DEBUG
offset_to_handler_code_from_vm_exec_core = readint8b(offset_to_insn_in_tc_table)
p offset_to_handler_code_from_vm_exec_core if $DEBUG
disassemble(vm_exec_core_offset + offset_to_handler_code_from_vm_exec_core)
end
def linux_scrape(instruction_id)
table_section, table_offset = get_symbol_section_and_offset('vm_exec_core.insns_address_table')
p [table_section, table_offset] if $DEBUG
handler_section, handler_offset = get_handler_offset(table_section, table_offset, instruction_id)
p [handler_section, handler_offset] if $DEBUG
disassemble(handler_offset)
end
def make_result(success, with_pc)
[success ? 1 : 0,
[
['yjit_with_ec', with_pc],
]
]
end
def scrape_instruction(instruction_id)
raise unless instruction_id.is_a?(Integer)
case target_platform
when :darwin
darwin_scrape(instruction_id)
when :linux
linux_scrape(instruction_id)
else
raise 'Unknown platform. Only Mach-O on macOS and ELF on Linux are supported'
end
end
def scrape
with_ec = scrape_instruction(RubyVM::Instructions.find_index { |insn| insn.name == 'yjit_call_example_with_ec' })
make_result(true, with_ec)
rescue => e
print_warning("scrape failed: #{e.message}")
int3 = '0xcc'
failure_result = ScrapeResult.new(int3, int3, ['int3'])
make_result(false, failure_result)
end
def print_warning(text)
text = "yjit warning: #{text}"
text = "\x1b[1m#{text}\x1b[0m" if STDOUT.tty?
STDOUT.puts(text)
end
def comma_separated_hex_string(nums)
nums.map{ |byte| '0x'+byte}.join(', ')
end
end
end
require_relative 'yjit/example_instructions'

69
tool/ruby_vm/models/yjit/example_instructions.rb
View file

@ -1,69 +0,0 @@
#! /your/favourite/path/to/ruby
# -*- Ruby -*-
# -*- frozen_string_literal: true; -*-
# -*- warn_indent: true; -*-
#
# Copyright (c) 2020 Wu, Alan. All rights reserved.
#
# This file is a part of the programming language Ruby. Permission is hereby
# granted, to either redistribute and/or modify this file, provided that the
# conditions mentioned in the file COPYING are met. Consult the file for
# details.
class RubyVM::YJIT::ExampleInstructions
include RubyVM::CEscape
attr_reader :name
def initialize(name)
@name = name
end
def pretty_name
return sprintf "%s(...)(...)(...)", @name
end
def jump_destination
return @orig.name
end
def bin
return sprintf "BIN(%s)", @name
end
def width
1
end
def operands_info
""
end
def rets
return ['...']
end
def pops
return ['...']
end
def attributes
return []
end
def has_attribute? *;
return false
end
def handles_sp?
false
end
def always_leaf?
false
end
def self.to_a
[new('yjit_call_example_with_ec')]
end
end

14
tool/ruby_vm/views/vm.inc.erb
View file

@ -25,20 +25,6 @@
<%= render 'insn_entry', locals: { insn: insn } -%>
% end
%
% RubyVM::YJIT::ExampleInstructions.to_a.each do |insn|
INSN_ENTRY(yjit_call_example_with_ec)
{
START_OF_ORIGINAL_INSN(yjit_call_example_with_ec);
#if USE_MACHINE_REGS
// assumes USE_MACHINE_REGS, aka reg_pc setup,
// aka #define SET_PC(x) (reg_cfp->pc = reg_pc = (x))
rb_yjit_empty_func_with_ec(GET_CFP(), ec);
RESTORE_REGS();
#endif
END_INSN(yjit_call_example_with_ec);
}
% end
%
% RubyVM::TraceInstructions.to_a.each do |insn|
<%= render 'trace_instruction', locals: { insn: insn } -%>
% end

24
tool/ruby_vm/views/yjit_hooks.inc.erb
View file

@ -1,24 +0,0 @@
/* -*- C -*- */
%# Copyright (c) 2020 Wu, Alan. All rights reserved.
%#
%# This file is a part of the programming language Ruby. Permission is hereby
%# granted, to either redistribute and/or modify this file, provided that the
%# conditions mentioned in the file COPYING are met. Consult the file for
%# details.
<%= render 'copyright' %>
<%= render 'notice', locals: {
this_file: 'contains raw instruction bytes that helps YJIT generate code',
edit: __FILE__,
} -%>
% success, byte_arrays = RubyVM::YJIT.scrape
static const uint8_t yjit_scrape_successful = <%= success %>;
% byte_arrays.each do |(prefix, scrape_result)|
// Disassembly:
% scrape_result.disassembly_lines.each do |line|
// <%= line %>
% end
static const uint8_t <%= prefix %>_pre_call_bytes[] = { <%= scrape_result.pre_call_bytes %> };
static const uint8_t <%= prefix %>_post_call_bytes[] = { <%= scrape_result.post_call_bytes %> };
% end

2
vm_exec.h
View file

@ -81,7 +81,7 @@ error !
RSTRING_PTR(rb_iseq_path(reg_cfp->iseq)), \
rb_iseq_line_no(reg_cfp->iseq, reg_pc - reg_cfp->iseq->body->iseq_encoded)); \
} \
if (USE_INSNS_COUNTER && BIN(insn) != BIN(yjit_call_example_with_ec)) vm_insns_counter_count_insn(BIN(insn));
if (USE_INSNS_COUNTER) vm_insns_counter_count_insn(BIN(insn));
#define INSN_DISPATCH_SIG(insn)

2
win32/Makefile.sub
View file

@ -1347,7 +1347,7 @@ $(MJIT_PRECOMPILED_HEADER): $(MJIT_PRECOMPILED_HEADER_NAME)
$(Q) $(MAKE_LINK) $(MJIT_PRECOMPILED_HEADER_NAME:.pch=.pdb) $(arch_hdrdir)/$(MJIT_PRECOMPILED_HEADER_NAME:.pch=.pdb)
INSNS = opt_sc.inc optinsn.inc optunifs.inc insns.inc insns_info.inc \
vmtc.inc vm.inc mjit_compile.inc yjit_hooks.inc
vmtc.inc vm.inc mjit_compile.inc
!if [exit > insns_rules.mk]
!else if [for %I in ($(INSNS)) do \

19
yjit_iface.c
View file

@ -13,7 +13,6 @@
#include "yjit_iface.h"
#include "yjit_codegen.h"
#include "yjit_core.h"
#include "yjit_hooks.inc"
#include "darray.h"
#if HAVE_LIBCAPSTONE
@ -48,22 +47,6 @@ static const rb_data_type_t yjit_block_type = {
0, 0, RUBY_TYPED_FREE_IMMEDIATELY
};
// Write the YJIT entry point pre-call bytes
void
cb_write_pre_call_bytes(codeblock_t* cb)
{
for (size_t i = 0; i < sizeof(yjit_with_ec_pre_call_bytes); ++i)
cb_write_byte(cb, yjit_with_ec_pre_call_bytes[i]);
}
// Write the YJIT exit post-call bytes
void
cb_write_post_call_bytes(codeblock_t* cb)
{
for (size_t i = 0; i < sizeof(yjit_with_ec_post_call_bytes); ++i)
cb_write_byte(cb, yjit_with_ec_post_call_bytes[i]);
}
// Get the PC for a given index in an iseq
VALUE *
yjit_iseq_pc_at_idx(const rb_iseq_t *iseq, uint32_t insn_idx)
@ -1021,7 +1004,7 @@ outgoing_ids(VALUE self)
void
rb_yjit_init(struct rb_yjit_options *options)
{
if (!yjit_scrape_successful || !PLATFORM_SUPPORTED_P) {
if (!PLATFORM_SUPPORTED_P) {
return;
}

3
yjit_iface.h
View file

@ -96,9 +96,6 @@ extern yjit_comment_array_t yjit_code_comments;
RUBY_EXTERN struct rb_yjit_options rb_yjit_opts;
RUBY_EXTERN struct rb_yjit_runtime_counters yjit_runtime_counters;
void cb_write_pre_call_bytes(codeblock_t* cb);
void cb_write_post_call_bytes(codeblock_t* cb);
void yjit_map_addr2insn(void *code_ptr, int insn);
VALUE *yjit_iseq_pc_at_idx(const rb_iseq_t *iseq, uint32_t insn_idx);
int yjit_opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc);