From 89e7997622038f82115f34dbb4ea382e02bed163 Mon Sep 17 00:00:00 2001 From: Alan Wu Date: Tue, 30 Jul 2019 21:36:05 -0400 Subject: [PATCH] Combine call info and cache to speed up method invocation To perform a regular method call, the VM needs two structs, `rb_call_info` and `rb_call_cache`. At the moment, we allocate these two structures in separate buffers. In the worst case, the CPU needs to read 4 cache lines to complete a method call. Putting the two structures together reduces the maximum number of cache line reads to 2. Combining the structures also saves 8 bytes per call site as the current layout uses separate two pointers for the call info and the call cache. This saves about 2 MiB on Discourse. This change improves the Optcarrot benchmark at least 3%. For more details, see attached bugs.ruby-lang.org ticket. Complications: - A new instruction attribute `comptime_sp_inc` is introduced to calculate SP increase at compile time without using call caches. At compile time, a `TS_CALLDATA` operand points to a call info struct, but at runtime, the same operand points to a call data struct. Instruction that explicitly define `sp_inc` also need to define `comptime_sp_inc`. - MJIT code for copying call cache becomes slightly more complicated. - This changes the bytecode format, which might break existing tools. [Misc #16258] --- common.mk | 2 +- compile.c | 237 +++++++++--------- insns.def | 100 ++++---- iseq.c | 45 ++-- mjit.c | 11 +- mjit_compile.c | 20 +- tool/ruby_vm/models/attribute.rb | 19 +- tool/ruby_vm/models/bare_instructions.rb | 21 ++ tool/ruby_vm/models/typemap.rb | 3 +- ....erb => _comptime_insn_stack_increase.erb} | 22 +- tool/ruby_vm/views/_mjit_compile_insn.erb | 4 +- tool/ruby_vm/views/_mjit_compile_send.erb | 5 +- tool/ruby_vm/views/insns_info.inc.erb | 2 +- tool/ruby_vm/views/mjit_compile.inc.erb | 4 +- vm_core.h | 23 +- vm_insnhelper.c | 66 +++-- vm_insnhelper.h | 2 +- 17 files changed, 322 insertions(+), 264 deletions(-) rename tool/ruby_vm/views/{_insn_stack_increase.erb => _comptime_insn_stack_increase.erb} (58%) diff --git a/common.mk b/common.mk index d44a62e174..5c580b156b 100644 --- a/common.mk +++ b/common.mk @@ -974,7 +974,7 @@ $(srcs_vpath)insns.inc: $(srcdir)/tool/ruby_vm/views/insns.inc.erb $(inc_common_ $(srcs_vpath)insns_info.inc: $(srcdir)/tool/ruby_vm/views/insns_info.inc.erb $(inc_common_headers) \ $(srcdir)/tool/ruby_vm/views/_insn_type_chars.erb $(srcdir)/tool/ruby_vm/views/_insn_name_info.erb \ $(srcdir)/tool/ruby_vm/views/_insn_len_info.erb $(srcdir)/tool/ruby_vm/views/_insn_operand_info.erb \ - $(srcdir)/tool/ruby_vm/views/_attributes.erb $(srcdir)/tool/ruby_vm/views/_insn_stack_increase.erb \ + $(srcdir)/tool/ruby_vm/views/_attributes.erb $(srcdir)/tool/ruby_vm/views/_comptime_insn_stack_increase.erb \ $(srcdir)/tool/ruby_vm/views/_insn_sp_pc_dependency.erb $(srcs_vpath)vmtc.inc: $(srcdir)/tool/ruby_vm/views/vmtc.inc.erb $(inc_common_headers) $(srcs_vpath)vm.inc: $(srcdir)/tool/ruby_vm/views/vm.inc.erb $(inc_common_headers) \ diff --git a/compile.c b/compile.c index 0ca8757c05..d0f4ce406e 100644 --- a/compile.c +++ b/compile.c @@ -546,6 +546,31 @@ verify_list(ISEQ_ARG_DECLARE const char *info, LINK_ANCHOR *const anchor) #define verify_list(info, anchor) verify_list(iseq, (info), (anchor)) #endif +static void +verify_call_cache(rb_iseq_t *iseq) +{ + return; /* comment out to enable */ + + VALUE *original = rb_iseq_original_iseq(iseq); + size_t i = 0; + while (i < iseq->body->iseq_size) { + VALUE insn = original[i]; + const char *types = insn_op_types(insn); + + for (int j=0; types[j]; j++) { + if (types[j] == TS_CALLDATA) { + struct rb_call_cache cc; + struct rb_call_data *cd = (struct rb_call_data *)original[i+j+1]; + MEMZERO(&cc, cc, 1); + if (memcmp(&cc, &cd->cc, sizeof(cc))) { + rb_bug("call cache not zero for fresh iseq"); + } + } + } + i += insn_len(insn); + } +} + /* * elem1, elem2 => elem1, elem2, elem */ @@ -1178,11 +1203,10 @@ new_callinfo(rb_iseq_t *iseq, ID mid, int argc, unsigned int flag, struct rb_cal static INSN * new_insn_send(rb_iseq_t *iseq, int line_no, ID id, VALUE argc, const rb_iseq_t *blockiseq, VALUE flag, struct rb_call_info_kw_arg *keywords) { - VALUE *operands = compile_data_alloc2(iseq, sizeof(VALUE), 3); + VALUE *operands = compile_data_alloc2(iseq, sizeof(VALUE), 2); operands[0] = (VALUE)new_callinfo(iseq, id, FIX2INT(argc), FIX2INT(flag), keywords, blockiseq != NULL); - operands[1] = Qfalse; /* cache */ - operands[2] = (VALUE)blockiseq; - return new_insn_core(iseq, line_no, BIN(send), 3, operands); + operands[1] = (VALUE)blockiseq; + return new_insn_core(iseq, line_no, BIN(send), 2, operands); } static rb_iseq_t * @@ -1358,6 +1382,7 @@ iseq_setup(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) VALUE str = rb_iseq_disasm(iseq); printf("%s\n", StringValueCStr(str)); } + verify_call_cache(iseq); debugs("[compile step: finish]\n"); return COMPILE_OK; @@ -2090,13 +2115,10 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) insns_info = ALLOC_N(struct iseq_insn_info_entry, insn_num); positions = ALLOC_N(unsigned int, insn_num); body->is_entries = ZALLOC_N(union iseq_inline_storage_entry, body->is_size); - body->ci_entries = - rb_xmalloc_mul_add_mul( - sizeof(struct rb_call_info), body->ci_size, - sizeof(struct rb_call_info_with_kwarg), body->ci_kw_size); - MEMZERO(body->ci_entries + body->ci_size, struct rb_call_info_with_kwarg, body->ci_kw_size); /* need to clear ci_kw entries */ - body->cc_entries = ZALLOC_N(struct rb_call_cache, body->ci_size + body->ci_kw_size); - + body->call_data = + rb_xcalloc_mul_add_mul( + sizeof(struct rb_call_data), body->ci_size, + sizeof(struct rb_kwarg_call_data), body->ci_kw_size); ISEQ_COMPILE_DATA(iseq)->ci_index = ISEQ_COMPILE_DATA(iseq)->ci_kw_index = 0; list = FIRST_ELEMENT(anchor); @@ -2180,33 +2202,27 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor) generated_iseq[code_index + 1 + j] = (VALUE)ic; break; } - case TS_CALLINFO: /* call info */ - { - struct rb_call_info *base_ci = (struct rb_call_info *)operands[j]; - struct rb_call_info *ci; + case TS_CALLDATA: + { + struct rb_call_info *source_ci = (struct rb_call_info *)operands[j]; + struct rb_call_data *cd; - if (base_ci->flag & VM_CALL_KWARG) { - struct rb_call_info_with_kwarg *ci_kw_entries = (struct rb_call_info_with_kwarg *)&body->ci_entries[body->ci_size]; - struct rb_call_info_with_kwarg *ci_kw = &ci_kw_entries[ISEQ_COMPILE_DATA(iseq)->ci_kw_index++]; - *ci_kw = *((struct rb_call_info_with_kwarg *)base_ci); - ci = (struct rb_call_info *)ci_kw; - assert(ISEQ_COMPILE_DATA(iseq)->ci_kw_index <= body->ci_kw_size); - } - else { - ci = &body->ci_entries[ISEQ_COMPILE_DATA(iseq)->ci_index++]; - *ci = *base_ci; - assert(ISEQ_COMPILE_DATA(iseq)->ci_index <= body->ci_size); - } + if (source_ci->flag & VM_CALL_KWARG) { + struct rb_kwarg_call_data *kw_calls = (struct rb_kwarg_call_data *)&body->call_data[body->ci_size]; + struct rb_kwarg_call_data *cd_kw = &kw_calls[ISEQ_COMPILE_DATA(iseq)->ci_kw_index++]; + cd_kw->ci_kw = *((struct rb_call_info_with_kwarg *)source_ci); + cd = (struct rb_call_data *)cd_kw; + assert(ISEQ_COMPILE_DATA(iseq)->ci_kw_index <= body->ci_kw_size); + } + else { + cd = &body->call_data[ISEQ_COMPILE_DATA(iseq)->ci_index++]; + cd->ci = *source_ci; + assert(ISEQ_COMPILE_DATA(iseq)->ci_index <= body->ci_size); + } - generated_iseq[code_index + 1 + j] = (VALUE)ci; - break; - } - case TS_CALLCACHE: - { - struct rb_call_cache *cc = &body->cc_entries[ISEQ_COMPILE_DATA(iseq)->ci_index + ISEQ_COMPILE_DATA(iseq)->ci_kw_index - 1]; - generated_iseq[code_index + 1 + j] = (VALUE)cc; - break; - } + generated_iseq[code_index + 1 + j] = (VALUE)cd; + break; + } case TS_ID: /* ID */ generated_iseq[code_index + 1 + j] = SYM2ID(operands[j]); break; @@ -2530,7 +2546,7 @@ remove_unreachable_chunk(rb_iseq_t *iseq, LINK_ELEMENT *i) case TS_OFFSET: unref_destination((INSN *)i, pos); break; - case TS_CALLINFO: + case TS_CALLDATA: if (((struct rb_call_info *)OPERAND_AT(i, pos))->flag & VM_CALL_KWARG) --(body->ci_kw_size); else @@ -3147,9 +3163,9 @@ iseq_peephole_optimize(rb_iseq_t *iseq, LINK_ELEMENT *list, const int do_tailcal } if (piobj) { - struct rb_call_info *ci = (struct rb_call_info *)piobj->operands[0]; + struct rb_call_info *ci = (struct rb_call_info *)OPERAND_AT(piobj, 0); if (IS_INSN_ID(piobj, send) || IS_INSN_ID(piobj, invokesuper)) { - if (piobj->operands[2] == 0) { /* no blockiseq */ + if (OPERAND_AT(piobj, 1) == 0) { /* no blockiseq */ ci->flag |= VM_CALL_TAILCALL; } } @@ -3208,12 +3224,10 @@ insn_set_specialized_instruction(rb_iseq_t *iseq, INSN *iobj, int insn_id) if (insn_id == BIN(opt_neq)) { VALUE *old_operands = iobj->operands; - iobj->operand_size = 4; + iobj->operand_size = 2; iobj->operands = compile_data_alloc2(iseq, iobj->operand_size, sizeof(VALUE)); - iobj->operands[0] = (VALUE)new_callinfo(iseq, idEq, 1, 0, NULL, FALSE); - iobj->operands[1] = Qfalse; /* CALL_CACHE */ - iobj->operands[2] = old_operands[0]; - iobj->operands[3] = Qfalse; /* CALL_CACHE */ + iobj->operands[0] = (VALUE)new_callinfo(iseq, idEq, 1, 0, NULL, FALSE); + iobj->operands[1] = old_operands[0]; } return COMPILE_OK; @@ -3247,7 +3261,7 @@ iseq_specialized_instruction(rb_iseq_t *iseq, INSN *iobj) if (IS_INSN_ID(iobj, send)) { struct rb_call_info *ci = (struct rb_call_info *)OPERAND_AT(iobj, 0); - const rb_iseq_t *blockiseq = (rb_iseq_t *)OPERAND_AT(iobj, 2); + const rb_iseq_t *blockiseq = (rb_iseq_t *)OPERAND_AT(iobj, 1); #define SP_INSN(opt) insn_set_specialized_instruction(iseq, iobj, BIN(opt_##opt)) if (ci->flag & VM_CALL_ARGS_SIMPLE) { @@ -4348,7 +4362,7 @@ compile_massign_lhs(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *const n CHECK(COMPILE_POPPED(ret, "masgn lhs (NODE_ATTRASGN)", node)); iobj = (INSN *)get_prev_insn((INSN *)LAST_ELEMENT(ret)); /* send insn */ - ci = (struct rb_call_info *)iobj->operands[0]; + ci = (struct rb_call_info *)OPERAND_AT(iobj, 0); ci->orig_argc += 1; dupidx = INT2FIX(ci->orig_argc); @@ -6667,14 +6681,12 @@ compile_call_precheck_freeze(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE ISEQ_COMPILE_DATA(iseq)->option->specialized_instruction) { VALUE str = rb_fstring(node->nd_recv->nd_lit); if (node->nd_mid == idUMinus) { - ADD_INSN3(ret, line, opt_str_uminus, str, - new_callinfo(iseq, idUMinus, 0, 0, NULL, FALSE), - Qundef /* CALL_CACHE */); + ADD_INSN2(ret, line, opt_str_uminus, str, + new_callinfo(iseq, idUMinus, 0, 0, NULL, FALSE)); } else { - ADD_INSN3(ret, line, opt_str_freeze, str, - new_callinfo(iseq, idFreeze, 0, 0, NULL, FALSE), - Qundef /* CALL_CACHE */); + ADD_INSN2(ret, line, opt_str_freeze, str, + new_callinfo(iseq, idFreeze, 0, 0, NULL, FALSE)); } RB_OBJ_WRITTEN(iseq, Qundef, str); if (popped) { @@ -6693,9 +6705,8 @@ compile_call_precheck_freeze(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE ISEQ_COMPILE_DATA(iseq)->option->specialized_instruction) { VALUE str = rb_fstring(node->nd_args->nd_head->nd_lit); CHECK(COMPILE(ret, "recv", node->nd_recv)); - ADD_INSN3(ret, line, opt_aref_with, str, - new_callinfo(iseq, idAREF, 1, 0, NULL, FALSE), - NULL/* CALL_CACHE */); + ADD_INSN2(ret, line, opt_aref_with, str, + new_callinfo(iseq, idAREF, 1, 0, NULL, FALSE)); RB_OBJ_WRITTEN(iseq, Qundef, str); if (popped) { ADD_INSN(ret, line, pop); @@ -7565,9 +7576,8 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, in ADD_INSN(ret, line, putself); ADD_SEQ(ret, args); - ADD_INSN3(ret, line, invokesuper, - new_callinfo(iseq, 0, argc, flag | VM_CALL_SUPER | (type == NODE_ZSUPER ? VM_CALL_ZSUPER : 0) | VM_CALL_FCALL, keywords, parent_block != NULL), - Qnil, /* CALL_CACHE */ + ADD_INSN2(ret, line, invokesuper, + new_callinfo(iseq, 0, argc, flag | VM_CALL_SUPER | (type == NODE_ZSUPER ? VM_CALL_ZSUPER : 0) | VM_CALL_FCALL, keywords, parent_block != NULL), parent_block); if (popped) { @@ -8262,9 +8272,8 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, in ADD_INSN(ret, line, swap); ADD_INSN1(ret, line, topn, INT2FIX(1)); } - ADD_INSN3(ret, line, opt_aset_with, str, - new_callinfo(iseq, idASET, 2, 0, NULL, FALSE), - NULL/* CALL_CACHE */); + ADD_INSN2(ret, line, opt_aset_with, str, + new_callinfo(iseq, idASET, 2, 0, NULL, FALSE)); RB_OBJ_WRITTEN(iseq, Qundef, str); ADD_INSN(ret, line, pop); break; @@ -8367,7 +8376,7 @@ insn_data_length(INSN *iobj) static int calc_sp_depth(int depth, INSN *insn) { - return insn_stack_increase(depth, insn->insn_id, insn->operands); + return comptime_insn_stack_increase(depth, insn->insn_id, insn->operands); } static VALUE @@ -8441,17 +8450,12 @@ insn_data_to_s_detail(INSN *iobj) case TS_ISE: /* inline storage entry */ rb_str_catf(str, "", FIX2INT(OPERAND_AT(iobj, j))); break; - case TS_CALLINFO: /* call info */ + case TS_CALLDATA: /* we store these as call infos at compile time */ { - struct rb_call_info *ci = (struct rb_call_info *)OPERAND_AT(iobj, j); - rb_str_cat2(str, "mid) rb_str_catf(str, "%"PRIsVALUE, rb_id2str(ci->mid)); - rb_str_catf(str, ", %d>", ci->orig_argc); - break; - } - case TS_CALLCACHE: /* call cache */ - { - rb_str_catf(str, ""); + const struct rb_call_info *ci = (struct rb_call_info *)OPERAND_AT(iobj, j); + rb_str_cat2(str, "mid) rb_str_catf(str, "%"PRIsVALUE, rb_id2str(ci->mid)); + rb_str_catf(str, ", %d>", ci->orig_argc); break; } case TS_CDHASH: /* case/when condition cache */ @@ -8832,12 +8836,9 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor, iseq->body->is_size = NUM2INT(op) + 1; } break; - case TS_CALLINFO: + case TS_CALLDATA: argv[j] = iseq_build_callinfo_from_hash(iseq, op); break; - case TS_CALLCACHE: - argv[j] = Qfalse; - break; case TS_ID: argv[j] = rb_to_symbol_type(op); break; @@ -9487,9 +9488,9 @@ ibf_load_id(const struct ibf_load *load, const ID id_index) /* dump/load: code */ static VALUE -ibf_dump_callinfo(struct ibf_dump *dump, const struct rb_call_info *ci) +ibf_dump_calldata(struct ibf_dump *dump, const struct rb_call_data *cd) { - return (ci->flag & VM_CALL_KWARG) ? Qtrue : Qfalse; + return (cd->ci.flag & VM_CALL_KWARG) ? Qtrue : Qfalse; } static ibf_offset_t ibf_dump_iseq_each(struct ibf_dump *dump, const rb_iseq_t *iseq); @@ -9646,16 +9647,13 @@ ibf_dump_code(struct ibf_dump *dump, const rb_iseq_t *iseq) ibf_dump_write_small_value(dump, (VALUE)i); } break; - case TS_CALLINFO: + case TS_CALLDATA: { - VALUE callinfo = ibf_dump_callinfo(dump, (const struct rb_call_info *)op); - /* ibf_dump_callinfo() always returns either Qtrue or Qfalse */ + VALUE callinfo = ibf_dump_calldata(dump, (const struct rb_call_data *)op); + /* ibf_dump_calldata() always returns either Qtrue or Qfalse */ ibf_dump_write_byte(dump, callinfo == Qtrue); } break; - case TS_CALLCACHE: - /* do nothing */ - break; case TS_ID: ibf_dump_write_small_value(dump, ibf_dump_id(dump, (ID)op)); break; @@ -9684,9 +9682,8 @@ ibf_load_code(const struct ibf_load *load, const rb_iseq_t *iseq, ibf_offset_t b VALUE *code = ALLOC_N(VALUE, iseq_size); struct rb_iseq_constant_body *load_body = iseq->body; - struct rb_call_info *ci_entries = load_body->ci_entries; - struct rb_call_info_with_kwarg *ci_kw_entries = (struct rb_call_info_with_kwarg *)&load_body->ci_entries[load_body->ci_size]; - struct rb_call_cache *cc_entries = load_body->cc_entries; + struct rb_call_data *cd_entries = load_body->call_data; + struct rb_kwarg_call_data *cd_kw_entries = (struct rb_kwarg_call_data *)&load_body->call_data[load_body->ci_size]; union iseq_inline_storage_entry *is_entries = load_body->is_entries; for (code_index=0; code_indexbody; const unsigned int ci_size = body->ci_size; const unsigned int ci_kw_size = body->ci_kw_size; - const struct rb_call_info *ci_entries = body->ci_entries; - const struct rb_call_info_with_kwarg *ci_kw_entries = (const struct rb_call_info_with_kwarg *)&body->ci_entries[ci_size]; + const struct rb_call_data *calls = body->call_data; + const struct rb_kwarg_call_data *kw_calls = (const struct rb_kwarg_call_data *)&body->call_data[ci_size]; ibf_offset_t offset = ibf_dump_pos(dump); unsigned int i; for (i = 0; i < ci_size; i++) { - VALUE mid = ibf_dump_id(dump, ci_entries[i].mid); + VALUE mid = ibf_dump_id(dump, calls[i].ci.mid); ibf_dump_write_small_value(dump, mid); - ibf_dump_write_small_value(dump, ci_entries[i].flag); - ibf_dump_write_small_value(dump, ci_entries[i].orig_argc); + ibf_dump_write_small_value(dump, calls[i].ci.flag); + ibf_dump_write_small_value(dump, calls[i].ci.orig_argc); } for (i = 0; i < ci_kw_size; i++) { - const struct rb_call_info_kw_arg *kw_arg = ci_kw_entries[i].kw_arg; + const struct rb_call_info_kw_arg *kw_arg = kw_calls[i].ci_kw.kw_arg; - VALUE mid = ibf_dump_id(dump, ci_kw_entries[i].ci.mid); + VALUE mid = ibf_dump_id(dump, kw_calls[i].ci_kw.ci.mid); ibf_dump_write_small_value(dump, mid); - ibf_dump_write_small_value(dump, ci_kw_entries[i].ci.flag); - ibf_dump_write_small_value(dump, ci_kw_entries[i].ci.orig_argc); + ibf_dump_write_small_value(dump, kw_calls[i].ci_kw.ci.flag); + ibf_dump_write_small_value(dump, kw_calls[i].ci_kw.ci.orig_argc); ibf_dump_write_small_value(dump, kw_arg->keyword_len); int j; - for (j = 0; j < ci_kw_entries[i].kw_arg->keyword_len; j++) { + for (j = 0; j < kw_calls[i].ci_kw.kw_arg->keyword_len; j++) { VALUE keyword = ibf_dump_object(dump, kw_arg->keywords[j]); /* kw_arg->keywords[n] is Symbol */ ibf_dump_write_small_value(dump, keyword); @@ -10043,7 +10037,8 @@ ibf_dump_ci_entries(struct ibf_dump *dump, const rb_iseq_t *iseq) return offset; } -static struct rb_call_info * +/* note that we dump out rb_call_info but load back rb_call_data */ +static struct rb_call_data * ibf_load_ci_entries(const struct ibf_load *load, ibf_offset_t ci_entries_offset, unsigned int ci_size, @@ -10053,43 +10048,43 @@ ibf_load_ci_entries(const struct ibf_load *load, unsigned int i; - struct rb_call_info *ci_entries = - rb_xmalloc_mul_add_mul( - sizeof(struct rb_call_info), ci_size, - sizeof(struct rb_call_info_with_kwarg), ci_kw_size); - struct rb_call_info_with_kwarg *ci_kw_entries = (struct rb_call_info_with_kwarg *)&ci_entries[ci_size]; + struct rb_call_data *calls = + rb_xcalloc_mul_add_mul( + sizeof(struct rb_call_data), ci_size, + sizeof(struct rb_kwarg_call_data), ci_kw_size); + struct rb_kwarg_call_data *kw_calls = (struct rb_kwarg_call_data *)&calls[ci_size]; for (i = 0; i < ci_size; i++) { VALUE mid_index = ibf_load_small_value(load, &reading_pos); - ci_entries[i].mid = ibf_load_id(load, mid_index); - ci_entries[i].flag = (unsigned int)ibf_load_small_value(load, &reading_pos); - ci_entries[i].orig_argc = (int)ibf_load_small_value(load, &reading_pos); + calls[i].ci.mid = ibf_load_id(load, mid_index); + calls[i].ci.flag = (unsigned int)ibf_load_small_value(load, &reading_pos); + calls[i].ci.orig_argc = (int)ibf_load_small_value(load, &reading_pos); } for (i = 0; i < ci_kw_size; i++) { VALUE mid_index = ibf_load_small_value(load, &reading_pos); - ci_kw_entries[i].ci.mid = ibf_load_id(load, mid_index); - ci_kw_entries[i].ci.flag = (unsigned int)ibf_load_small_value(load, &reading_pos); - ci_kw_entries[i].ci.orig_argc = (int)ibf_load_small_value(load, &reading_pos); + kw_calls[i].ci_kw.ci.mid = ibf_load_id(load, mid_index); + kw_calls[i].ci_kw.ci.flag = (unsigned int)ibf_load_small_value(load, &reading_pos); + kw_calls[i].ci_kw.ci.orig_argc = (int)ibf_load_small_value(load, &reading_pos); int keyword_len = (int)ibf_load_small_value(load, &reading_pos); - ci_kw_entries[i].kw_arg = + kw_calls[i].ci_kw.kw_arg = rb_xmalloc_mul_add(keyword_len - 1, sizeof(VALUE), sizeof(struct rb_call_info_kw_arg)); - ci_kw_entries[i].kw_arg->keyword_len = keyword_len; + kw_calls[i].ci_kw.kw_arg->keyword_len = keyword_len; int j; - for (j = 0; j < ci_kw_entries[i].kw_arg->keyword_len; j++) { + for (j = 0; j < kw_calls[i].ci_kw.kw_arg->keyword_len; j++) { VALUE keyword = ibf_load_small_value(load, &reading_pos); - ci_kw_entries[i].kw_arg->keywords[j] = ibf_load_object(load, keyword); + kw_calls[i].ci_kw.kw_arg->keywords[j] = ibf_load_object(load, keyword); } } - return ci_entries; + return calls; } static ibf_offset_t @@ -10343,8 +10338,7 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset) load_body->catch_except_p = catch_except_p; load_body->is_entries = ZALLOC_N(union iseq_inline_storage_entry, is_size); - load_body->ci_entries = ibf_load_ci_entries(load, ci_entries_offset, ci_size, ci_kw_size); - load_body->cc_entries = ZALLOC_N(struct rb_call_cache, ci_size + ci_kw_size); + load_body->call_data = ibf_load_ci_entries(load, ci_entries_offset, ci_size, ci_kw_size); load_body->param.opt_table = ibf_load_param_opt_table(load, param_opt_table_offset, param_opt_num); load_body->param.keyword = ibf_load_param_keyword(load, param_keyword_offset); load_body->param.flags.has_kw = (param_flags >> 4) & 1; @@ -10399,6 +10393,7 @@ ibf_load_iseq_each(struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t offset) #if IBF_ISEQ_ENABLE_LOCAL_BUFFER load->current_buffer = saved_buffer; #endif + verify_call_cache(iseq); } static void diff --git a/insns.def b/insns.def index f365106e2c..b59ae6a760 100644 --- a/insns.def +++ b/insns.def @@ -783,13 +783,14 @@ definesmethod /* invoke method. */ DEFINE_INSN send -(CALL_INFO ci, CALL_CACHE cc, ISEQ blockiseq) +(CALL_DATA cd, ISEQ blockiseq) (...) (VALUE val) -// attr rb_snum_t sp_inc = sp_inc_of_sendish(ci); +// attr rb_snum_t sp_inc = sp_inc_of_sendish(&cd->ci); +// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci); { - VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), ci, blockiseq, false); - val = vm_sendish(ec, GET_CFP(), ci, cc, bh, vm_search_method_wrap); + VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), &cd->ci, blockiseq, false); + val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap); if (val == Qundef) { RESTORE_REGS(); @@ -800,14 +801,15 @@ send /* Invoke method without block */ DEFINE_INSN opt_send_without_block -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (...) (VALUE val) // attr bool handles_sp = true; -// attr rb_snum_t sp_inc = sp_inc_of_sendish(ci); +// attr rb_snum_t sp_inc = sp_inc_of_sendish(&cd->ci); +// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci); { VALUE bh = VM_BLOCK_HANDLER_NONE; - val = vm_sendish(ec, GET_CFP(), ci, cc, bh, vm_search_method_wrap); + val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_method_wrap); if (val == Qundef) { RESTORE_REGS(); @@ -817,7 +819,7 @@ opt_send_without_block DEFINE_INSN opt_str_freeze -(VALUE str, CALL_INFO ci, CALL_CACHE cc) +(VALUE str, CALL_DATA cd) () (VALUE val) { @@ -832,11 +834,11 @@ opt_str_freeze /* optimized nil? */ DEFINE_INSN opt_nil_p -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv) (VALUE val) { - val = vm_opt_nil_p(ci, cc, recv); + val = vm_opt_nil_p(cd, recv); if (val == Qundef) { CALL_SIMPLE_METHOD(); @@ -845,7 +847,7 @@ opt_nil_p DEFINE_INSN opt_str_uminus -(VALUE str, CALL_INFO ci, CALL_CACHE cc) +(VALUE str, CALL_DATA cd) () (VALUE val) { @@ -887,13 +889,14 @@ opt_newarray_min /* super(args) # args.size => num */ DEFINE_INSN invokesuper -(CALL_INFO ci, CALL_CACHE cc, ISEQ blockiseq) +(CALL_DATA cd, ISEQ blockiseq) (...) (VALUE val) -// attr rb_snum_t sp_inc = sp_inc_of_sendish(ci); +// attr rb_snum_t sp_inc = sp_inc_of_sendish(&cd->ci); +// attr rb_snum_t comptime_sp_inc = sp_inc_of_sendish(ci); { - VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), ci, blockiseq, true); - val = vm_sendish(ec, GET_CFP(), ci, cc, bh, vm_search_super_method); + VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), &cd->ci, blockiseq, true); + val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_super_method); if (val == Qundef) { RESTORE_REGS(); @@ -904,18 +907,19 @@ invokesuper /* yield(args) */ DEFINE_INSN invokeblock -(CALL_INFO ci) +(CALL_DATA cd) (...) (VALUE val) // attr bool handles_sp = true; -// attr rb_snum_t sp_inc = sp_inc_of_invokeblock(ci); +// attr rb_snum_t sp_inc = sp_inc_of_invokeblock(&cd->ci); +// attr rb_snum_t comptime_sp_inc = sp_inc_of_invokeblock(ci); { - static struct rb_call_cache cc = { - 0, 0, NULL, NULL, vm_invokeblock_i, - }; + if (UNLIKELY(cd->cc.call != vm_invokeblock_i)) { + cd->cc.call = vm_invokeblock_i; // check before setting to avoid CoW + } VALUE bh = VM_BLOCK_HANDLER_NONE; - val = vm_sendish(ec, GET_CFP(), ci, &cc, bh, vm_search_invokeblock); + val = vm_sendish(ec, GET_CFP(), cd, bh, vm_search_invokeblock); if (val == Qundef) { RESTORE_REGS(); @@ -1098,7 +1102,7 @@ opt_case_dispatch /* optimized X+Y. */ DEFINE_INSN opt_plus -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) { @@ -1112,7 +1116,7 @@ opt_plus /* optimized X-Y. */ DEFINE_INSN opt_minus -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) { @@ -1126,7 +1130,7 @@ opt_minus /* optimized X*Y. */ DEFINE_INSN opt_mult -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) { @@ -1140,7 +1144,7 @@ opt_mult /* optimized X/Y. */ DEFINE_INSN opt_div -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) /* In case of division by zero, it raises. Thus @@ -1157,7 +1161,7 @@ opt_div /* optimized X%Y. */ DEFINE_INSN opt_mod -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) /* Same discussion as opt_mod. */ @@ -1173,11 +1177,11 @@ opt_mod /* optimized X==Y. */ DEFINE_INSN opt_eq -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) { - val = opt_eq_func(recv, obj, ci, cc); + val = opt_eq_func(recv, obj, cd); if (val == Qundef) { CALL_SIMPLE_METHOD(); @@ -1187,11 +1191,11 @@ opt_eq /* optimized X!=Y. */ DEFINE_INSN opt_neq -(CALL_INFO ci_eq, CALL_CACHE cc_eq, CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd_eq, CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) { - val = vm_opt_neq(ci, cc, ci_eq, cc_eq, recv, obj); + val = vm_opt_neq(cd, cd_eq, recv, obj); if (val == Qundef) { CALL_SIMPLE_METHOD(); @@ -1201,7 +1205,7 @@ opt_neq /* optimized XY. */ DEFINE_INSN opt_gt -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) { @@ -1243,7 +1247,7 @@ opt_gt /* optimized X>=Y. */ DEFINE_INSN opt_ge -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) { @@ -1257,7 +1261,7 @@ opt_ge /* << */ DEFINE_INSN opt_ltlt -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) /* This instruction can append an integer, as a codepoint, into a @@ -1275,7 +1279,7 @@ opt_ltlt /* optimized X&Y. */ DEFINE_INSN opt_and -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) { @@ -1289,7 +1293,7 @@ opt_and /* optimized X|Y. */ DEFINE_INSN opt_or -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) { @@ -1303,7 +1307,7 @@ opt_or /* [] */ DEFINE_INSN opt_aref -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj) (VALUE val) /* This is complicated. In case of hash, vm_opt_aref() resorts to @@ -1322,7 +1326,7 @@ opt_aref /* recv[obj] = set */ DEFINE_INSN opt_aset -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv, VALUE obj, VALUE set) (VALUE val) /* This is another story than opt_aref. When vm_opt_aset() resorts @@ -1339,7 +1343,7 @@ opt_aset /* recv[str] = set */ DEFINE_INSN opt_aset_with -(VALUE key, CALL_INFO ci, CALL_CACHE cc) +(VALUE key, CALL_DATA cd) (VALUE recv, VALUE val) (VALUE val) /* Same discussion as opt_aset. */ @@ -1362,7 +1366,7 @@ opt_aset_with /* recv[str] */ DEFINE_INSN opt_aref_with -(VALUE key, CALL_INFO ci, CALL_CACHE cc) +(VALUE key, CALL_DATA cd) (VALUE recv) (VALUE val) /* Same discussion as opt_aref. */ @@ -1381,7 +1385,7 @@ opt_aref_with /* optimized length */ DEFINE_INSN opt_length -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv) (VALUE val) { @@ -1395,7 +1399,7 @@ opt_length /* optimized size */ DEFINE_INSN opt_size -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv) (VALUE val) { @@ -1409,7 +1413,7 @@ opt_size /* optimized empty? */ DEFINE_INSN opt_empty_p -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv) (VALUE val) { @@ -1423,7 +1427,7 @@ opt_empty_p /* optimized succ */ DEFINE_INSN opt_succ -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv) (VALUE val) { @@ -1437,11 +1441,11 @@ opt_succ /* optimized not */ DEFINE_INSN opt_not -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE recv) (VALUE val) { - val = vm_opt_not(ci, cc, recv); + val = vm_opt_not(cd, recv); if (val == Qundef) { CALL_SIMPLE_METHOD(); @@ -1451,7 +1455,7 @@ opt_not /* optimized regexp match 2 */ DEFINE_INSN opt_regexpmatch2 -(CALL_INFO ci, CALL_CACHE cc) +(CALL_DATA cd) (VALUE obj2, VALUE obj1) (VALUE val) // attr bool leaf = false; /* match_at() has rb_thread_check_ints() */ diff --git a/iseq.c b/iseq.c index 10d0723a83..c9fef82aef 100644 --- a/iseq.c +++ b/iseq.c @@ -102,15 +102,14 @@ rb_iseq_free(const rb_iseq_t *iseq) ruby_xfree((void *)body->local_table); ruby_xfree((void *)body->is_entries); - if (body->ci_entries) { + if (body->call_data) { unsigned int i; - struct rb_call_info_with_kwarg *ci_kw_entries = (struct rb_call_info_with_kwarg *)&body->ci_entries[body->ci_size]; + struct rb_kwarg_call_data *kw_calls = (struct rb_kwarg_call_data *)&body->call_data[body->ci_size]; for (i=0; ici_kw_size; i++) { - const struct rb_call_info_kw_arg *kw_arg = ci_kw_entries[i].kw_arg; + const struct rb_call_info_kw_arg *kw_arg = kw_calls[i].ci_kw.kw_arg; ruby_xfree((void *)kw_arg); } - ruby_xfree(body->ci_entries); - ruby_xfree(body->cc_entries); + ruby_xfree(body->call_data); } ruby_xfree((void *)body->catch_table); ruby_xfree((void *)body->param.opt_table); @@ -379,7 +378,7 @@ rb_iseq_memsize(const rb_iseq_t *iseq) /* TODO: should we count original_iseq? */ if (ISEQ_EXECUTABLE_P(iseq) && body) { - struct rb_call_info_with_kwarg *ci_kw_entries = (struct rb_call_info_with_kwarg *)&body->ci_entries[body->ci_size]; + struct rb_kwarg_call_data *kw_calls = (struct rb_kwarg_call_data *)&body->call_data[body->ci_size]; size += sizeof(struct rb_iseq_constant_body); size += body->iseq_size * sizeof(VALUE); @@ -394,19 +393,15 @@ rb_iseq_memsize(const rb_iseq_t *iseq) /* body->is_entries */ size += body->is_size * sizeof(union iseq_inline_storage_entry); - /* body->ci_entries */ - size += body->ci_size * sizeof(struct rb_call_info); - size += body->ci_kw_size * sizeof(struct rb_call_info_with_kwarg); + /* body->call_data */ + size += body->ci_size * sizeof(struct rb_call_data); + size += body->ci_kw_size * sizeof(struct rb_kwarg_call_data); - /* body->cc_entries */ - size += body->ci_size * sizeof(struct rb_call_cache); - size += body->ci_kw_size * sizeof(struct rb_call_cache); - - if (ci_kw_entries) { + if (kw_calls) { unsigned int i; for (i = 0; i < body->ci_kw_size; i++) { - const struct rb_call_info_kw_arg *kw_arg = ci_kw_entries[i].kw_arg; + const struct rb_call_info_kw_arg *kw_arg = kw_calls[i].ci_kw.kw_arg; if (kw_arg) { size += rb_call_info_kw_arg_bytes(kw_arg->keyword_len); @@ -1916,9 +1911,10 @@ rb_insn_operand_intern(const rb_iseq_t *iseq, ret = rb_sprintf("", (union iseq_inline_storage_entry *)op - iseq->body->is_entries); break; - case TS_CALLINFO: + case TS_CALLDATA: { - struct rb_call_info *ci = (struct rb_call_info *)op; + struct rb_call_data *cd = (struct rb_call_data *)op; + struct rb_call_info *ci = &cd->ci; VALUE ary = rb_ary_new(); if (ci->mid) { @@ -1950,12 +1946,9 @@ rb_insn_operand_intern(const rb_iseq_t *iseq, CALL_FLAG(OPT_SEND); /* maybe not reachable */ rb_ary_push(ary, rb_ary_join(flags, rb_str_new2("|"))); } - ret = rb_sprintf("", rb_ary_join(ary, rb_str_new2(", "))); - } - break; - case TS_CALLCACHE: - ret = rb_str_new2(""); + ret = rb_sprintf("", rb_ary_join(ary, rb_str_new2(", "))); + } break; case TS_CDHASH: @@ -2723,9 +2716,10 @@ iseq_data_to_ary(const rb_iseq_t *iseq) rb_ary_push(ary, INT2FIX(is - iseq_body->is_entries)); } break; - case TS_CALLINFO: + case TS_CALLDATA: { - struct rb_call_info *ci = (struct rb_call_info *)*seq; + struct rb_call_data *cd = (struct rb_call_data *)*seq; + struct rb_call_info *ci = &cd->ci; VALUE e = rb_hash_new(); int orig_argc = ci->orig_argc; @@ -2749,9 +2743,6 @@ iseq_data_to_ary(const rb_iseq_t *iseq) rb_ary_push(ary, e); } break; - case TS_CALLCACHE: - rb_ary_push(ary, Qfalse); - break; case TS_ID: rb_ary_push(ary, ID2SYM(*seq)); break; diff --git a/mjit.c b/mjit.c index 3ae9410179..f1e3934326 100644 --- a/mjit.c +++ b/mjit.c @@ -46,7 +46,16 @@ mjit_copy_job_handler(void *data) const struct rb_iseq_constant_body *body = job->iseq->body; if (job->cc_entries) { - memcpy(job->cc_entries, body->cc_entries, sizeof(struct rb_call_cache) * (body->ci_size + body->ci_kw_size)); + unsigned int i; + struct rb_call_cache *sink = job->cc_entries; + const struct rb_call_data *calls = body->call_data; + const struct rb_kwarg_call_data *kw_calls = (struct rb_kwarg_call_data *)&body->call_data[body->ci_size]; + for (i = 0; i < body->ci_size; i++) { + *sink++ = calls[i].cc; + } + for (i = 0; i < body->ci_kw_size; i++) { + *sink++ = kw_calls[i].cc; + } } if (job->is_entries) { memcpy(job->is_entries, body->is_entries, sizeof(union iseq_inline_storage_entry) * body->is_size); diff --git a/mjit_compile.c b/mjit_compile.c index b6ed984b20..27ea836ef4 100644 --- a/mjit_compile.c +++ b/mjit_compile.c @@ -25,6 +25,21 @@ #define NOT_COMPILED_STACK_SIZE -1 #define ALREADY_COMPILED_P(status, pos) (status->stack_size_for_pos[pos] != NOT_COMPILED_STACK_SIZE) +static size_t +call_data_index(CALL_DATA cd, const struct rb_iseq_constant_body *body) +{ + const struct rb_kwarg_call_data *kw_calls = (const struct rb_kwarg_call_data *)&body->call_data[body->ci_size]; + const struct rb_kwarg_call_data *kw_cd = (const struct rb_kwarg_call_data *)cd; + + VM_ASSERT(cd >= body->call_data && kw_cd < (kw_calls + body->ci_kw_size)); + if (kw_cd < kw_calls) { + return cd - body->call_data; + } + else { + return kw_cd - kw_calls + body->ci_size; + } +} + // For propagating information needed for lazily pushing a frame. struct inlined_call_context { int orig_argc; // ci->orig_argc @@ -383,8 +398,9 @@ precompile_inlinable_iseqs(FILE *f, const rb_iseq_t *iseq, struct compile_status #endif if (insn == BIN(opt_send_without_block)) { // `compile_inlined_cancel_handler` supports only `opt_send_without_block` - CALL_INFO ci = (CALL_INFO)body->iseq_encoded[pos + 1]; - CALL_CACHE cc_copy = status->cc_entries + ((CALL_CACHE)body->iseq_encoded[pos + 2] - body->cc_entries); // use copy to avoid race condition + CALL_DATA cd = (CALL_DATA)body->iseq_encoded[pos + 1]; + CALL_INFO ci = &cd->ci; + CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); // use copy to avoid race condition const rb_iseq_t *child_iseq; if (has_valid_method_type(cc_copy) && diff --git a/tool/ruby_vm/models/attribute.rb b/tool/ruby_vm/models/attribute.rb index cc16a5f898..de35e7234a 100644 --- a/tool/ruby_vm/models/attribute.rb +++ b/tool/ruby_vm/models/attribute.rb @@ -21,6 +21,13 @@ class RubyVM::Attribute @key = opts[:name] @expr = RubyVM::CExpr.new location: opts[:location], expr: opts[:expr] @type = opts[:type] + @ope_decls = @insn.opes.map do |operand| + decl = operand[:decl] + if @key == 'comptime_sp_inc' && operand[:type] == 'CALL_DATA' + decl = decl.gsub('CALL_DATA', 'CALL_INFO').gsub('cd', 'ci') + end + decl + end end def name @@ -32,22 +39,20 @@ class RubyVM::Attribute end def declaration - opes = @insn.opes - if opes.empty? + if @ope_decls.empty? argv = "void" else - argv = opes.map {|o| o[:decl] }.join(', ') + argv = @ope_decls.join(', ') end sprintf '%s %s(%s)', @type, name, argv end def definition - opes = @insn.opes - if opes.empty? + if @ope_decls.empty? argv = "void" else - argv = opes.map {|o| "MAYBE_UNUSED(#{o[:decl]})" }.join(",\n ") - argv = "\n #{argv}\n" if opes.size > 1 + argv = @ope_decls.map {|decl| "MAYBE_UNUSED(#{decl})" }.join(",\n ") + argv = "\n #{argv}\n" if @ope_decls.size > 1 end sprintf "%s\n%s(%s)", @type, name, argv end diff --git a/tool/ruby_vm/models/bare_instructions.rb b/tool/ruby_vm/models/bare_instructions.rb index e0fac5ff91..b0cc83a65a 100755 --- a/tool/ruby_vm/models/bare_instructions.rb +++ b/tool/ruby_vm/models/bare_instructions.rb @@ -33,6 +33,7 @@ class RubyVM::BareInstructions h[a.key] = a } @attrs_orig = @attrs.dup + check_attribute_consistency predefine_attributes end @@ -139,8 +140,28 @@ class RubyVM::BareInstructions return @pops.any? {|i| i[:name] == var[:name] } end + def use_call_data? + @use_call_data ||= + @variables.find { |_, var_info| var_info[:type] == 'CALL_DATA' } + end + private + def check_attribute_consistency + if has_attribute?('sp_inc') \ + && use_call_data? \ + && !has_attribute?('comptime_sp_inc') + # As the call cache caches information that can only be obtained at + # runtime, we do not need it when compiling from AST to bytecode. This + # attribute defines an expression that computes the stack pointer + # increase based on just the call info to avoid reserving space for the + # call cache at compile time. In the expression, all call data operands + # are mapped to their call info counterpart. Additionally, all mentions + # of `cd` in the operand name are replaced with `ci`. + raise "Please define attribute `comptime_sp_inc` for `#{@name}`" + end + end + def generate_attribute t, k, v @attrs[k] ||= RubyVM::Attribute.new \ insn: self, \ diff --git a/tool/ruby_vm/models/typemap.rb b/tool/ruby_vm/models/typemap.rb index 65ddfea41d..1125c4bbf6 100644 --- a/tool/ruby_vm/models/typemap.rb +++ b/tool/ruby_vm/models/typemap.rb @@ -12,8 +12,7 @@ RubyVM::Typemap = { "..." => %w[. TS_VARIABLE], - "CALL_CACHE" => %w[E TS_CALLCACHE], - "CALL_INFO" => %w[C TS_CALLINFO], + "CALL_DATA" => %w[C TS_CALLDATA], "CDHASH" => %w[H TS_CDHASH], "GENTRY" => %w[G TS_GENTRY], "IC" => %w[K TS_IC], diff --git a/tool/ruby_vm/views/_insn_stack_increase.erb b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb similarity index 58% rename from tool/ruby_vm/views/_insn_stack_increase.erb rename to tool/ruby_vm/views/_comptime_insn_stack_increase.erb index 315d695cf1..b633ab4d32 100644 --- a/tool/ruby_vm/views/_insn_stack_increase.erb +++ b/tool/ruby_vm/views/_comptime_insn_stack_increase.erb @@ -6,11 +6,11 @@ %# conditions mentioned in the file COPYING are met. Consult the file for %# details. %# -PUREFUNC(MAYBE_UNUSED(static int insn_stack_increase(int depth, int insn, const VALUE *opes))); -PUREFUNC(static rb_snum_t insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); +PUREFUNC(MAYBE_UNUSED(static int comptime_insn_stack_increase(int depth, int insn, const VALUE *opes))); +PUREFUNC(static rb_snum_t comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes)); rb_snum_t -insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) +comptime_insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) { static const signed char t[] = { % RubyVM::Instructions.each_slice 8 do |a| @@ -34,9 +34,19 @@ insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) UNREACHABLE; % RubyVM::Instructions.each do |i| % next unless i.has_attribute?('sp_inc') +% attr_function = +% if i.has_attribute?('comptime_sp_inc') +% "attr_comptime_sp_inc_#{i.name}" +% else +% "attr_sp_inc_#{i.name}" +% end case <%= i.bin %>: - return attr_sp_inc_<%= i.name %>(<%= + return <%= attr_function %>(<%= i.opes.map.with_index do |v, j| + if v[:type] == 'CALL_DATA' && i.has_attribute?('comptime_sp_inc') + v = v.dup + v[:type] = 'CALL_INFO' + end i.cast_from_VALUE v, "opes[#{j}]" end.join(", ") %>); @@ -45,8 +55,8 @@ insn_stack_increase_dispatch(enum ruby_vminsn_type insn, const VALUE *opes) } int -insn_stack_increase(int depth, int insn, const VALUE *opes) +comptime_insn_stack_increase(int depth, int insn, const VALUE *opes) { enum ruby_vminsn_type itype = (enum ruby_vminsn_type)insn; - return depth + (int)insn_stack_increase_dispatch(itype, opes); + return depth + (int)comptime_insn_stack_increase_dispatch(itype, opes); } diff --git a/tool/ruby_vm/views/_mjit_compile_insn.erb b/tool/ruby_vm/views/_mjit_compile_insn.erb index 4488876da3..b2dea03e38 100644 --- a/tool/ruby_vm/views/_mjit_compile_insn.erb +++ b/tool/ruby_vm/views/_mjit_compile_insn.erb @@ -34,8 +34,8 @@ % case ope.fetch(:type) % when 'ID' comment_id(f, (ID)operands[<%= i %>]); -% when 'CALL_INFO' - comment_id(f, ((CALL_INFO)operands[<%= i %>])->mid); +% when 'CALL_DATA' + comment_id(f, ((CALL_DATA)operands[<%= i %>])->ci.mid); % when 'VALUE' if (SYMBOL_P((VALUE)operands[<%= i %>])) comment_id(f, SYM2ID((VALUE)operands[<%= i %>])); % end diff --git a/tool/ruby_vm/views/_mjit_compile_send.erb b/tool/ruby_vm/views/_mjit_compile_send.erb index da7e96581b..95e7846820 100644 --- a/tool/ruby_vm/views/_mjit_compile_send.erb +++ b/tool/ruby_vm/views/_mjit_compile_send.erb @@ -14,10 +14,11 @@ MAYBE_UNUSED(<%= ope.fetch(:decl) %>) = (<%= ope.fetch(:type) %>)operands[<%= i %>]; % end % # compiler: Use copied cc to avoid race condition - CALL_CACHE cc_copy = status->cc_entries + (cc - body->cc_entries); + CALL_CACHE cc_copy = status->cc_entries + call_data_index(cd, body); % if (!status->compile_info->disable_send_cache && has_valid_method_type(cc_copy)) { const rb_iseq_t *iseq; + const CALL_INFO ci = &cd->ci; unsigned int argc = ci->orig_argc; // this `argc` variable is for calculating a value's position on stack considering `blockarg`. % if insn.name == 'send' argc += ((ci->flag & VM_CALL_ARGS_BLOCKARG) ? 1 : 0); // simulate `vm_caller_setup_arg_block`'s `--reg_cfp->sp` @@ -58,7 +59,7 @@ fprintf(f, " {\n"); fprintf(f, " struct rb_calling_info calling;\n"); % if insn.name == 'send' - fprintf(f, " calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (CALL_INFO)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", operands[0], operands[2]); + fprintf(f, " calling.block_handler = vm_caller_setup_arg_block(ec, reg_cfp, (CALL_INFO)0x%"PRIxVALUE", (rb_iseq_t *)0x%"PRIxVALUE", FALSE);\n", (VALUE)ci, (VALUE)blockiseq); % else fprintf(f, " calling.block_handler = VM_BLOCK_HANDLER_NONE;\n"); % end diff --git a/tool/ruby_vm/views/insns_info.inc.erb b/tool/ruby_vm/views/insns_info.inc.erb index e5793a2a70..2ca5aca7cf 100644 --- a/tool/ruby_vm/views/insns_info.inc.erb +++ b/tool/ruby_vm/views/insns_info.inc.erb @@ -18,5 +18,5 @@ <%= render 'leaf_helpers' %> <%= render 'sp_inc_helpers' %> <%= render 'attributes' %> -<%= render 'insn_stack_increase' %> +<%= render 'comptime_insn_stack_increase' %> <%= render 'insn_sp_pc_dependency' %> diff --git a/tool/ruby_vm/views/mjit_compile.inc.erb b/tool/ruby_vm/views/mjit_compile.inc.erb index d9092a756d..95e71183d9 100644 --- a/tool/ruby_vm/views/mjit_compile.inc.erb +++ b/tool/ruby_vm/views/mjit_compile.inc.erb @@ -56,8 +56,8 @@ switch (insn) { <%= render 'mjit_compile_send', locals: { insn: insn } -%> % when *send_compatible_opt_insns % # To avoid cancel, just emit `opt_send_without_block` instead of `opt_*` insn if call cache is populated. -% cc_index = insn.opes.index { |o| o.fetch(:type) == 'CALL_CACHE' } - if (has_valid_method_type(status->cc_entries + ((CALL_CACHE)operands[<%= cc_index %>] - body->cc_entries))) { +% cd_index = insn.opes.index { |o| o.fetch(:type) == 'CALL_DATA' } + if (has_valid_method_type(status->cc_entries + call_data_index((CALL_DATA)operands[<%= cd_index %>], body))) { <%= render 'mjit_compile_send', locals: { insn: opt_send_without_block } -%> <%= render 'mjit_compile_insn', locals: { insn: opt_send_without_block } -%> break; diff --git a/vm_core.h b/vm_core.h index 685005336b..5c8c91b593 100644 --- a/vm_core.h +++ b/vm_core.h @@ -258,6 +258,16 @@ struct rb_calling_info { int kw_splat; }; +struct rb_call_data { + struct rb_call_cache cc; + struct rb_call_info ci; +}; + +struct rb_kwarg_call_data { + struct rb_call_cache cc; + struct rb_call_info_with_kwarg ci_kw; +}; + struct rb_execution_context_struct; typedef VALUE (*vm_call_handler)(struct rb_execution_context_struct *ec, struct rb_control_frame_struct *cfp, struct rb_calling_info *calling, const struct rb_call_info *ci, struct rb_call_cache *cc); @@ -417,12 +427,12 @@ struct rb_iseq_constant_body { struct rb_iseq_struct *local_iseq; /* local_iseq->flip_cnt can be modified */ union iseq_inline_storage_entry *is_entries; - struct rb_call_info *ci_entries; /* struct rb_call_info ci_entries[ci_size]; - * struct rb_call_info_with_kwarg cikw_entries[ci_kw_size]; - * So that: - * struct rb_call_info_with_kwarg *cikw_entries = &body->ci_entries[ci_size]; - */ - struct rb_call_cache *cc_entries; /* size is ci_size + ci_kw_size */ + struct rb_call_data *call_data; /* A buffer for two arrays: + * struct rb_call_data calls[ci_size]; + * struct rb_kwarg_call_data kw_calls[ci_kw_size]; + * Such that: + * struct rb_kwarg_call_data *kw_calls = &body->call_data[ci_size]; + */ struct { rb_snum_t flip_count; @@ -1121,6 +1131,7 @@ typedef struct iseq_inline_cache_entry *IC; typedef union iseq_inline_storage_entry *ISE; typedef struct rb_call_info *CALL_INFO; typedef struct rb_call_cache *CALL_CACHE; +typedef struct rb_call_data *CALL_DATA; void rb_vm_change_state(void); diff --git a/vm_insnhelper.c b/vm_insnhelper.c index c7dd213035..44a44d5db1 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -1463,17 +1463,16 @@ check_cfunc(const rb_callable_method_entry_t *me, VALUE (*func)()) } static inline int -vm_method_cfunc_is(CALL_INFO ci, CALL_CACHE cc, - VALUE recv, VALUE (*func)()) +vm_method_cfunc_is(CALL_DATA cd, VALUE recv, VALUE (*func)()) { - vm_search_method(ci, cc, recv); - return check_cfunc(cc->me, func); + vm_search_method(&cd->ci, &cd->cc, recv); + return check_cfunc(cd->cc.me, func); } static VALUE -opt_equal_fallback(VALUE recv, VALUE obj, CALL_INFO ci, CALL_CACHE cc) +opt_equal_fallback(VALUE recv, VALUE obj, CALL_DATA cd) { - if (vm_method_cfunc_is(ci, cc, recv, rb_obj_equal)) { + if (vm_method_cfunc_is(cd, recv, rb_obj_equal)) { return recv == obj ? Qtrue : Qfalse; } @@ -1533,7 +1532,7 @@ static inline #endif VALUE -opt_eq_func(VALUE recv, VALUE obj, CALL_INFO ci, CALL_CACHE cc) +opt_eq_func(VALUE recv, VALUE obj, CALL_DATA cd) { switch (comparable_by_identity(recv, obj)) { case 1: @@ -1556,7 +1555,7 @@ opt_eq_func(VALUE recv, VALUE obj, CALL_INFO ci, CALL_CACHE cc) } fallback: - return opt_equal_fallback(recv, obj, ci, cc); + return opt_equal_fallback(recv, obj, cd); } static @@ -1564,7 +1563,7 @@ static inline #endif VALUE -opt_eql_func(VALUE recv, VALUE obj, CALL_INFO ci, CALL_CACHE cc) +opt_eql_func(VALUE recv, VALUE obj, CALL_DATA cd) { switch (comparable_by_identity(recv, obj)) { case 1: @@ -1586,7 +1585,7 @@ opt_eql_func(VALUE recv, VALUE obj, CALL_INFO ci, CALL_CACHE cc) } fallback: - return opt_equal_fallback(recv, obj, ci, cc); + return opt_equal_fallback(recv, obj, cd); } #undef BUILTIN_CLASS_P #undef EQ_UNREDEFINED_P @@ -1594,27 +1593,25 @@ opt_eql_func(VALUE recv, VALUE obj, CALL_INFO ci, CALL_CACHE cc) VALUE rb_equal_opt(VALUE obj1, VALUE obj2) { - struct rb_call_info ci; - struct rb_call_cache cc; + struct rb_call_data cd; - ci.mid = idEq; - cc.method_state = 0; - cc.class_serial = 0; - cc.me = NULL; - return opt_eq_func(obj1, obj2, &ci, &cc); + cd.ci.mid = idEq; + cd.cc.method_state = 0; + cd.cc.class_serial = 0; + cd.cc.me = NULL; + return opt_eq_func(obj1, obj2, &cd); } VALUE rb_eql_opt(VALUE obj1, VALUE obj2) { - struct rb_call_info ci; - struct rb_call_cache cc; + struct rb_call_data cd; - ci.mid = idEqlP; - cc.method_state = 0; - cc.class_serial = 0; - cc.me = NULL; - return opt_eql_func(obj1, obj2, &ci, &cc); + cd.ci.mid = idEqlP; + cd.cc.method_state = 0; + cd.cc.class_serial = 0; + cd.cc.me = NULL; + return opt_eql_func(obj1, obj2, &cd); } extern VALUE rb_vm_call0(rb_execution_context_t *ec, VALUE, ID, int, const VALUE*, const rb_callable_method_entry_t *, int kw_splat); @@ -3752,8 +3749,7 @@ static VALUE vm_sendish( struct rb_execution_context_struct *ec, struct rb_control_frame_struct *reg_cfp, - struct rb_call_info *ci, - struct rb_call_cache *cc, + struct rb_call_data *cd, VALUE block_handler, void (*method_explorer)( const struct rb_control_frame_struct *reg_cfp, @@ -3761,6 +3757,8 @@ vm_sendish( struct rb_call_cache *cc, VALUE recv)) { + CALL_INFO ci = &cd->ci; + CALL_CACHE cc = &cd->cc; VALUE val; int argc = ci->orig_argc; VALUE recv = TOPN(argc); @@ -4116,12 +4114,10 @@ vm_opt_mod(VALUE recv, VALUE obj) } static VALUE -vm_opt_neq(CALL_INFO ci, CALL_CACHE cc, - CALL_INFO ci_eq, CALL_CACHE cc_eq, - VALUE recv, VALUE obj) +vm_opt_neq(CALL_DATA cd, CALL_DATA cd_eq, VALUE recv, VALUE obj) { - if (vm_method_cfunc_is(ci, cc, recv, rb_obj_not_equal)) { - VALUE val = opt_eq_func(recv, obj, ci_eq, cc_eq); + if (vm_method_cfunc_is(cd, recv, rb_obj_not_equal)) { + VALUE val = opt_eq_func(recv, obj, cd_eq); if (val != Qundef) { return RTEST(val) ? Qfalse : Qtrue; @@ -4392,7 +4388,7 @@ vm_opt_empty_p(VALUE recv) VALUE rb_false(VALUE obj); static VALUE -vm_opt_nil_p(CALL_INFO ci, CALL_CACHE cc, VALUE recv) +vm_opt_nil_p(CALL_DATA cd, VALUE recv) { if (recv == Qnil) { if (BASIC_OP_UNREDEFINED_P(BOP_NIL_P, NIL_REDEFINED_OP_FLAG)) { @@ -4403,7 +4399,7 @@ vm_opt_nil_p(CALL_INFO ci, CALL_CACHE cc, VALUE recv) } } else { - if (vm_method_cfunc_is(ci, cc, recv, rb_false)) { + if (vm_method_cfunc_is(cd, recv, rb_false)) { return Qfalse; } else { @@ -4460,9 +4456,9 @@ vm_opt_succ(VALUE recv) } static VALUE -vm_opt_not(CALL_INFO ci, CALL_CACHE cc, VALUE recv) +vm_opt_not(CALL_DATA cd, VALUE recv) { - if (vm_method_cfunc_is(ci, cc, recv, rb_obj_not)) { + if (vm_method_cfunc_is(cd, recv, rb_obj_not)) { return RTEST(recv) ? Qfalse : Qtrue; } else { diff --git a/vm_insnhelper.h b/vm_insnhelper.h index 9f09d4ab5e..99555fd4ed 100644 --- a/vm_insnhelper.h +++ b/vm_insnhelper.h @@ -166,7 +166,7 @@ enum vm_regan_acttype { #ifndef MJIT_HEADER #define CALL_SIMPLE_METHOD() do { \ rb_snum_t x = leaf ? INSN_ATTR(width) : 0; \ - rb_snum_t y = attr_width_opt_send_without_block(0, 0); \ + rb_snum_t y = attr_width_opt_send_without_block(0); \ rb_snum_t z = x - y; \ ADD_PC(z); \ DISPATCH_ORIGINAL_INSN(opt_send_without_block); \