Inline getconstant on JIT (#3906)

* Inline getconstant on JIT

* Support USE_MJIT=0

insns.def

@@ -1023,7 +1023,7 @@ opt_getinlinecache
 ()
 (VALUE val)
 {
-    if (vm_ic_hit_p(ic, GET_EP())) {
+    if (vm_ic_hit_p(ic->ic_serial, ic->ic_cref, GET_EP())) {
 	val = ic->value;
 	JUMP(dst);
     }

mjit.c

@@ -82,6 +82,24 @@ mjit_gc_exit_hook(void)
     CRITICAL_SECTION_FINISH(4, "mjit_gc_exit_hook");
 }
 
+// Lock setinlinecache
+void
+rb_mjit_before_vm_ic_update(void)
+{
+    if (!mjit_enabled)
+        return;
+    CRITICAL_SECTION_START(3, "before vm_ic_update");
+}
+
+// Unlock setinlinecache
+void
+rb_mjit_after_vm_ic_update(void)
+{
+    if (!mjit_enabled)
+        return;
+    CRITICAL_SECTION_FINISH(3, "after vm_ic_update");
+}
+
 // Deal with ISeq movement from compactor
 void
 mjit_update_references(const rb_iseq_t *iseq)
@@ -378,6 +396,14 @@ rb_mjit_recompile_inlining(const rb_iseq_t *iseq)
     mjit_recompile(iseq);
 }
 
+// Recompile iseq, disabling getconstant inlining
+void
+rb_mjit_recompile_const(const rb_iseq_t *iseq)
+{
+    rb_mjit_iseq_compile_info(iseq->body)->disable_const_cache = true;
+    mjit_recompile(iseq);
+}
+
 extern VALUE ruby_archlibdir_path, ruby_prefix_path;
 
 // Initialize header_file, pch_file, libruby_pathflag. Return true on success.

mjit.h

@@ -70,6 +70,8 @@ struct rb_mjit_compile_info {
     bool disable_send_cache;
     // Disable method inlining
     bool disable_inlining;
+    // Disable opt_getinlinecache inlining
+    bool disable_const_cache;
 };
 
 typedef VALUE (*mjit_func_t)(rb_execution_context_t *, rb_control_frame_t *);
@@ -81,10 +83,13 @@ RUBY_EXTERN bool mjit_call_p;
 extern void rb_mjit_add_iseq_to_process(const rb_iseq_t *iseq);
 extern VALUE rb_mjit_wait_call(rb_execution_context_t *ec, struct rb_iseq_constant_body *body);
 extern struct rb_mjit_compile_info* rb_mjit_iseq_compile_info(const struct rb_iseq_constant_body *body);
+extern void rb_mjit_before_vm_ic_update(void);
+extern void rb_mjit_after_vm_ic_update(void);
 extern void rb_mjit_recompile_send(const rb_iseq_t *iseq);
 extern void rb_mjit_recompile_ivar(const rb_iseq_t *iseq);
 extern void rb_mjit_recompile_exivar(const rb_iseq_t *iseq);
 extern void rb_mjit_recompile_inlining(const rb_iseq_t *iseq);
+extern void rb_mjit_recompile_const(const rb_iseq_t *iseq);
 RUBY_SYMBOL_EXPORT_END
 
 extern bool mjit_compile(FILE *f, const rb_iseq_t *iseq, const char *funcname, int id);
@@ -189,6 +194,8 @@ void mjit_finish(bool close_handle_p);
 
 # else // USE_MJIT
 
+static inline void rb_mjit_before_vm_ic_update(void){}
+static inline void rb_mjit_after_vm_ic_update(void){}
 static inline struct mjit_cont *mjit_cont_new(rb_execution_context_t *ec){return NULL;}
 static inline void mjit_cont_free(struct mjit_cont *cont){}
 static inline void mjit_gc_start_hook(void){}

mjit_compile.c

@@ -307,6 +307,10 @@ compile_cancel_handler(FILE *f, const struct rb_iseq_constant_body *body, struct
     fprintf(f, "    rb_mjit_recompile_exivar(original_iseq);\n");
     fprintf(f, "    goto cancel;\n");
 
+    fprintf(f, "\nconst_cancel:\n");
+    fprintf(f, "    rb_mjit_recompile_const(original_iseq);\n");
+    fprintf(f, "    goto cancel;\n");
+
     fprintf(f, "\ncancel:\n");
     fprintf(f, "    RB_DEBUG_COUNTER_INC(mjit_cancel);\n");
     if (status->local_stack_p) {

tool/ruby_vm/views/_mjit_compile_getinlinecache.erb

@@ -0,0 +1,36 @@
+% # -*- C -*-
+% # Copyright (c) 2020 Takashi Kokubun.  All rights reserved.
+% #
+% # This file is a part of  the programming language Ruby.  Permission is hereby
+% # granted, to either  redistribute and/or modify this file,  provided that the
+% # conditions mentioned  in the  file COPYING  are met.   Consult the  file for
+% # details.
+%
+% # compiler: Declare dst and ic
+% insn.opes.each_with_index do |ope, i|
+    <%= ope.fetch(:decl) %> = (<%= ope.fetch(:type) %>)operands[<%= i %>];
+% end
+
+% # compiler: Capture IC values, locking getinlinecache
+    rb_mjit_before_vm_ic_update();
+    rb_serial_t ic_serial = ic->ic_serial;
+    const rb_cref_t *ic_cref = ic->ic_cref;
+    VALUE ic_value = ic->value;
+    rb_mjit_after_vm_ic_update();
+
+    if (ic_serial && !status->compile_info->disable_const_cache) {
+%       # JIT: Inline everything in IC, and cancel the slow path
+        fprintf(f, "    if (vm_ic_hit_p((rb_serial_t)%"PRI_SERIALT_PREFIX"u, (const rb_cref_t *)0x%"PRIxVALUE", reg_cfp->ep)) {", ic_serial, (VALUE)ic_cref);
+        fprintf(f, "        stack[%d] = 0x%"PRIxVALUE";\n", b->stack_size, ic_value);
+        fprintf(f, "        goto label_%d;\n", pos + insn_len(insn) + (int)dst);
+        fprintf(f, "    }");
+        fprintf(f, "    else {");
+        fprintf(f, "        reg_cfp->sp = vm_base_ptr(reg_cfp) + %d;\n", b->stack_size);
+        fprintf(f, "        reg_cfp->pc = original_body_iseq + %d;\n", pos);
+        fprintf(f, "        goto const_cancel;\n");
+        fprintf(f, "    }");
+
+%       # compiler: Move JIT compiler's internal stack pointer
+        b->stack_size += <%= insn.call_attribute('sp_inc') %>;
+        break;
+    }

tool/ruby_vm/views/mjit_compile.inc.erb

@@ -65,6 +65,8 @@ switch (insn) {
 <%=   render 'mjit_compile_ivar', locals: { insn: insn } -%>
 %   when 'invokebuiltin', 'opt_invokebuiltin_delegate'
 <%=   render 'mjit_compile_invokebuiltin', locals: { insn: insn } -%>
+%   when 'opt_getinlinecache'
+<%=   render 'mjit_compile_getinlinecache', locals: { insn: insn } -%>
 %   when 'leave', 'opt_invokebuiltin_delegate_leave'
 %     # opt_invokebuiltin_delegate_leave also implements leave insn. We need to handle it here for inlining.
 %     if insn.name == 'opt_invokebuiltin_delegate_leave'

vm_insnhelper.c

@@ -4577,12 +4577,11 @@ vm_opt_newarray_min(rb_num_t num, const VALUE *ptr)
 #undef id_cmp
 
 static int
-vm_ic_hit_p(IC ic, const VALUE *reg_ep)
+vm_ic_hit_p(const rb_serial_t ic_serial, const rb_cref_t *ic_cref, const VALUE *reg_ep)
 {
-    if (ic->ic_serial == GET_GLOBAL_CONSTANT_STATE() &&
+    if (ic_serial == GET_GLOBAL_CONSTANT_STATE() && rb_ractor_main_p()) {
-        rb_ractor_main_p()) {
+        return (ic_cref == NULL || // no need to check CREF
-        return (ic->ic_cref == NULL || // no need to check CREF
+                ic_cref == vm_get_cref(reg_ep));
-                ic->ic_cref == vm_get_cref(reg_ep));
     }
     return FALSE;
 }
@@ -4591,9 +4590,11 @@ static void
 vm_ic_update(IC ic, VALUE val, const VALUE *reg_ep)
 {
     VM_ASSERT(ic->value != Qundef);
+    rb_mjit_before_vm_ic_update();
     ic->value = val;
     ic->ic_serial = GET_GLOBAL_CONSTANT_STATE() - ruby_vm_const_missing_count;
     ic->ic_cref = vm_get_const_key_cref(reg_ep);
+    rb_mjit_after_vm_ic_update();
     ruby_vm_const_missing_count = 0;
 }