A64: Only clear icache when writing out new code (https://github.com/Shopify/ruby/pull/442)

Previously we cleared the cache for all the code in the system when we
flip memory protection, which was prohibitively expensive since the
operation is not constant time. Instead, only clear the cache for the
memory region of newly written code when we write out new code.

This brings the runtime for the 30k_if_else test down to about 6 seconds
from the previous 45 seconds on my laptop.

yjit.c

@@ -78,11 +78,17 @@ rb_yjit_mark_executable(void *mem_block, uint32_t mem_size)
         rb_bug("Couldn't make JIT page (%p, %lu bytes) executable, errno: %s\n",
             mem_block, (unsigned long)mem_size, strerror(errno));
     }
+}
 
+// `start` is inclusive and `end` is exclusive.
+void
+rb_yjit_icache_invalidate(void *start, void *end)
+{
     // Clear/invalidate the instruction cache. Compiles to nothing on x86_64
-    // but required on ARM. On Darwin it's the same as calling sys_icache_invalidate().
+    // but required on ARM before running freshly written code.
+    // On Darwin it's the same as calling sys_icache_invalidate().
 #ifdef __GNUC__
-    __builtin___clear_cache(mem_block, (char *)mem_block + mem_size);
+    __builtin___clear_cache(start, end);
 #endif
 }
 

yjit/bindgen/src/main.rs

@@ -273,6 +273,7 @@ fn main() {
         .allowlist_function("rb_RSTRING_LEN")
         .allowlist_function("rb_ENCODING_GET")
         .allowlist_function("rb_yjit_exit_locations_dict")
+        .allowlist_function("rb_yjit_icache_invalidate")
 
         // from vm_sync.h
         .allowlist_function("rb_vm_barrier")

yjit/src/backend/arm64/mod.rs

@@ -961,6 +961,16 @@ impl Assembler
             };
         }
 
+        // Invalidate icache for newly written out region so we don't run
+        // stale code.
+        {
+            let start = cb.get_ptr(start_write_pos).raw_ptr();
+            let write_ptr = cb.get_write_ptr().raw_ptr();
+            let codeblock_end = cb.get_ptr(cb.get_mem_size()).raw_ptr();
+            let end = std::cmp::min(write_ptr, codeblock_end);
+            unsafe { rb_yjit_icache_invalidate(start as _, end as _) };
+        }
+
         gc_offsets
     }
 

yjit/src/cruby_bindings.inc.rs

@@ -1020,6 +1020,12 @@ extern "C" {
 extern "C" {
     pub fn rb_yjit_mark_executable(mem_block: *mut ::std::os::raw::c_void, mem_size: u32);
 }
+extern "C" {
+    pub fn rb_yjit_icache_invalidate(
+        start: *mut ::std::os::raw::c_void,
+        end: *mut ::std::os::raw::c_void,
+    );
+}
 extern "C" {
     pub fn rb_yjit_exit_locations_dict(
         yjit_raw_samples: *mut VALUE,