Implement object shapes for T_CLASS and T_MODULE (#6637)

* Avoid RCLASS_IV_TBL in marshal.c
* Avoid RCLASS_IV_TBL for class names
* Avoid RCLASS_IV_TBL for autoload
* Avoid RCLASS_IV_TBL for class variables
* Avoid copying RCLASS_IV_TBL onto ICLASSes
* Use object shapes for Class and Module IVs

class.c

@@ -213,7 +213,6 @@ class_alloc(VALUE flags, VALUE klass)
 #endif
 
     /* ZALLOC
-      RCLASS_IV_TBL(obj) = 0;
       RCLASS_CONST_TBL(obj) = 0;
       RCLASS_M_TBL(obj) = 0;
       RCLASS_IV_INDEX_TBL(obj) = 0;
@@ -402,23 +401,19 @@ class_init_copy_check(VALUE clone, VALUE orig)
 static void
 copy_tables(VALUE clone, VALUE orig)
 {
-    if (RCLASS_IV_TBL(clone)) {
-        st_free_table(RCLASS_IV_TBL(clone));
-        RCLASS_IV_TBL(clone) = 0;
-    }
     if (RCLASS_CONST_TBL(clone)) {
         rb_free_const_table(RCLASS_CONST_TBL(clone));
         RCLASS_CONST_TBL(clone) = 0;
     }
     RCLASS_M_TBL(clone) = 0;
-    if (RCLASS_IV_TBL(orig)) {
+    if (!RB_TYPE_P(clone, T_ICLASS)) {
         st_data_t id;
 
         rb_iv_tbl_copy(clone, orig);
         CONST_ID(id, "__tmp_classpath__");
-        st_delete(RCLASS_IV_TBL(clone), &id, 0);
+        rb_attr_delete(clone, id);
         CONST_ID(id, "__classpath__");
-        st_delete(RCLASS_IV_TBL(clone), &id, 0);
+        rb_attr_delete(clone, id);
     }
     if (RCLASS_CONST_TBL(orig)) {
         struct clone_const_arg arg;
@@ -520,7 +515,6 @@ rb_mod_init_copy(VALUE clone, VALUE orig)
             prev_clone_p = clone_p;
             RCLASS_M_TBL(clone_p) = RCLASS_M_TBL(p);
             RCLASS_CONST_TBL(clone_p) = RCLASS_CONST_TBL(p);
-            RCLASS_IV_TBL(clone_p) = RCLASS_IV_TBL(p);
             RCLASS_ALLOCATOR(clone_p) = RCLASS_ALLOCATOR(p);
             if (RB_TYPE_P(clone, T_CLASS)) {
                 RCLASS_SET_INCLUDER(clone_p, clone);
@@ -607,9 +601,7 @@ rb_singleton_class_clone_and_attach(VALUE obj, VALUE attach)
 
         RCLASS_SET_SUPER(clone, RCLASS_SUPER(klass));
         RCLASS_ALLOCATOR(clone) = RCLASS_ALLOCATOR(klass);
-        if (RCLASS_IV_TBL(klass)) {
+        rb_iv_tbl_copy(clone, klass);
-            rb_iv_tbl_copy(clone, klass);
-        }
         if (RCLASS_CONST_TBL(klass)) {
             struct clone_const_arg arg;
             arg.tbl = RCLASS_CONST_TBL(clone) = rb_id_table_create(0);
@@ -1062,13 +1054,10 @@ rb_include_class_new(VALUE module, VALUE super)
         module = METACLASS_OF(module);
     }
     RUBY_ASSERT(!RB_TYPE_P(module, T_ICLASS));
-    if (!RCLASS_IV_TBL(module)) {
-        RCLASS_IV_TBL(module) = st_init_numtable();
-    }
     if (!RCLASS_CONST_TBL(module)) {
         RCLASS_CONST_TBL(module) = rb_id_table_create(0);
     }
-    RCLASS_IV_TBL(klass) = RCLASS_IV_TBL(module);
+
     RCLASS_CVC_TBL(klass) = RCLASS_CVC_TBL(module);
     RCLASS_CONST_TBL(klass) = RCLASS_CONST_TBL(module);
 

gc.c

@@ -3447,8 +3447,8 @@ obj_free(rb_objspace_t *objspace, VALUE obj)
       case T_CLASS:
         rb_id_table_free(RCLASS_M_TBL(obj));
         cc_table_free(objspace, obj, FALSE);
-        if (RCLASS_IV_TBL(obj)) {
+        if (RCLASS_IVPTR(obj)) {
-            st_free_table(RCLASS_IV_TBL(obj));
+            xfree(RCLASS_IVPTR(obj));
         }
         if (RCLASS_CONST_TBL(obj)) {
             rb_free_const_table(RCLASS_CONST_TBL(obj));
@@ -4865,15 +4865,11 @@ obj_memsize_of(VALUE obj, int use_all_types)
             if (RCLASS_M_TBL(obj)) {
                 size += rb_id_table_memsize(RCLASS_M_TBL(obj));
             }
-            if (RCLASS_IV_TBL(obj)) {
+            // class IV sizes are allocated as powers of two
-                size += st_memsize(RCLASS_IV_TBL(obj));
+            size += SIZEOF_VALUE << bit_length(RCLASS_IV_COUNT(obj));
-            }
             if (RCLASS_CVC_TBL(obj)) {
                 size += rb_id_table_memsize(RCLASS_CVC_TBL(obj));
             }
-            if (RCLASS_EXT(obj)->iv_tbl) {
-                size += st_memsize(RCLASS_EXT(obj)->iv_tbl);
-            }
             if (RCLASS_EXT(obj)->const_tbl) {
                 size += rb_id_table_memsize(RCLASS_EXT(obj)->const_tbl);
             }
@@ -7212,7 +7208,9 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj)
 
         mark_m_tbl(objspace, RCLASS_M_TBL(obj));
         cc_table_mark(objspace, obj);
-        mark_tbl_no_pin(objspace, RCLASS_IV_TBL(obj));
+        for (attr_index_t i = 0; i < RCLASS_IV_COUNT(obj); i++) {
+            gc_mark(objspace, RCLASS_IVPTR(obj)[i]);
+        }
         mark_const_tbl(objspace, RCLASS_CONST_TBL(obj));
         break;
 
@@ -10439,7 +10437,9 @@ gc_update_object_references(rb_objspace_t *objspace, VALUE obj)
         update_cvc_tbl(objspace, obj);
         update_superclasses(objspace, obj);
 
-        gc_update_tbl_refs(objspace, RCLASS_IV_TBL(obj));
+        for (attr_index_t i = 0; i < RCLASS_IV_COUNT(obj); i++) {
+            UPDATE_IF_MOVED(objspace, RCLASS_IVPTR(obj)[i]);
+        }
 
         update_class_ext(objspace, RCLASS_EXT(obj));
         update_const_tbl(objspace, RCLASS_CONST_TBL(obj));
@@ -10454,9 +10454,6 @@ gc_update_object_references(rb_objspace_t *objspace, VALUE obj)
             UPDATE_IF_MOVED(objspace, RCLASS(obj)->super);
         }
         if (!RCLASS_EXT(obj)) break;
-        if (RCLASS_IV_TBL(obj)) {
-            gc_update_tbl_refs(objspace, RCLASS_IV_TBL(obj));
-        }
         update_class_ext(objspace, RCLASS_EXT(obj));
         update_m_tbl(objspace, RCLASS_CALLABLE_M_TBL(obj));
         update_cc_tbl(objspace, obj);

internal/class.h

@@ -33,7 +33,7 @@ struct rb_cvar_class_tbl_entry {
 };
 
 struct rb_classext_struct {
-    struct st_table *iv_tbl;
+    VALUE *iv_ptr;
     struct rb_id_table *const_tbl;
     struct rb_id_table *callable_m_tbl;
     struct rb_id_table *cc_tbl; /* ID -> [[ci, cc1], cc2, ...] */
@@ -75,9 +75,9 @@ typedef struct rb_classext_struct rb_classext_t;
 #else
 #  define RCLASS_EXT(c) (RCLASS(c)->ptr)
 #endif
-#define RCLASS_IV_TBL(c) (RCLASS_EXT(c)->iv_tbl)
 #define RCLASS_CONST_TBL(c) (RCLASS_EXT(c)->const_tbl)
 #define RCLASS_M_TBL(c) (RCLASS(c)->m_tbl)
+#define RCLASS_IVPTR(c) (RCLASS_EXT(c)->iv_ptr)
 #define RCLASS_CALLABLE_M_TBL(c) (RCLASS_EXT(c)->callable_m_tbl)
 #define RCLASS_CC_TBL(c) (RCLASS_EXT(c)->cc_tbl)
 #define RCLASS_CVC_TBL(c) (RCLASS_EXT(c)->cvc_tbl)

internal/variable.h

@@ -24,7 +24,6 @@ void rb_gc_update_global_tbl(void);
 size_t rb_generic_ivar_memsize(VALUE);
 VALUE rb_search_class_path(VALUE);
 VALUE rb_attr_delete(VALUE, ID);
-VALUE rb_ivar_lookup(VALUE obj, ID id, VALUE undef);
 void rb_autoload_str(VALUE mod, ID id, VALUE file);
 VALUE rb_autoload_at_p(VALUE, ID, int);
 NORETURN(VALUE rb_mod_const_missing(VALUE,VALUE));
@@ -49,6 +48,7 @@ void rb_iv_tbl_copy(VALUE dst, VALUE src);
 RUBY_SYMBOL_EXPORT_END
 
 MJIT_SYMBOL_EXPORT_BEGIN
+VALUE rb_ivar_lookup(VALUE obj, ID id, VALUE undef);
 VALUE rb_gvar_get(ID);
 VALUE rb_gvar_set(ID, VALUE);
 VALUE rb_gvar_defined(ID);

marshal.c

@@ -523,7 +523,7 @@ hash_each(VALUE key, VALUE value, VALUE v)
 
 #define SINGLETON_DUMP_UNABLE_P(klass) \
     (rb_id_table_size(RCLASS_M_TBL(klass)) > 0 || \
-     (RCLASS_IV_TBL(klass) && RCLASS_IV_TBL(klass)->num_entries > 1))
+     rb_ivar_count(klass) > 1)
 
 static void
 w_extended(VALUE klass, struct dump_arg *arg, int check)

object.c

@@ -298,20 +298,22 @@ init_copy(VALUE dest, VALUE obj)
     rb_copy_generic_ivar(dest, obj);
     rb_gc_copy_finalizer(dest, obj);
 
-    rb_shape_t *shape_to_set = rb_shape_get_shape(obj);
+    if (!RB_TYPE_P(obj, T_CLASS) && !RB_TYPE_P(obj, T_MODULE)) {
+        rb_shape_t *shape_to_set = rb_shape_get_shape(obj);
 
-    // If the object is frozen, the "dup"'d object will *not* be frozen,
+        // If the object is frozen, the "dup"'d object will *not* be frozen,
-    // so we need to copy the frozen shape's parent to the new object.
+        // so we need to copy the frozen shape's parent to the new object.
-    if (rb_shape_frozen_shape_p(shape_to_set)) {
+        if (rb_shape_frozen_shape_p(shape_to_set)) {
-        shape_to_set = rb_shape_get_shape_by_id(shape_to_set->parent_id);
+            shape_to_set = rb_shape_get_shape_by_id(shape_to_set->parent_id);
+        }
+
+        // shape ids are different
+        rb_shape_set_shape(dest, shape_to_set);
     }
 
     if (RB_TYPE_P(obj, T_OBJECT)) {
         rb_obj_copy_ivar(dest, obj);
     }
-
-    // shape ids are different
-    rb_shape_set_shape(dest, shape_to_set);
 }
 
 static VALUE immutable_obj_clone(VALUE obj, VALUE kwfreeze);

shape.c

@@ -54,9 +54,10 @@ rb_shape_get_shape_by_id_without_assertion(shape_id_t shape_id)
 }
 
 #if !SHAPE_IN_BASIC_FLAGS
-static inline shape_id_t
+shape_id_t
-RCLASS_SHAPE_ID(VALUE obj)
+rb_rclass_shape_id(VALUE obj)
 {
+    RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE));
     return RCLASS_EXT(obj)->shape_id;
 }
 
@@ -115,7 +116,9 @@ static rb_shape_t*
 get_next_shape_internal(rb_shape_t* shape, ID id, VALUE obj, enum shape_type shape_type)
 {
     rb_shape_t *res = NULL;
-    RUBY_ASSERT(SHAPE_FROZEN != (enum shape_type)shape->type);
+
+    RUBY_ASSERT(SHAPE_FROZEN != (enum shape_type)shape->type || RB_TYPE_P(obj, T_MODULE) || RB_TYPE_P(obj, T_CLASS));
+
     RB_VM_LOCK_ENTER();
     {
         if (rb_shape_lookup_id(shape, id, shape_type)) {

shape.h

@@ -90,6 +90,13 @@ ROBJECT_SET_SHAPE_ID(VALUE obj, shape_id_t shape_id)
     RBASIC_SET_SHAPE_ID(obj, shape_id);
 }
 
+static inline shape_id_t
+RCLASS_SHAPE_ID(VALUE obj)
+{
+    RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE));
+    return RBASIC_SHAPE_ID(obj);
+}
+
 #else
 
 static inline shape_id_t
@@ -105,6 +112,15 @@ ROBJECT_SET_SHAPE_ID(VALUE obj, shape_id_t shape_id)
     RBASIC(obj)->flags &= SHAPE_FLAG_MASK;
     RBASIC(obj)->flags |= ((VALUE)(shape_id) << SHAPE_FLAG_SHIFT);
 }
+
+MJIT_SYMBOL_EXPORT_BEGIN
+shape_id_t rb_rclass_shape_id(VALUE obj);
+MJIT_SYMBOL_EXPORT_END
+
+static inline shape_id_t RCLASS_SHAPE_ID(VALUE obj) {
+    return rb_rclass_shape_id(obj);
+}
+
 #endif
 
 bool rb_shape_root_shape_p(rb_shape_t* shape);
@@ -134,6 +150,14 @@ ROBJECT_IV_COUNT(VALUE obj)
     return ivc;
 }
 
+static inline uint32_t
+RCLASS_IV_COUNT(VALUE obj)
+{
+    RUBY_ASSERT(RB_TYPE_P(obj, RUBY_T_CLASS) || RB_TYPE_P(obj, RUBY_T_MODULE));
+    uint32_t ivc = rb_shape_get_shape_by_id(RCLASS_SHAPE_ID(obj))->next_iv_index;
+    return ivc;
+}
+
 rb_shape_t * rb_shape_alloc(ID edge_name, rb_shape_t * parent);
 rb_shape_t * rb_shape_alloc_with_parent_id(ID edge_name, shape_id_t parent_id);
 

variable.c

@@ -111,18 +111,16 @@ rb_namespace_p(VALUE obj)
 static VALUE
 classname(VALUE klass, int *permanent)
 {
-    st_table *ivtbl;
-    st_data_t n;
-
     *permanent = 0;
     if (!RCLASS_EXT(klass)) return Qnil;
-    if (!(ivtbl = RCLASS_IV_TBL(klass))) return Qnil;
+
-    if (st_lookup(ivtbl, (st_data_t)classpath, &n)) {
+    VALUE classpathv = rb_ivar_lookup(klass, classpath, Qnil);
+    if (RTEST(classpathv)) {
         *permanent = 1;
-        return (VALUE)n;
+        return classpathv;
     }
-    if (st_lookup(ivtbl, (st_data_t)tmp_classpath, &n)) return (VALUE)n;
+
-    return Qnil;
+    return rb_ivar_lookup(klass, tmp_classpath, Qnil);;
 }
 
 /*
@@ -946,6 +944,8 @@ gen_ivtbl_get_unlocked(VALUE obj, ID id, struct gen_ivtbl **ivtbl)
 MJIT_FUNC_EXPORTED int
 rb_gen_ivtbl_get(VALUE obj, ID id, struct gen_ivtbl **ivtbl)
 {
+    RUBY_ASSERT(!RB_TYPE_P(obj, T_ICLASS));
+
     st_data_t data;
     int r = 0;
 
@@ -1116,54 +1116,6 @@ gen_ivtbl_count(const struct gen_ivtbl *ivtbl)
     return n;
 }
 
-static int
-lock_st_lookup(st_table *tab, st_data_t key, st_data_t *value)
-{
-    int r;
-    RB_VM_LOCK_ENTER();
-    {
-        r = st_lookup(tab, key, value);
-    }
-    RB_VM_LOCK_LEAVE();
-    return r;
-}
-
-static int
-lock_st_delete(st_table *tab, st_data_t *key, st_data_t *value)
-{
-    int r;
-    RB_VM_LOCK_ENTER();
-    {
-        r = st_delete(tab, key, value);
-    }
-    RB_VM_LOCK_LEAVE();
-    return r;
-}
-
-static int
-lock_st_is_member(st_table *tab, st_data_t key)
-{
-    int r;
-    RB_VM_LOCK_ENTER();
-    {
-        r = st_is_member(tab, key);
-    }
-    RB_VM_LOCK_LEAVE();
-    return r;
-}
-
-static int
-lock_st_insert(st_table *tab, st_data_t key, st_data_t value)
-{
-    int r;
-    RB_VM_LOCK_ENTER();
-    {
-        r = st_insert(tab, key, value);
-    }
-    RB_VM_LOCK_LEAVE();
-    return r;
-}
-
 VALUE
 rb_ivar_lookup(VALUE obj, ID id, VALUE undef)
 {
@@ -1181,21 +1133,39 @@ rb_ivar_lookup(VALUE obj, ID id, VALUE undef)
       case T_CLASS:
       case T_MODULE:
         {
-            st_data_t val;
+            bool found;
+            VALUE val;
 
-            if (RCLASS_IV_TBL(obj) &&
+            RB_VM_LOCK_ENTER();
-                lock_st_lookup(RCLASS_IV_TBL(obj), (st_data_t)id, &val)) {
+            {
-                if (rb_is_instance_id(id) &&
+#if !SHAPE_IN_BASIC_FLAGS
+                shape_id = RCLASS_SHAPE_ID(obj);
+#endif
+
+                attr_index_t index = 0;
+                shape = rb_shape_get_shape_by_id(shape_id);
+                found = rb_shape_get_iv_index(shape, id, &index);
+
+                if (found) {
+                    ivar_list = RCLASS_IVPTR(obj);
+                    RUBY_ASSERT(ivar_list);
+
+                    val = ivar_list[index];
+                }
+                else {
+                    val = undef;
+                }
+            }
+            RB_VM_LOCK_LEAVE();
+
+            if (found &&
+                    rb_is_instance_id(id) &&
                     UNLIKELY(!rb_ractor_main_p()) &&
                     !rb_ractor_shareable_p(val)) {
-                    rb_raise(rb_eRactorIsolationError,
+                rb_raise(rb_eRactorIsolationError,
-                             "can not get unshareable values from instance variables of classes/modules from non-main Ractors");
+                        "can not get unshareable values from instance variables of classes/modules from non-main Ractors");
-                }
-                return val;
-            }
-            else {
-                return undef;
             }
+            return val;
         }
       case T_OBJECT:
         {
@@ -1247,19 +1217,25 @@ rb_ivar_delete(VALUE obj, ID id, VALUE undef)
 {
     rb_check_frozen(obj);
 
-    VALUE val = Qnil;
+    VALUE val = undef;
     attr_index_t index;
 
     switch (BUILTIN_TYPE(obj)) {
       case T_CLASS:
       case T_MODULE:
         IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id);
-        if (RCLASS_IV_TBL(obj)) {
+
-            st_data_t id_data = (st_data_t)id, val;
+        RB_VM_LOCK_ENTER();
-            if (lock_st_delete(RCLASS_IV_TBL(obj), &id_data, &val)) {
+        {
-                return (VALUE)val;
+            rb_shape_t * shape = rb_shape_get_shape(obj);
+            if (rb_shape_get_iv_index(shape, id, &index)) {
+                rb_shape_transition_shape_remove_ivar(obj, id, shape);
+                val = RCLASS_IVPTR(obj)[index];
+                RCLASS_IVPTR(obj)[index] = Qundef;
             }
         }
+        RB_VM_LOCK_LEAVE();
+
         break;
       case T_OBJECT: {
         rb_shape_t * shape = rb_shape_get_shape(obj);
@@ -1267,7 +1243,6 @@ rb_ivar_delete(VALUE obj, ID id, VALUE undef)
             rb_shape_transition_shape_remove_ivar(obj, id, shape);
             val = ROBJECT_IVPTR(obj)[index];
             ROBJECT_IVPTR(obj)[index] = Qundef;
-            return val;
         }
 
         break;
@@ -1281,14 +1256,13 @@ rb_ivar_delete(VALUE obj, ID id, VALUE undef)
             rb_gen_ivtbl_get(obj, id, &ivtbl);
             val = ivtbl->ivptr[index];
             ivtbl->ivptr[index] = Qundef;
-            return val;
         }
 
         break;
       }
     }
 
-    return undef;
+    return val;
 }
 
 VALUE
@@ -1553,10 +1527,9 @@ ivar_set(VALUE obj, ID id, VALUE val)
       }
       case T_CLASS:
       case T_MODULE:
-        // TODO: Transition shapes on classes
-        //rb_shape_transition_shape(obj, id, rb_shape_get_shape_by_id(RCLASS_SHAPE_ID(obj)));
         IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id);
         rb_class_ivar_set(obj, id, val);
+
         break;
       default:
         generic_ivar_set(obj, id, val);
@@ -1587,13 +1560,7 @@ rb_ivar_defined(VALUE obj, ID id)
     attr_index_t index;
 
     if (SPECIAL_CONST_P(obj)) return Qfalse;
-    switch (BUILTIN_TYPE(obj)) {
+    return RBOOL(rb_shape_get_iv_index(rb_shape_get_shape(obj), id, &index));
-      case T_CLASS:
-      case T_MODULE:
-        return RBOOL(RCLASS_IV_TBL(obj) && lock_st_is_member(RCLASS_IV_TBL(obj), (st_data_t)id));
-      default:
-        return RBOOL(rb_shape_get_iv_index(rb_shape_get_shape(obj), id, &index));
-    }
 }
 
 typedef int rb_ivar_foreach_callback_func(ID key, VALUE val, st_data_t arg);
@@ -1636,6 +1603,15 @@ gen_ivar_each(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg)
     iterate_over_shapes_with_callback(shape, ivtbl->ivptr, func, arg);
 }
 
+static void
+class_ivar_each(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg)
+{
+    RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE));
+
+    rb_shape_t* shape = rb_shape_get_shape(obj);
+    iterate_over_shapes_with_callback(shape, RCLASS_IVPTR(obj), func, arg);
+}
+
 void
 rb_copy_generic_ivar(VALUE clone, VALUE obj)
 {
@@ -1720,13 +1696,11 @@ rb_ivar_foreach(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg)
       case T_CLASS:
       case T_MODULE:
         IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(0);
-        if (RCLASS_IV_TBL(obj)) {
+        RB_VM_LOCK_ENTER();
-            RB_VM_LOCK_ENTER();
+        {
-            {
+            class_ivar_each(obj, func, arg);
-                st_foreach_safe(RCLASS_IV_TBL(obj), func, arg);
-            }
-            RB_VM_LOCK_LEAVE();
         }
+        RB_VM_LOCK_LEAVE();
         break;
       default:
         if (FL_TEST(obj, FL_EXIVAR)) {
@@ -1739,8 +1713,6 @@ rb_ivar_foreach(VALUE obj, rb_ivar_foreach_callback_func *func, st_data_t arg)
 st_index_t
 rb_ivar_count(VALUE obj)
 {
-    st_table *tbl;
-
     if (SPECIAL_CONST_P(obj)) return 0;
 
     switch (BUILTIN_TYPE(obj)) {
@@ -1758,8 +1730,22 @@ rb_ivar_count(VALUE obj)
         break;
       case T_CLASS:
       case T_MODULE:
-        if ((tbl = RCLASS_IV_TBL(obj)) != 0) {
+        if (rb_shape_get_shape(obj)->next_iv_index > 0) {
-            return tbl->num_entries;
+            st_index_t count = 0;
+
+            RB_VM_LOCK_ENTER();
+            {
+                st_index_t i, num = rb_shape_get_shape(obj)->next_iv_index;
+                const VALUE *const ivptr = RCLASS_IVPTR(obj);
+                for (i = count = 0; i < num; ++i) {
+                    if (ivptr[i] != Qundef) {
+                        count++;
+                    }
+                }
+            }
+            RB_VM_LOCK_LEAVE();
+
+            return count;
         }
         break;
       default:
@@ -1879,11 +1865,12 @@ rb_obj_remove_instance_variable(VALUE obj, VALUE name)
       case T_CLASS:
       case T_MODULE:
         IVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR(id);
-        if (RCLASS_IV_TBL(obj)) {
+        rb_shape_t * shape = rb_shape_get_shape(obj);
-            st_data_t id_data = (st_data_t)id, val;
+        if (rb_shape_get_iv_index(shape, id, &index)) {
-            if (lock_st_delete(RCLASS_IV_TBL(obj), &id_data, &val)) {
+            rb_shape_transition_shape_remove_ivar(obj, id, shape);
-                return (VALUE)val;
+            val = RCLASS_IVPTR(obj)[index];
-            }
+            RCLASS_IVPTR(obj)[index] = Qundef;
+            return val;
         }
         break;
       case T_OBJECT: {
@@ -2030,8 +2017,22 @@ autoload_data(VALUE mod, ID id)
     struct st_table *tbl;
     st_data_t val;
 
+    // If we are called with a non-origin ICLASS, fetch the autoload data from
+    // the original module.
+    if (RB_TYPE_P(mod, T_ICLASS)) {
+        if (FL_TEST_RAW(mod, RICLASS_IS_ORIGIN)) {
+            return 0;
+        } else {
+            mod = RBASIC(mod)->klass;
+        }
+    }
+
+    RUBY_ASSERT(RB_TYPE_P(mod, T_CLASS) || RB_TYPE_P(mod, T_MODULE));
+
     // Look up the instance variable table for `autoload`, then index into that table with the given constant name `id`.
-    if (!st_lookup(RCLASS_IV_TBL(mod), autoload, &val) || !(tbl = check_autoload_table((VALUE)val)) || !st_lookup(tbl, (st_data_t)id, &val)) {
+
+    VALUE tbl_value = rb_ivar_lookup(mod, autoload, 0);
+    if (!tbl_value || !(tbl = check_autoload_table(tbl_value)) || !st_lookup(tbl, (st_data_t)id, &val)) {
         return 0;
     }
 
@@ -2229,23 +2230,14 @@ autoload_feature_lookup_or_create(VALUE feature, struct autoload_data **autoload
 static struct st_table *
 autoload_table_lookup_or_create(VALUE module)
 {
-    // Get or create an autoload table in the class instance variables:
+    VALUE autoload_table_value = rb_ivar_lookup(module, autoload, 0);
-    struct st_table *table = RCLASS_IV_TBL(module);
+    if (autoload_table_value) {
-    VALUE autoload_table_value;
+        return check_autoload_table(autoload_table_value);
-
+    } else {
-    if (table && st_lookup(table, (st_data_t)autoload, &autoload_table_value)) {
+        autoload_table_value = TypedData_Wrap_Struct(0, &autoload_table_type, 0);
-        return check_autoload_table((VALUE)autoload_table_value);
+        rb_class_ivar_set(module, autoload, autoload_table_value);
+        return (DATA_PTR(autoload_table_value) = st_init_numtable());
     }
-
-    if (!table) {
-        table = RCLASS_IV_TBL(module) = st_init_numtable();
-    }
-
-    autoload_table_value = TypedData_Wrap_Struct(0, &autoload_table_type, 0);
-    st_add_direct(table, (st_data_t)autoload, (st_data_t)autoload_table_value);
-
-    RB_OBJ_WRITTEN(module, Qnil, autoload_table_value);
-    return (DATA_PTR(autoload_table_value) = st_init_numtable());
 }
 
 static VALUE
@@ -2314,10 +2306,13 @@ autoload_delete(VALUE module, ID name)
 {
     RUBY_ASSERT_CRITICAL_SECTION_ENTER();
 
-    st_data_t value, load = 0, key = name;
+    st_data_t load = 0, key = name;
 
-    if (st_lookup(RCLASS_IV_TBL(module), (st_data_t)autoload, &value)) {
+    RUBY_ASSERT(RB_TYPE_P(module, T_CLASS) || RB_TYPE_P(module, T_MODULE));
-        struct st_table *table = check_autoload_table((VALUE)value);
+
+    VALUE table_value = rb_ivar_lookup(module, autoload, 0);
+    if (table_value) {
+        struct st_table *table = check_autoload_table(table_value);
 
         st_delete(table, &key, &load);
 
@@ -2342,8 +2337,7 @@ autoload_delete(VALUE module, ID name)
 
             // If the autoload table is empty, we can delete it.
             if (table->num_entries == 0) {
-                name = autoload;
+                rb_attr_delete(module, autoload);
-                st_delete(RCLASS_IV_TBL(module), &name, &value);
             }
         }
     }
@@ -3129,10 +3123,7 @@ set_namespace_path_i(ID id, VALUE v, void *payload)
         return ID_TABLE_CONTINUE;
     }
     set_namespace_path(value, build_const_path(parental_path, id));
-    if (RCLASS_IV_TBL(value)) {
+    rb_attr_delete(value, tmp_classpath);
-        st_data_t tmp = tmp_classpath;
-        st_delete(RCLASS_IV_TBL(value), &tmp, 0);
-    }
 
     return ID_TABLE_CONTINUE;
 }
@@ -3469,8 +3460,20 @@ original_module(VALUE c)
 static int
 cvar_lookup_at(VALUE klass, ID id, st_data_t *v)
 {
-    if (!RCLASS_IV_TBL(klass)) return 0;
+    if (RB_TYPE_P(klass, T_ICLASS)) {
-    return st_lookup(RCLASS_IV_TBL(klass), (st_data_t)id, v);
+        if (FL_TEST_RAW(klass, RICLASS_IS_ORIGIN)) {
+            return 0;
+        } else {
+            // check the original module
+            klass = RBASIC(klass)->klass;
+        }
+    }
+
+    VALUE n = rb_ivar_lookup(klass, id, Qundef);
+    if (n == Qundef) return 0;
+
+    if (v) *v = n;
+    return 1;
 }
 
 static VALUE
@@ -3489,8 +3492,6 @@ static void
 cvar_overtaken(VALUE front, VALUE target, ID id)
 {
     if (front && target != front) {
-        st_data_t did = (st_data_t)id;
-
         if (original_module(front) != original_module(target)) {
             rb_raise(rb_eRuntimeError,
                      "class variable % "PRIsVALUE" of %"PRIsVALUE" is overtaken by %"PRIsVALUE"",
@@ -3498,7 +3499,7 @@ cvar_overtaken(VALUE front, VALUE target, ID id)
                        rb_class_name(original_module(target)));
         }
         if (BUILTIN_TYPE(front) == T_CLASS) {
-            st_delete(RCLASS_IV_TBL(front), &did, 0);
+            rb_ivar_delete(front, id, Qundef);
         }
     }
 }
@@ -3543,9 +3544,8 @@ find_cvar(VALUE klass, VALUE * front, VALUE * target, ID id)
 static void
 check_for_cvar_table(VALUE subclass, VALUE key)
 {
-    st_table *tbl = RCLASS_IV_TBL(subclass);
+    // Must not check ivar on ICLASS
-
+    if (!RB_TYPE_P(subclass, T_ICLASS) && RTEST(rb_ivar_defined(subclass, key))) {
-    if (tbl && st_lookup(tbl, key, NULL)) {
         RB_DEBUG_COUNTER_INC(cvar_class_invalidate);
         ruby_vm_global_cvar_state++;
         return;
@@ -3688,9 +3688,9 @@ mod_cvar_at(VALUE mod, void *data)
     if (!tbl) {
         tbl = st_init_numtable();
     }
-    if (RCLASS_IV_TBL(mod)) {
+    mod = original_module(mod);
-        st_foreach_safe(RCLASS_IV_TBL(mod), cv_i, (st_data_t)tbl);
+
-    }
+    rb_ivar_foreach(mod, cv_i, (st_data_t)tbl);
     return tbl;
 }
 
@@ -3793,13 +3793,14 @@ VALUE
 rb_mod_remove_cvar(VALUE mod, VALUE name)
 {
     const ID id = id_for_var_message(mod, name, class, "wrong class variable name %1$s");
-    st_data_t val, n = id;
+    st_data_t val;
 
     if (!id) {
         goto not_defined;
     }
     rb_check_frozen(mod);
-    if (RCLASS_IV_TBL(mod) && st_delete(RCLASS_IV_TBL(mod), &n, &val)) {
+    val = rb_ivar_delete(mod, id, Qundef);
+    if (val != Qundef) {
         return (VALUE)val;
     }
     if (rb_cvar_defined(mod, id)) {
@@ -3834,30 +3835,63 @@ rb_iv_set(VALUE obj, const char *name, VALUE val)
 int
 rb_class_ivar_set(VALUE obj, ID key, VALUE value)
 {
-    if (!RCLASS_IV_TBL(obj)) {
+    RUBY_ASSERT(RB_TYPE_P(obj, T_CLASS) || RB_TYPE_P(obj, T_MODULE));
-        RCLASS_IV_TBL(obj) = st_init_numtable();
+    int found;
-    }
 
-    st_table *tbl = RCLASS_IV_TBL(obj);
+    RB_VM_LOCK_ENTER();
-    int result = lock_st_insert(tbl, (st_data_t)key, (st_data_t)value);
+    {
-    RB_OBJ_WRITTEN(obj, Qundef, value);
+        rb_shape_t * shape = rb_shape_get_shape(obj);
-    return result;
+        attr_index_t idx;
+        found = rb_shape_get_iv_index(shape, key, &idx);
+
+        if (found) {
+            // Changing an existing instance variable
+            RUBY_ASSERT(RCLASS_IVPTR(obj));
+
+            RCLASS_IVPTR(obj)[idx] = value;
+            RB_OBJ_WRITTEN(obj, Qundef, value);
+        } else {
+            // Creating and setting a new instance variable
+
+            // Move to a shape which fits the new ivar
+            idx = shape->next_iv_index;
+            shape = rb_shape_get_next(shape, obj, key);
+
+            // We always allocate a power of two sized IV array. This way we
+            // only need to realloc when we expand into a new power of two size
+            if ((idx & (idx - 1)) == 0) {
+                size_t newsize = idx ? idx * 2 : 1;
+                REALLOC_N(RCLASS_IVPTR(obj), VALUE, newsize);
+            }
+
+            RUBY_ASSERT(RCLASS_IVPTR(obj));
+
+            RB_OBJ_WRITE(obj, &RCLASS_IVPTR(obj)[idx], value);
+            rb_shape_set_shape(obj, shape);
+        }
+    }
+    RB_VM_LOCK_LEAVE();
+
+    return found;
 }
 
 static int
-tbl_copy_i(st_data_t key, st_data_t value, st_data_t data)
+tbl_copy_i(st_data_t key, st_data_t val, st_data_t dest) {
-{
+    rb_class_ivar_set(dest, key, val);
-    RB_OBJ_WRITTEN((VALUE)data, Qundef, (VALUE)value);
+
     return ST_CONTINUE;
 }
 
 void
 rb_iv_tbl_copy(VALUE dst, VALUE src)
 {
-    st_table *orig_tbl = RCLASS_IV_TBL(src);
+    RUBY_ASSERT(rb_type(dst) == rb_type(src));
-    st_table *new_tbl = st_copy(orig_tbl);
+    RUBY_ASSERT(RB_TYPE_P(dst, T_CLASS) || RB_TYPE_P(dst, T_MODULE));
-    st_foreach(new_tbl, tbl_copy_i, (st_data_t)dst);
+
-    RCLASS_IV_TBL(dst) = new_tbl;
+    RUBY_ASSERT(RCLASS_SHAPE_ID(dst) == ROOT_SHAPE_ID);
+    RUBY_ASSERT(!RCLASS_IVPTR(dst));
+
+    rb_ivar_foreach(src, tbl_copy_i, dst);
 }
 
 MJIT_FUNC_EXPORTED rb_const_entry_t *

vm_insnhelper.c

@@ -1169,7 +1169,23 @@ vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_call
       case T_CLASS:
       case T_MODULE:
         {
-            goto general_path;
+            if (UNLIKELY(!rb_ractor_main_p())) {
+                // For two reasons we can only use the fast path on the main
+                // ractor.
+                // First, only the main ractor is allowed to set ivars on classes
+                // and modules. So we can skip locking.
+                // Second, other ractors need to check the shareability of the
+                // values returned from the class ivars.
+                goto general_path;
+            }
+
+            ivar_list = RCLASS_IVPTR(obj);
+
+#if !SHAPE_IN_BASIC_FLAGS
+            shape_id = RCLASS_SHAPE_ID(obj);
+#endif
+
+            break;
         }
       default:
         if (FL_TEST_RAW(obj, FL_EXIVAR)) {
@@ -1507,15 +1523,13 @@ vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID
 {
     const rb_cref_t *cref;
 
-    if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE()) {
+    if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && LIKELY(rb_ractor_main_p())) {
-        VALUE v = Qundef;
         RB_DEBUG_COUNTER_INC(cvar_read_inline_hit);
 
-        if (st_lookup(RCLASS_IV_TBL(ic->entry->class_value), (st_data_t)id, &v) &&
+        VALUE v = rb_ivar_lookup(ic->entry->class_value, id, Qundef);
-            LIKELY(rb_ractor_main_p())) {
+        RUBY_ASSERT(v != Qundef);
 
-            return v;
+        return v;
-        }
     }
 
     cref = vm_get_cref(GET_EP());