kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity
ruby--ruby/gc.c
Aaron Patterson f0654b1027 More precisely iterate over Object instance variables 
Shapes provides us with an (almost) exact count of instance variables.
We only need to check for Qundef when an IV has been "undefined"
Prefer to use ROBJECT_IV_COUNT when iterating IVs
2022-10-15 10:44:10 -07:00

14476 lines
420 KiB
C
Raw Blame History

/**********************************************************************
gc.c -
$Author$
created at: Tue Oct 5 09:44:46 JST 1993
Copyright (C) 1993-2007 Yukihiro Matsumoto
Copyright (C) 2000 Network Applied Communication Laboratory, Inc.
Copyright (C) 2000 Information-technology Promotion Agency, Japan
**********************************************************************/
#define rb_data_object_alloc rb_data_object_alloc
#define rb_data_typed_object_alloc rb_data_typed_object_alloc
#include "ruby/internal/config.h"
#ifdef _WIN32
# include "ruby/ruby.h"
#endif
#include <signal.h>
#define sighandler_t ruby_sighandler_t
#ifndef _WIN32
#include <unistd.h>
#include <sys/mman.h>
#endif
#if defined(__wasm__) && !defined(__EMSCRIPTEN__)
# include "wasm/setjmp.h"
# include "wasm/machine.h"
#else
# include <setjmp.h>
#endif
#include <stdarg.h>
#include <stdio.h>
/* MALLOC_HEADERS_BEGIN */
#ifndef HAVE_MALLOC_USABLE_SIZE
# ifdef _WIN32
# define HAVE_MALLOC_USABLE_SIZE
# define malloc_usable_size(a) _msize(a)
# elif defined HAVE_MALLOC_SIZE
# define HAVE_MALLOC_USABLE_SIZE
# define malloc_usable_size(a) malloc_size(a)
# endif
#endif
#ifdef HAVE_MALLOC_USABLE_SIZE
# ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
/* Alternative malloc header is included in ruby/missing.h */
# elif defined(HAVE_MALLOC_H)
# include <malloc.h>
# elif defined(HAVE_MALLOC_NP_H)
# include <malloc_np.h>
# elif defined(HAVE_MALLOC_MALLOC_H)
# include <malloc/malloc.h>
# endif
#endif
#if !defined(PAGE_SIZE) && defined(HAVE_SYS_USER_H)
/* LIST_HEAD conflicts with sys/queue.h on macOS */
# include <sys/user.h>
#endif
/* MALLOC_HEADERS_END */
#ifdef HAVE_SYS_TIME_H
# include <sys/time.h>
#endif
#ifdef HAVE_SYS_RESOURCE_H
# include <sys/resource.h>
#endif
#if defined _WIN32 || defined __CYGWIN__
# include <windows.h>
#elif defined(HAVE_POSIX_MEMALIGN)
#elif defined(HAVE_MEMALIGN)
# include <malloc.h>
#endif
#include <sys/types.h>
#ifdef __EMSCRIPTEN__
#include <emscripten.h>
#endif
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
# include <mach/task.h>
# include <mach/mach_init.h>
# include <mach/mach_port.h>
#endif
#undef LIST_HEAD /* ccan/list conflicts with BSD-origin sys/queue.h. */
#include "constant.h"
#include "debug_counter.h"
#include "eval_intern.h"
#include "gc.h"
#include "id_table.h"
#include "internal.h"
#include "internal/class.h"
#include "internal/complex.h"
#include "internal/cont.h"
#include "internal/error.h"
#include "internal/eval.h"
#include "internal/gc.h"
#include "internal/hash.h"
#include "internal/imemo.h"
#include "internal/io.h"
#include "internal/numeric.h"
#include "internal/object.h"
#include "internal/proc.h"
#include "internal/rational.h"
#include "internal/sanitizers.h"
#include "internal/struct.h"
#include "internal/symbol.h"
#include "internal/thread.h"
#include "internal/variable.h"
#include "internal/warnings.h"
#include "mjit.h"
#include "probes.h"
#include "regint.h"
#include "ruby/debug.h"
#include "ruby/io.h"
#include "ruby/re.h"
#include "ruby/st.h"
#include "ruby/thread.h"
#include "ruby/util.h"
#include "ruby_assert.h"
#include "ruby_atomic.h"
#include "symbol.h"
#include "transient_heap.h"
#include "vm_core.h"
#include "vm_sync.h"
#include "vm_callinfo.h"
#include "ractor_core.h"
#include "builtin.h"
#define rb_setjmp(env) RUBY_SETJMP(env)
#define rb_jmp_buf rb_jmpbuf_t
#undef rb_data_object_wrap
#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
#define MAP_ANONYMOUS MAP_ANON
#endif
static inline struct rbimpl_size_mul_overflow_tag
size_add_overflow(size_t x, size_t y)
{
size_t z;
bool p;
#if 0
#elif __has_builtin(__builtin_add_overflow)
p = __builtin_add_overflow(x, y, &z);
#elif defined(DSIZE_T)
RB_GNUC_EXTENSION DSIZE_T dx = x;
RB_GNUC_EXTENSION DSIZE_T dy = y;
RB_GNUC_EXTENSION DSIZE_T dz = dx + dy;
p = dz > SIZE_MAX;
z = (size_t)dz;
#else
z = x + y;
p = z < y;
#endif
return (struct rbimpl_size_mul_overflow_tag) { p, z, };
}
static inline struct rbimpl_size_mul_overflow_tag
size_mul_add_overflow(size_t x, size_t y, size_t z) /* x * y + z */
{
struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(x, y);
struct rbimpl_size_mul_overflow_tag u = size_add_overflow(t.right, z);
return (struct rbimpl_size_mul_overflow_tag) { t.left || u.left, u.right };
}
static inline struct rbimpl_size_mul_overflow_tag
size_mul_add_mul_overflow(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
{
struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(x, y);
struct rbimpl_size_mul_overflow_tag u = rbimpl_size_mul_overflow(z, w);
struct rbimpl_size_mul_overflow_tag v = size_add_overflow(t.right, u.right);
return (struct rbimpl_size_mul_overflow_tag) { t.left || u.left || v.left, v.right };
}
PRINTF_ARGS(NORETURN(static void gc_raise(VALUE, const char*, ...)), 2, 3);
static inline size_t
size_mul_or_raise(size_t x, size_t y, VALUE exc)
{
struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(x, y);
if (LIKELY(!t.left)) {
return t.right;
}
else if (rb_during_gc()) {
rb_memerror(); /* or...? */
}
else {
gc_raise(
exc,
"integer overflow: %"PRIuSIZE
" * %"PRIuSIZE
" > %"PRIuSIZE,
x, y, (size_t)SIZE_MAX);
}
}
size_t
rb_size_mul_or_raise(size_t x, size_t y, VALUE exc)
{
return size_mul_or_raise(x, y, exc);
}
static inline size_t
size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
{
struct rbimpl_size_mul_overflow_tag t = size_mul_add_overflow(x, y, z);
if (LIKELY(!t.left)) {
return t.right;
}
else if (rb_during_gc()) {
rb_memerror(); /* or...? */
}
else {
gc_raise(
exc,
"integer overflow: %"PRIuSIZE
" * %"PRIuSIZE
" + %"PRIuSIZE
" > %"PRIuSIZE,
x, y, z, (size_t)SIZE_MAX);
}
}
size_t
rb_size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
{
return size_mul_add_or_raise(x, y, z, exc);
}
static inline size_t
size_mul_add_mul_or_raise(size_t x, size_t y, size_t z, size_t w, VALUE exc)
{
struct rbimpl_size_mul_overflow_tag t = size_mul_add_mul_overflow(x, y, z, w);
if (LIKELY(!t.left)) {
return t.right;
}
else if (rb_during_gc()) {
rb_memerror(); /* or...? */
}
else {
gc_raise(
exc,
"integer overflow: %"PRIdSIZE
" * %"PRIdSIZE
" + %"PRIdSIZE
" * %"PRIdSIZE
" > %"PRIdSIZE,
x, y, z, w, (size_t)SIZE_MAX);
}
}
#if defined(HAVE_RB_GC_GUARDED_PTR_VAL) && HAVE_RB_GC_GUARDED_PTR_VAL
/* trick the compiler into thinking a external signal handler uses this */
volatile VALUE rb_gc_guarded_val;
volatile VALUE *
rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val)
{
rb_gc_guarded_val = val;
return ptr;
}
#endif
#ifndef GC_HEAP_INIT_SLOTS
#define GC_HEAP_INIT_SLOTS 10000
#endif
#ifndef GC_HEAP_FREE_SLOTS
#define GC_HEAP_FREE_SLOTS 4096
#endif
#ifndef GC_HEAP_GROWTH_FACTOR
#define GC_HEAP_GROWTH_FACTOR 1.8
#endif
#ifndef GC_HEAP_GROWTH_MAX_SLOTS
#define GC_HEAP_GROWTH_MAX_SLOTS 0 /* 0 is disable */
#endif
#ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR
#define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0
#endif
#ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO
#define GC_HEAP_FREE_SLOTS_MIN_RATIO 0.20
#endif
#ifndef GC_HEAP_FREE_SLOTS_GOAL_RATIO
#define GC_HEAP_FREE_SLOTS_GOAL_RATIO 0.40
#endif
#ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO
#define GC_HEAP_FREE_SLOTS_MAX_RATIO 0.65
#endif
#ifndef GC_MALLOC_LIMIT_MIN
#define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
#endif
#ifndef GC_MALLOC_LIMIT_MAX
#define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 /* 32MB */)
#endif
#ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR
#define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4
#endif
#ifndef GC_OLDMALLOC_LIMIT_MIN
#define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
#endif
#ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
#define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2
#endif
#ifndef GC_OLDMALLOC_LIMIT_MAX
#define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 /* 128MB */)
#endif
#ifndef PRINT_MEASURE_LINE
#define PRINT_MEASURE_LINE 0
#endif
#ifndef PRINT_ENTER_EXIT_TICK
#define PRINT_ENTER_EXIT_TICK 0
#endif
#ifndef PRINT_ROOT_TICKS
#define PRINT_ROOT_TICKS 0
#endif
#define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_MEASURE_LINE || PRINT_ROOT_TICKS)
#define TICK_TYPE 1
typedef struct {
size_t heap_init_slots;
size_t heap_free_slots;
double growth_factor;
size_t growth_max_slots;
double heap_free_slots_min_ratio;
double heap_free_slots_goal_ratio;
double heap_free_slots_max_ratio;
double oldobject_limit_factor;
size_t malloc_limit_min;
size_t malloc_limit_max;
double malloc_limit_growth_factor;
size_t oldmalloc_limit_min;
size_t oldmalloc_limit_max;
double oldmalloc_limit_growth_factor;
VALUE gc_stress;
} ruby_gc_params_t;
static ruby_gc_params_t gc_params = {
GC_HEAP_INIT_SLOTS,
GC_HEAP_FREE_SLOTS,
GC_HEAP_GROWTH_FACTOR,
GC_HEAP_GROWTH_MAX_SLOTS,
GC_HEAP_FREE_SLOTS_MIN_RATIO,
GC_HEAP_FREE_SLOTS_GOAL_RATIO,
GC_HEAP_FREE_SLOTS_MAX_RATIO,
GC_HEAP_OLDOBJECT_LIMIT_FACTOR,
GC_MALLOC_LIMIT_MIN,
GC_MALLOC_LIMIT_MAX,
GC_MALLOC_LIMIT_GROWTH_FACTOR,
GC_OLDMALLOC_LIMIT_MIN,
GC_OLDMALLOC_LIMIT_MAX,
GC_OLDMALLOC_LIMIT_GROWTH_FACTOR,
FALSE,
};
/* GC_DEBUG:
* enable to embed GC debugging information.
*/
#ifndef GC_DEBUG
#define GC_DEBUG 0
#endif
/* RGENGC_DEBUG:
* 1: basic information
* 2: remember set operation
* 3: mark
* 4:
* 5: sweep
*/
#ifndef RGENGC_DEBUG
#ifdef RUBY_DEVEL
#define RGENGC_DEBUG -1
#else
#define RGENGC_DEBUG 0
#endif
#endif
#if RGENGC_DEBUG < 0 && !defined(_MSC_VER)
# define RGENGC_DEBUG_ENABLED(level) (-(RGENGC_DEBUG) >= (level) && ruby_rgengc_debug >= (level))
#elif defined(HAVE_VA_ARGS_MACRO)
# define RGENGC_DEBUG_ENABLED(level) ((RGENGC_DEBUG) >= (level))
#else
# define RGENGC_DEBUG_ENABLED(level) 0
#endif
int ruby_rgengc_debug;
/* RGENGC_CHECK_MODE
* 0: disable all assertions
* 1: enable assertions (to debug RGenGC)
* 2: enable internal consistency check at each GC (for debugging)
* 3: enable internal consistency check at each GC steps (for debugging)
* 4: enable liveness check
* 5: show all references
*/
#ifndef RGENGC_CHECK_MODE
#define RGENGC_CHECK_MODE 0
#endif
// Note: using RUBY_ASSERT_WHEN() extend a macro in expr (info by nobu).
#define GC_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(RGENGC_CHECK_MODE > 0, expr, #expr)
/* RGENGC_OLD_NEWOBJ_CHECK
* 0: disable all assertions
* >0: make a OLD object when new object creation.
*
* Make one OLD object per RGENGC_OLD_NEWOBJ_CHECK WB protected objects creation.
*/
#ifndef RGENGC_OLD_NEWOBJ_CHECK
#define RGENGC_OLD_NEWOBJ_CHECK 0
#endif
/* RGENGC_PROFILE
* 0: disable RGenGC profiling
* 1: enable profiling for basic information
* 2: enable profiling for each types
*/
#ifndef RGENGC_PROFILE
#define RGENGC_PROFILE 0
#endif
/* RGENGC_ESTIMATE_OLDMALLOC
* Enable/disable to estimate increase size of malloc'ed size by old objects.
* If estimation exceeds threshold, then will invoke full GC.
* 0: disable estimation.
* 1: enable estimation.
*/
#ifndef RGENGC_ESTIMATE_OLDMALLOC
#define RGENGC_ESTIMATE_OLDMALLOC 1
#endif
/* RGENGC_FORCE_MAJOR_GC
* Force major/full GC if this macro is not 0.
*/
#ifndef RGENGC_FORCE_MAJOR_GC
#define RGENGC_FORCE_MAJOR_GC 0
#endif
#ifndef GC_PROFILE_MORE_DETAIL
#define GC_PROFILE_MORE_DETAIL 0
#endif
#ifndef GC_PROFILE_DETAIL_MEMORY
#define GC_PROFILE_DETAIL_MEMORY 0
#endif
#ifndef GC_ENABLE_INCREMENTAL_MARK
#define GC_ENABLE_INCREMENTAL_MARK USE_RINCGC
#endif
#ifndef GC_ENABLE_LAZY_SWEEP
#define GC_ENABLE_LAZY_SWEEP 1
#endif
#ifndef CALC_EXACT_MALLOC_SIZE
#define CALC_EXACT_MALLOC_SIZE USE_GC_MALLOC_OBJ_INFO_DETAILS
#endif
#if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
#ifndef MALLOC_ALLOCATED_SIZE
#define MALLOC_ALLOCATED_SIZE 0
#endif
#else
#define MALLOC_ALLOCATED_SIZE 0
#endif
#ifndef MALLOC_ALLOCATED_SIZE_CHECK
#define MALLOC_ALLOCATED_SIZE_CHECK 0
#endif
#ifndef GC_DEBUG_STRESS_TO_CLASS
#define GC_DEBUG_STRESS_TO_CLASS 0
#endif
#ifndef RGENGC_OBJ_INFO
#define RGENGC_OBJ_INFO (RGENGC_DEBUG | RGENGC_CHECK_MODE)
#endif
typedef enum {
GPR_FLAG_NONE = 0x000,
/* major reason */
GPR_FLAG_MAJOR_BY_NOFREE = 0x001,
GPR_FLAG_MAJOR_BY_OLDGEN = 0x002,
GPR_FLAG_MAJOR_BY_SHADY = 0x004,
GPR_FLAG_MAJOR_BY_FORCE = 0x008,
#if RGENGC_ESTIMATE_OLDMALLOC
GPR_FLAG_MAJOR_BY_OLDMALLOC = 0x020,
#endif
GPR_FLAG_MAJOR_MASK = 0x0ff,
/* gc reason */
GPR_FLAG_NEWOBJ = 0x100,
GPR_FLAG_MALLOC = 0x200,
GPR_FLAG_METHOD = 0x400,
GPR_FLAG_CAPI = 0x800,
GPR_FLAG_STRESS = 0x1000,
/* others */
GPR_FLAG_IMMEDIATE_SWEEP = 0x2000,
GPR_FLAG_HAVE_FINALIZE = 0x4000,
GPR_FLAG_IMMEDIATE_MARK = 0x8000,
GPR_FLAG_FULL_MARK = 0x10000,
GPR_FLAG_COMPACT = 0x20000,
GPR_DEFAULT_REASON =
(GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK |
GPR_FLAG_IMMEDIATE_SWEEP | GPR_FLAG_CAPI),
} gc_profile_record_flag;
typedef struct gc_profile_record {
unsigned int flags;
double gc_time;
double gc_invoke_time;
size_t heap_total_objects;
size_t heap_use_size;
size_t heap_total_size;
size_t moved_objects;
#if GC_PROFILE_MORE_DETAIL
double gc_mark_time;
double gc_sweep_time;
size_t heap_use_pages;
size_t heap_live_objects;
size_t heap_free_objects;
size_t allocate_increase;
size_t allocate_limit;
double prepare_time;
size_t removing_objects;
size_t empty_objects;
#if GC_PROFILE_DETAIL_MEMORY
long maxrss;
long minflt;
long majflt;
#endif
#endif
#if MALLOC_ALLOCATED_SIZE
size_t allocated_size;
#endif
#if RGENGC_PROFILE > 0
size_t old_objects;
size_t remembered_normal_objects;
size_t remembered_shady_objects;
#endif
} gc_profile_record;
struct RMoved {
VALUE flags;
VALUE dummy;
VALUE destination;
};
#define RMOVED(obj) ((struct RMoved *)(obj))
typedef struct RVALUE {
union {
struct {
VALUE flags; /* always 0 for freed obj */
struct RVALUE *next;
} free;
struct RMoved moved;
struct RBasic basic;
struct RObject object;
struct RClass klass;
struct RFloat flonum;
struct RString string;
struct RArray array;
struct RRegexp regexp;
struct RHash hash;
struct RData data;
struct RTypedData typeddata;
struct RStruct rstruct;
struct RBignum bignum;
struct RFile file;
struct RMatch match;
struct RRational rational;
struct RComplex complex;
struct RSymbol symbol;
union {
rb_cref_t cref;
struct vm_svar svar;
struct vm_throw_data throw_data;
struct vm_ifunc ifunc;
struct MEMO memo;
struct rb_method_entry_struct ment;
const rb_iseq_t iseq;
rb_env_t env;
struct rb_imemo_tmpbuf_struct alloc;
rb_ast_t ast;
} imemo;
struct {
struct RBasic basic;
VALUE v1;
VALUE v2;
VALUE v3;
} values;
} as;
#if GC_DEBUG
const char *file;
int line;
#endif
} RVALUE;
#if GC_DEBUG
STATIC_ASSERT(sizeof_rvalue, offsetof(RVALUE, file) == SIZEOF_VALUE * 5);
#else
STATIC_ASSERT(sizeof_rvalue, sizeof(RVALUE) == SIZEOF_VALUE * 5);
#endif
STATIC_ASSERT(alignof_rvalue, RUBY_ALIGNOF(RVALUE) == SIZEOF_VALUE);
typedef uintptr_t bits_t;
enum {
BITS_SIZE = sizeof(bits_t),
BITS_BITLENGTH = ( BITS_SIZE * CHAR_BIT )
};
#define popcount_bits rb_popcount_intptr
struct heap_page_header {
struct heap_page *page;
};
struct heap_page_body {
struct heap_page_header header;
/* char gap[]; */
/* RVALUE values[]; */
};
struct gc_list {
VALUE *varptr;
struct gc_list *next;
};
#define STACK_CHUNK_SIZE 500
typedef struct stack_chunk {
VALUE data[STACK_CHUNK_SIZE];
struct stack_chunk *next;
} stack_chunk_t;
typedef struct mark_stack {
stack_chunk_t *chunk;
stack_chunk_t *cache;
int index;
int limit;
size_t cache_size;
size_t unused_cache_size;
} mark_stack_t;
#define SIZE_POOL_EDEN_HEAP(size_pool) (&(size_pool)->eden_heap)
#define SIZE_POOL_TOMB_HEAP(size_pool) (&(size_pool)->tomb_heap)
typedef struct rb_heap_struct {
struct heap_page *free_pages;
struct ccan_list_head pages;
struct heap_page *sweeping_page; /* iterator for .pages */
struct heap_page *compact_cursor;
uintptr_t compact_cursor_index;
#if GC_ENABLE_INCREMENTAL_MARK
struct heap_page *pooled_pages;
#endif
size_t total_pages; /* total page count in a heap */
size_t total_slots; /* total slot count (about total_pages * HEAP_PAGE_OBJ_LIMIT) */
} rb_heap_t;
typedef struct rb_size_pool_struct {
short slot_size;
size_t allocatable_pages;
/* Basic statistics */
size_t total_allocated_pages;
size_t total_freed_pages;
size_t force_major_gc_count;
#if USE_RVARGC
/* Sweeping statistics */
size_t freed_slots;
size_t empty_slots;
#endif
rb_heap_t eden_heap;
rb_heap_t tomb_heap;
} rb_size_pool_t;
enum gc_mode {
gc_mode_none,
gc_mode_marking,
gc_mode_sweeping,
gc_mode_compacting,
};
typedef struct rb_objspace {
struct {
size_t limit;
size_t increase;
#if MALLOC_ALLOCATED_SIZE
size_t allocated_size;
size_t allocations;
#endif
} malloc_params;
struct {
unsigned int mode : 2;
unsigned int immediate_sweep : 1;
unsigned int dont_gc : 1;
unsigned int dont_incremental : 1;
unsigned int during_gc : 1;
unsigned int during_compacting : 1;
unsigned int gc_stressful: 1;
unsigned int has_hook: 1;
unsigned int during_minor_gc : 1;
#if GC_ENABLE_INCREMENTAL_MARK
unsigned int during_incremental_marking : 1;
#endif
unsigned int measure_gc : 1;
} flags;
rb_event_flag_t hook_events;
size_t total_allocated_objects;
VALUE next_object_id;
rb_size_pool_t size_pools[SIZE_POOL_COUNT];
struct {
rb_atomic_t finalizing;
} atomic_flags;
mark_stack_t mark_stack;
size_t marked_slots;
struct {
struct heap_page **sorted;
size_t allocated_pages;
size_t allocatable_pages;
size_t sorted_length;
uintptr_t range[2];
size_t freeable_pages;
/* final */
size_t final_slots;
VALUE deferred_final;
} heap_pages;
st_table *finalizer_table;
struct {
int run;
unsigned int latest_gc_info;
gc_profile_record *records;
gc_profile_record *current_record;
size_t next_index;
size_t size;
#if GC_PROFILE_MORE_DETAIL
double prepare_time;
#endif
double invoke_time;
size_t minor_gc_count;
size_t major_gc_count;
size_t compact_count;
size_t read_barrier_faults;
#if RGENGC_PROFILE > 0
size_t total_generated_normal_object_count;
size_t total_generated_shady_object_count;
size_t total_shade_operation_count;
size_t total_promoted_count;
size_t total_remembered_normal_object_count;
size_t total_remembered_shady_object_count;
#if RGENGC_PROFILE >= 2
size_t generated_normal_object_count_types[RUBY_T_MASK];
size_t generated_shady_object_count_types[RUBY_T_MASK];
size_t shade_operation_count_types[RUBY_T_MASK];
size_t promoted_types[RUBY_T_MASK];
size_t remembered_normal_object_count_types[RUBY_T_MASK];
size_t remembered_shady_object_count_types[RUBY_T_MASK];
#endif
#endif /* RGENGC_PROFILE */
/* temporary profiling space */
double gc_sweep_start_time;
size_t total_allocated_objects_at_gc_start;
size_t heap_used_at_gc_start;
/* basic statistics */
size_t count;
size_t total_freed_objects;
uint64_t total_time_ns;
struct timespec start_time;
} profile;
struct gc_list *global_list;
VALUE gc_stress_mode;
struct {
VALUE parent_object;
int need_major_gc;
size_t last_major_gc;
size_t uncollectible_wb_unprotected_objects;
size_t uncollectible_wb_unprotected_objects_limit;
size_t old_objects;
size_t old_objects_limit;
#if RGENGC_ESTIMATE_OLDMALLOC
size_t oldmalloc_increase;
size_t oldmalloc_increase_limit;
#endif
#if RGENGC_CHECK_MODE >= 2
struct st_table *allrefs_table;
size_t error_count;
#endif
} rgengc;
struct {
size_t considered_count_table[T_MASK];
size_t moved_count_table[T_MASK];
size_t moved_up_count_table[T_MASK];
size_t moved_down_count_table[T_MASK];
size_t total_moved;
} rcompactor;
#if GC_ENABLE_INCREMENTAL_MARK
struct {
size_t pooled_slots;
size_t step_slots;
} rincgc;
#endif
st_table *id_to_obj_tbl;
st_table *obj_to_id_tbl;
#if GC_DEBUG_STRESS_TO_CLASS
VALUE stress_to_class;
#endif
} rb_objspace_t;
#ifndef HEAP_PAGE_ALIGN_LOG
/* default tiny heap size: 64KiB */
#define HEAP_PAGE_ALIGN_LOG 16
#endif
#define BASE_SLOT_SIZE sizeof(RVALUE)
#define CEILDIV(i, mod) roomof(i, mod)
enum {
HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG),
HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)),
HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN,
HEAP_PAGE_OBJ_LIMIT = (unsigned int)((HEAP_PAGE_SIZE - sizeof(struct heap_page_header)) / BASE_SLOT_SIZE),
HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, BASE_SLOT_SIZE), BITS_BITLENGTH),
HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
};
#define HEAP_PAGE_ALIGN (1 << HEAP_PAGE_ALIGN_LOG)
#define HEAP_PAGE_SIZE HEAP_PAGE_ALIGN
#if GC_ENABLE_INCREMENTAL_MARK && !defined(INCREMENTAL_MARK_STEP_ALLOCATIONS)
# define INCREMENTAL_MARK_STEP_ALLOCATIONS 500
#endif
#undef INIT_HEAP_PAGE_ALLOC_USE_MMAP
/* Must define either HEAP_PAGE_ALLOC_USE_MMAP or
* INIT_HEAP_PAGE_ALLOC_USE_MMAP. */
#ifndef HAVE_MMAP
/* We can't use mmap of course, if it is not available. */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
#elif defined(__wasm__)
/* wasmtime does not have proper support for mmap.
* See https://github.com/bytecodealliance/wasmtime/blob/main/docs/WASI-rationale.md#why-no-mmap-and-friends
*/
static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
#elif HAVE_CONST_PAGE_SIZE
/* If we have the PAGE_SIZE and it is a constant, then we can directly use it. */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = (PAGE_SIZE <= HEAP_PAGE_SIZE);
#elif defined(PAGE_MAX_SIZE) && (PAGE_MAX_SIZE <= HEAP_PAGE_SIZE)
/* If we can use the maximum page size. */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = true;
#elif defined(PAGE_SIZE)
/* If the PAGE_SIZE macro can be used dynamically. */
# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (PAGE_SIZE <= HEAP_PAGE_SIZE)
#elif defined(HAVE_SYSCONF) && defined(_SC_PAGE_SIZE)
/* If we can use sysconf to determine the page size. */
# define INIT_HEAP_PAGE_ALLOC_USE_MMAP (sysconf(_SC_PAGE_SIZE) <= HEAP_PAGE_SIZE)
#else
/* Otherwise we can't determine the system page size, so don't use mmap. */
static const bool HEAP_PAGE_ALLOC_USE_MMAP = false;
#endif
#ifdef INIT_HEAP_PAGE_ALLOC_USE_MMAP
/* We can determine the system page size at runtime. */
# define HEAP_PAGE_ALLOC_USE_MMAP (heap_page_alloc_use_mmap != false)
static bool heap_page_alloc_use_mmap;
#endif
struct heap_page {
short slot_size;
short total_slots;
short free_slots;
short pinned_slots;
short final_slots;
struct {
unsigned int before_sweep : 1;
unsigned int has_remembered_objects : 1;
unsigned int has_uncollectible_shady_objects : 1;
unsigned int in_tomb : 1;
} flags;
rb_size_pool_t *size_pool;
struct heap_page *free_next;
uintptr_t start;
RVALUE *freelist;
struct ccan_list_node page_node;
bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT];
/* the following three bitmaps are cleared at the beginning of full GC */
bits_t mark_bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT];
/* If set, the object is not movable */
bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT];
};
/*
* When asan is enabled, this will prohibit writing to the freelist until it is unlocked
*/
static void
asan_lock_freelist(struct heap_page *page)
{
asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
}
/*
* When asan is enabled, this will enable the ability to write to the freelist
*/
static void
asan_unlock_freelist(struct heap_page *page)
{
asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
}
#define GET_PAGE_BODY(x) ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_PAGE_ALIGN_MASK)))
#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
#define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)
#define NUM_IN_PAGE(p) (((bits_t)(p) & HEAP_PAGE_ALIGN_MASK) / BASE_SLOT_SIZE)
#define BITMAP_INDEX(p) (NUM_IN_PAGE(p) / BITS_BITLENGTH )
#define BITMAP_OFFSET(p) (NUM_IN_PAGE(p) & (BITS_BITLENGTH-1))
#define BITMAP_BIT(p) ((bits_t)1 << BITMAP_OFFSET(p))
/* Bitmap Operations */
#define MARKED_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] & BITMAP_BIT(p))
#define MARK_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] | BITMAP_BIT(p))
#define CLEAR_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] & ~BITMAP_BIT(p))
/* getting bitmap */
#define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0])
#define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0])
#define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
#define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
#define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0])
#define GC_SWEEP_PAGES_FREEABLE_PER_STEP 3
/* Aliases */
#define rb_objspace (*rb_objspace_of(GET_VM()))
#define rb_objspace_of(vm) ((vm)->objspace)
#define ruby_initial_gc_stress gc_params.gc_stress
VALUE *ruby_initial_gc_stress_ptr = &ruby_initial_gc_stress;
#define malloc_limit objspace->malloc_params.limit
#define malloc_increase objspace->malloc_params.increase
#define malloc_allocated_size objspace->malloc_params.allocated_size
#define heap_pages_sorted objspace->heap_pages.sorted
#define heap_allocated_pages objspace->heap_pages.allocated_pages
#define heap_pages_sorted_length objspace->heap_pages.sorted_length
#define heap_pages_lomem objspace->heap_pages.range[0]
#define heap_pages_himem objspace->heap_pages.range[1]
#define heap_pages_freeable_pages objspace->heap_pages.freeable_pages
#define heap_pages_final_slots objspace->heap_pages.final_slots
#define heap_pages_deferred_final objspace->heap_pages.deferred_final
#define size_pools objspace->size_pools
#define during_gc objspace->flags.during_gc
#define finalizing objspace->atomic_flags.finalizing
#define finalizer_table objspace->finalizer_table
#define global_list objspace->global_list
#define ruby_gc_stressful objspace->flags.gc_stressful
#define ruby_gc_stress_mode objspace->gc_stress_mode
#if GC_DEBUG_STRESS_TO_CLASS
#define stress_to_class objspace->stress_to_class
#else
#define stress_to_class 0
#endif
#if 0
#define dont_gc_on() (fprintf(stderr, "dont_gc_on@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 1)
#define dont_gc_off() (fprintf(stderr, "dont_gc_off@%s:%d\n", __FILE__, __LINE__), objspace->flags.dont_gc = 0)
#define dont_gc_set(b) (fprintf(stderr, "dont_gc_set(%d)@%s:%d\n", __FILE__, __LINE__), (int)b), objspace->flags.dont_gc = (b))
#define dont_gc_val() (objspace->flags.dont_gc)
#else
#define dont_gc_on() (objspace->flags.dont_gc = 1)
#define dont_gc_off() (objspace->flags.dont_gc = 0)
#define dont_gc_set(b) (((int)b), objspace->flags.dont_gc = (b))
#define dont_gc_val() (objspace->flags.dont_gc)
#endif
static inline enum gc_mode
gc_mode_verify(enum gc_mode mode)
{
#if RGENGC_CHECK_MODE > 0
switch (mode) {
case gc_mode_none:
case gc_mode_marking:
case gc_mode_sweeping:
case gc_mode_compacting:
break;
default:
rb_bug("gc_mode_verify: unreachable (%d)", (int)mode);
}
#endif
return mode;
}
static inline bool
has_sweeping_pages(rb_objspace_t *objspace)
{
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
if (SIZE_POOL_EDEN_HEAP(&size_pools[i])->sweeping_page) {
return TRUE;
}
}
return FALSE;
}
static inline size_t
heap_eden_total_pages(rb_objspace_t *objspace)
{
size_t count = 0;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
count += SIZE_POOL_EDEN_HEAP(&size_pools[i])->total_pages;
}
return count;
}
static inline size_t
heap_eden_total_slots(rb_objspace_t *objspace)
{
size_t count = 0;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
count += SIZE_POOL_EDEN_HEAP(&size_pools[i])->total_slots;
}
return count;
}
static inline size_t
heap_tomb_total_pages(rb_objspace_t *objspace)
{
size_t count = 0;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
count += SIZE_POOL_TOMB_HEAP(&size_pools[i])->total_pages;
}
return count;
}
static inline size_t
heap_allocatable_pages(rb_objspace_t *objspace)
{
size_t count = 0;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
count += size_pools[i].allocatable_pages;
}
return count;
}
static inline size_t
heap_allocatable_slots(rb_objspace_t *objspace)
{
size_t count = 0;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
int slot_size_multiple = size_pool->slot_size / BASE_SLOT_SIZE;
count += size_pool->allocatable_pages * HEAP_PAGE_OBJ_LIMIT / slot_size_multiple;
}
return count;
}
static inline size_t
total_allocated_pages(rb_objspace_t *objspace)
{
size_t count = 0;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
count += size_pool->total_allocated_pages;
}
return count;
}
static inline size_t
total_freed_pages(rb_objspace_t *objspace)
{
size_t count = 0;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
count += size_pool->total_freed_pages;
}
return count;
}
#define gc_mode(objspace) gc_mode_verify((enum gc_mode)(objspace)->flags.mode)
#define gc_mode_set(objspace, mode) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(mode))
#define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking)
#define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping)
#define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE)
#if GC_ENABLE_INCREMENTAL_MARK
#define is_incremental_marking(objspace) ((objspace)->flags.during_incremental_marking != FALSE)
#else
#define is_incremental_marking(objspace) FALSE
#endif
#if GC_ENABLE_INCREMENTAL_MARK
#define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
#else
#define will_be_incremental_marking(objspace) FALSE
#endif
#if GC_ENABLE_INCREMENTAL_MARK
#define GC_INCREMENTAL_SWEEP_SLOT_COUNT 2048
#endif
#define is_lazy_sweeping(objspace) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(objspace))
#if SIZEOF_LONG == SIZEOF_VOIDP
# define nonspecial_obj_id(obj) (VALUE)((SIGNED_VALUE)(obj)|FIXNUM_FLAG)
# define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG) /* unset FIXNUM_FLAG */
#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
# define nonspecial_obj_id(obj) LL2NUM((SIGNED_VALUE)(obj) / 2)
# define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
#else
# error not supported
#endif
#define RANY(o) ((RVALUE*)(o))
struct RZombie {
struct RBasic basic;
VALUE next;
void (*dfree)(void *);
void *data;
};
#define RZOMBIE(o) ((struct RZombie *)(o))
#define nomem_error GET_VM()->special_exceptions[ruby_error_nomemory]
#if RUBY_MARK_FREE_DEBUG
int ruby_gc_debug_indent = 0;
#endif
VALUE rb_mGC;
int ruby_disable_gc = 0;
int ruby_enable_autocompact = 0;
void rb_iseq_mark(const rb_iseq_t *iseq);
void rb_iseq_update_references(rb_iseq_t *iseq);
void rb_iseq_free(const rb_iseq_t *iseq);
size_t rb_iseq_memsize(const rb_iseq_t *iseq);
void rb_vm_update_references(void *ptr);
void rb_gcdebug_print_obj_condition(VALUE obj);
static VALUE define_final0(VALUE obj, VALUE block);
NORETURN(static void *gc_vraise(void *ptr));
NORETURN(static void gc_raise(VALUE exc, const char *fmt, ...));
NORETURN(static void negative_size_allocation_error(const char *));
static void init_mark_stack(mark_stack_t *stack);
static int ready_to_gc(rb_objspace_t *objspace);
static int garbage_collect(rb_objspace_t *, unsigned int reason);
static int gc_start(rb_objspace_t *objspace, unsigned int reason);
static void gc_rest(rb_objspace_t *objspace);
enum gc_enter_event {
gc_enter_event_start,
gc_enter_event_mark_continue,
gc_enter_event_sweep_continue,
gc_enter_event_rest,
gc_enter_event_finalizer,
gc_enter_event_rb_memerror,
};
static inline void gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev);
static inline void gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev);
static void gc_marks(rb_objspace_t *objspace, int full_mark);
static void gc_marks_start(rb_objspace_t *objspace, int full);
static void gc_marks_finish(rb_objspace_t *objspace);
static void gc_marks_rest(rb_objspace_t *objspace);
static void gc_marks_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap);
static void gc_sweep(rb_objspace_t *objspace);
static void gc_sweep_start(rb_objspace_t *objspace);
#if USE_RVARGC
static void gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool);
#endif
static void gc_sweep_finish(rb_objspace_t *objspace);
static int gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap);
static void gc_sweep_rest(rb_objspace_t *objspace);
static void gc_sweep_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap);
static inline void gc_mark(rb_objspace_t *objspace, VALUE ptr);
static inline void gc_pin(rb_objspace_t *objspace, VALUE ptr);
static inline void gc_mark_and_pin(rb_objspace_t *objspace, VALUE ptr);
static void gc_mark_ptr(rb_objspace_t *objspace, VALUE ptr);
NO_SANITIZE("memory", static void gc_mark_maybe(rb_objspace_t *objspace, VALUE ptr));
static void gc_mark_children(rb_objspace_t *objspace, VALUE ptr);
static int gc_mark_stacked_objects_incremental(rb_objspace_t *, size_t count);
static int gc_mark_stacked_objects_all(rb_objspace_t *);
static void gc_grey(rb_objspace_t *objspace, VALUE ptr);
static inline int gc_mark_set(rb_objspace_t *objspace, VALUE obj);
NO_SANITIZE("memory", static inline int is_pointer_to_heap(rb_objspace_t *objspace, void *ptr));
static void push_mark_stack(mark_stack_t *, VALUE);
static int pop_mark_stack(mark_stack_t *, VALUE *);
static size_t mark_stack_size(mark_stack_t *stack);
static void shrink_stack_chunk_cache(mark_stack_t *stack);
static size_t obj_memsize_of(VALUE obj, int use_all_types);
static void gc_verify_internal_consistency(rb_objspace_t *objspace);
static int gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj);
static int gc_verify_heap_pages(rb_objspace_t *objspace);
static void gc_stress_set(rb_objspace_t *objspace, VALUE flag);
static VALUE gc_disable_no_rest(rb_objspace_t *);
static double getrusage_time(void);
static inline void gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason);
static inline void gc_prof_timer_start(rb_objspace_t *);
static inline void gc_prof_timer_stop(rb_objspace_t *);
static inline void gc_prof_mark_timer_start(rb_objspace_t *);
static inline void gc_prof_mark_timer_stop(rb_objspace_t *);
static inline void gc_prof_sweep_timer_start(rb_objspace_t *);
static inline void gc_prof_sweep_timer_stop(rb_objspace_t *);
static inline void gc_prof_set_malloc_info(rb_objspace_t *);
static inline void gc_prof_set_heap_info(rb_objspace_t *);
#define TYPED_UPDATE_IF_MOVED(_objspace, _type, _thing) do { \
if (gc_object_moved_p((_objspace), (VALUE)(_thing))) { \
*(_type *)&(_thing) = (_type)RMOVED(_thing)->destination; \
} \
} while (0)
#define UPDATE_IF_MOVED(_objspace, _thing) TYPED_UPDATE_IF_MOVED(_objspace, VALUE, _thing)
#define gc_prof_record(objspace) (objspace)->profile.current_record
#define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record)
#ifdef HAVE_VA_ARGS_MACRO
# define gc_report(level, objspace, ...) \
if (!RGENGC_DEBUG_ENABLED(level)) {} else gc_report_body(level, objspace, __VA_ARGS__)
#else
# define gc_report if (!RGENGC_DEBUG_ENABLED(0)) {} else gc_report_body
#endif
PRINTF_ARGS(static void gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...), 3, 4);
static const char *obj_info(VALUE obj);
static const char *obj_type_name(VALUE obj);
/*
* 1 - TSC (H/W Time Stamp Counter)
* 2 - getrusage
*/
#ifndef TICK_TYPE
#define TICK_TYPE 1
#endif
#if USE_TICK_T
#if TICK_TYPE == 1
/* the following code is only for internal tuning. */
/* Source code to use RDTSC is quoted and modified from
* https://www.mcs.anl.gov/~kazutomo/rdtsc.html
* written by Kazutomo Yoshii <kazutomo@mcs.anl.gov>
*/
#if defined(__GNUC__) && defined(__i386__)
typedef unsigned long long tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
unsigned long long int x;
__asm__ __volatile__ ("rdtsc" : "=A" (x));
return x;
}
#elif defined(__GNUC__) && defined(__x86_64__)
typedef unsigned long long tick_t;
#define PRItick "llu"
static __inline__ tick_t
tick(void)
{
unsigned long hi, lo;
__asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi));
return ((unsigned long long)lo)|( ((unsigned long long)hi)<<32);
}
#elif defined(__powerpc64__) && GCC_VERSION_SINCE(4,8,0)
typedef unsigned long long tick_t;
#define PRItick "llu"
static __inline__ tick_t
tick(void)
{
unsigned long long val = __builtin_ppc_get_timebase();
return val;
}
#elif defined(__aarch64__) && defined(__GNUC__)
typedef unsigned long tick_t;
#define PRItick "lu"
static __inline__ tick_t
tick(void)
{
unsigned long val;
__asm__ __volatile__ ("mrs %0, cntvct_el0" : "=r" (val));
return val;
}
#elif defined(_WIN32) && defined(_MSC_VER)
#include <intrin.h>
typedef unsigned __int64 tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
return __rdtsc();
}
#else /* use clock */
typedef clock_t tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
return clock();
}
#endif /* TSC */
#elif TICK_TYPE == 2
typedef double tick_t;
#define PRItick "4.9f"
static inline tick_t
tick(void)
{
return getrusage_time();
}
#else /* TICK_TYPE */
#error "choose tick type"
#endif /* TICK_TYPE */
#define MEASURE_LINE(expr) do { \
volatile tick_t start_time = tick(); \
volatile tick_t end_time; \
expr; \
end_time = tick(); \
fprintf(stderr, "0\t%"PRItick"\t%s\n", end_time - start_time, #expr); \
} while (0)
#else /* USE_TICK_T */
#define MEASURE_LINE(expr) expr
#endif /* USE_TICK_T */
static inline void *
asan_unpoison_object_temporary(VALUE obj)
{
void *ptr = asan_poisoned_object_p(obj);
asan_unpoison_object(obj, false);
return ptr;
}
static inline void *
asan_poison_object_restore(VALUE obj, void *ptr)
{
if (ptr) {
asan_poison_object(obj);
}
return NULL;
}
#define asan_unpoisoning_object(obj) \
for (void *poisoned = asan_unpoison_object_temporary(obj), \
*unpoisoning = &poisoned; /* flag to loop just once */ \
unpoisoning; \
unpoisoning = asan_poison_object_restore(obj, poisoned))
#define FL_CHECK2(name, x, pred) \
((RGENGC_CHECK_MODE && SPECIAL_CONST_P(x)) ? \
(rb_bug(name": SPECIAL_CONST (%p)", (void *)(x)), 0) : (pred))
#define FL_TEST2(x,f) FL_CHECK2("FL_TEST2", x, FL_TEST_RAW((x),(f)) != 0)
#define FL_SET2(x,f) FL_CHECK2("FL_SET2", x, RBASIC(x)->flags |= (f))
#define FL_UNSET2(x,f) FL_CHECK2("FL_UNSET2", x, RBASIC(x)->flags &= ~(f))
#define RVALUE_MARK_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj))
#define RVALUE_PIN_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), (obj))
#define RVALUE_PAGE_MARKED(page, obj) MARKED_IN_BITMAP((page)->mark_bits, (obj))
#define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj))
#define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj))
#define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj))
#define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj))
#define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
#define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj))
#define RVALUE_OLD_AGE 3
#define RVALUE_AGE_SHIFT 5 /* FL_PROMOTED0 bit */
static int rgengc_remembered(rb_objspace_t *objspace, VALUE obj);
static int rgengc_remembered_sweep(rb_objspace_t *objspace, VALUE obj);
static int rgengc_remember(rb_objspace_t *objspace, VALUE obj);
static void rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap);
static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap);
static inline int
RVALUE_FLAGS_AGE(VALUE flags)
{
return (int)((flags & (FL_PROMOTED0 | FL_PROMOTED1)) >> RVALUE_AGE_SHIFT);
}
static int
check_rvalue_consistency_force(const VALUE obj, int terminate)
{
int err = 0;
rb_objspace_t *objspace = &rb_objspace;
RB_VM_LOCK_ENTER_NO_BARRIER();
{
if (SPECIAL_CONST_P(obj)) {
fprintf(stderr, "check_rvalue_consistency: %p is a special const.\n", (void *)obj);
err++;
}
else if (!is_pointer_to_heap(objspace, (void *)obj)) {
/* check if it is in tomb_pages */
struct heap_page *page = NULL;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
ccan_list_for_each(&size_pool->tomb_heap.pages, page, page_node) {
if (page->start <= (uintptr_t)obj &&
(uintptr_t)obj < (page->start + (page->total_slots * size_pool->slot_size))) {
fprintf(stderr, "check_rvalue_consistency: %p is in a tomb_heap (%p).\n",
(void *)obj, (void *)page);
err++;
goto skip;
}
}
}
bp();
fprintf(stderr, "check_rvalue_consistency: %p is not a Ruby object.\n", (void *)obj);
err++;
skip:
;
}
else {
const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
const int mark_bit = RVALUE_MARK_BITMAP(obj) != 0;
const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0, remembered_bit = marking_bit;
const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
if (GET_HEAP_PAGE(obj)->flags.in_tomb) {
fprintf(stderr, "check_rvalue_consistency: %s is in tomb page.\n", obj_info(obj));
err++;
}
if (BUILTIN_TYPE(obj) == T_NONE) {
fprintf(stderr, "check_rvalue_consistency: %s is T_NONE.\n", obj_info(obj));
err++;
}
if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
fprintf(stderr, "check_rvalue_consistency: %s is T_ZOMBIE.\n", obj_info(obj));
err++;
}
obj_memsize_of((VALUE)obj, FALSE);
/* check generation
*
* OLD == age == 3 && old-bitmap && mark-bit (except incremental marking)
*/
if (age > 0 && wb_unprotected_bit) {
fprintf(stderr, "check_rvalue_consistency: %s is not WB protected, but age is %d > 0.\n", obj_info(obj), age);
err++;
}
if (!is_marking(objspace) && uncollectible_bit && !mark_bit) {
fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.\n", obj_info(obj));
err++;
}
if (!is_full_marking(objspace)) {
if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) {
fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.\n",
obj_info(obj), age);
err++;
}
if (remembered_bit && age != RVALUE_OLD_AGE) {
fprintf(stderr, "check_rvalue_consistency: %s is remembered, but not old (age: %d).\n",
obj_info(obj), age);
err++;
}
}
/*
* check coloring
*
* marking:false marking:true
* marked:false white *invalid*
* marked:true black grey
*/
if (is_incremental_marking(objspace) && marking_bit) {
if (!is_marking(objspace) && !mark_bit) {
fprintf(stderr, "check_rvalue_consistency: %s is marking, but not marked.\n", obj_info(obj));
err++;
}
}
}
}
RB_VM_LOCK_LEAVE_NO_BARRIER();
if (err > 0 && terminate) {
rb_bug("check_rvalue_consistency_force: there is %d errors.", err);
}
return err;
}
#if RGENGC_CHECK_MODE == 0
static inline VALUE
check_rvalue_consistency(const VALUE obj)
{
return obj;
}
#else
static VALUE
check_rvalue_consistency(const VALUE obj)
{
check_rvalue_consistency_force(obj, TRUE);
return obj;
}
#endif
static inline int
gc_object_moved_p(rb_objspace_t * objspace, VALUE obj)
{
if (RB_SPECIAL_CONST_P(obj)) {
return FALSE;
}
else {
void *poisoned = asan_unpoison_object_temporary(obj);
int ret = BUILTIN_TYPE(obj) == T_MOVED;
/* Re-poison slot if it's not the one we want */
if (poisoned) {
GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
asan_poison_object(obj);
}
return ret;
}
}
static inline int
RVALUE_MARKED(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_MARK_BITMAP(obj) != 0;
}
static inline int
RVALUE_PINNED(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_PIN_BITMAP(obj) != 0;
}
static inline int
RVALUE_WB_UNPROTECTED(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
}
static inline int
RVALUE_MARKING(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_MARKING_BITMAP(obj) != 0;
}
static inline int
RVALUE_REMEMBERED(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_MARKING_BITMAP(obj) != 0;
}
static inline int
RVALUE_UNCOLLECTIBLE(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
}
static inline int
RVALUE_OLD_P_RAW(VALUE obj)
{
const VALUE promoted = FL_PROMOTED0 | FL_PROMOTED1;
return (RBASIC(obj)->flags & promoted) == promoted;
}
static inline int
RVALUE_OLD_P(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_OLD_P_RAW(obj);
}
#if RGENGC_CHECK_MODE || GC_DEBUG
static inline int
RVALUE_AGE(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
}
#endif
static inline void
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
objspace->rgengc.old_objects++;
rb_transient_heap_promote(obj);
#if RGENGC_PROFILE >= 2
objspace->profile.total_promoted_count++;
objspace->profile.promoted_types[BUILTIN_TYPE(obj)]++;
#endif
}
static inline void
RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj)
{
RB_DEBUG_COUNTER_INC(obj_promote);
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj);
}
static inline VALUE
RVALUE_FLAGS_AGE_SET(VALUE flags, int age)
{
flags &= ~(FL_PROMOTED0 | FL_PROMOTED1);
flags |= (age << RVALUE_AGE_SHIFT);
return flags;
}
/* set age to age+1 */
static inline void
RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj)
{
VALUE flags = RBASIC(obj)->flags;
int age = RVALUE_FLAGS_AGE(flags);
if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", obj_info(obj));
}
age++;
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(flags, age);
if (age == RVALUE_OLD_AGE) {
RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
}
check_rvalue_consistency(obj);
}
/* set age to RVALUE_OLD_AGE */
static inline void
RVALUE_AGE_SET_OLD(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(obj);
GC_ASSERT(!RVALUE_OLD_P(obj));
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, RVALUE_OLD_AGE);
RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
check_rvalue_consistency(obj);
}
/* set age to RVALUE_OLD_AGE - 1 */
static inline void
RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(obj);
GC_ASSERT(!RVALUE_OLD_P(obj));
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, RVALUE_OLD_AGE - 1);
check_rvalue_consistency(obj);
}
static inline void
RVALUE_DEMOTE_RAW(rb_objspace_t *objspace, VALUE obj)
{
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0);
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
}
static inline void
RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(obj);
GC_ASSERT(RVALUE_OLD_P(obj));
if (!is_incremental_marking(objspace) && RVALUE_REMEMBERED(obj)) {
CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
}
RVALUE_DEMOTE_RAW(objspace, obj);
if (RVALUE_MARKED(obj)) {
objspace->rgengc.old_objects--;
}
check_rvalue_consistency(obj);
}
static inline void
RVALUE_AGE_RESET_RAW(VALUE obj)
{
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0);
}
static inline void
RVALUE_AGE_RESET(VALUE obj)
{
check_rvalue_consistency(obj);
GC_ASSERT(!RVALUE_OLD_P(obj));
RVALUE_AGE_RESET_RAW(obj);
check_rvalue_consistency(obj);
}
static inline int
RVALUE_BLACK_P(VALUE obj)
{
return RVALUE_MARKED(obj) && !RVALUE_MARKING(obj);
}
#if 0
static inline int
RVALUE_GREY_P(VALUE obj)
{
return RVALUE_MARKED(obj) && RVALUE_MARKING(obj);
}
#endif
static inline int
RVALUE_WHITE_P(VALUE obj)
{
return RVALUE_MARKED(obj) == FALSE;
}
/*
--------------------------- ObjectSpace -----------------------------
*/
static inline void *
calloc1(size_t n)
{
return calloc(1, n);
}
rb_objspace_t *
rb_objspace_alloc(void)
{
rb_objspace_t *objspace = calloc1(sizeof(rb_objspace_t));
objspace->flags.measure_gc = 1;
malloc_limit = gc_params.malloc_limit_min;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
size_pool->slot_size = (1 << i) * BASE_SLOT_SIZE;
ccan_list_head_init(&SIZE_POOL_EDEN_HEAP(size_pool)->pages);
ccan_list_head_init(&SIZE_POOL_TOMB_HEAP(size_pool)->pages);
}
dont_gc_on();
return objspace;
}
static void free_stack_chunks(mark_stack_t *);
static void mark_stack_free_cache(mark_stack_t *);
static void heap_page_free(rb_objspace_t *objspace, struct heap_page *page);
void
rb_objspace_free(rb_objspace_t *objspace)
{
if (is_lazy_sweeping(objspace))
rb_bug("lazy sweeping underway when freeing object space");
if (objspace->profile.records) {
free(objspace->profile.records);
objspace->profile.records = 0;
}
if (global_list) {
struct gc_list *list, *next;
for (list = global_list; list; list = next) {
next = list->next;
xfree(list);
}
}
if (heap_pages_sorted) {
size_t i;
for (i = 0; i < heap_allocated_pages; ++i) {
heap_page_free(objspace, heap_pages_sorted[i]);
}
free(heap_pages_sorted);
heap_allocated_pages = 0;
heap_pages_sorted_length = 0;
heap_pages_lomem = 0;
heap_pages_himem = 0;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
SIZE_POOL_EDEN_HEAP(size_pool)->total_pages = 0;
SIZE_POOL_EDEN_HEAP(size_pool)->total_slots = 0;
}
}
st_free_table(objspace->id_to_obj_tbl);
st_free_table(objspace->obj_to_id_tbl);
free_stack_chunks(&objspace->mark_stack);
mark_stack_free_cache(&objspace->mark_stack);
free(objspace);
}
static void
heap_pages_expand_sorted_to(rb_objspace_t *objspace, size_t next_length)
{
struct heap_page **sorted;
size_t size = size_mul_or_raise(next_length, sizeof(struct heap_page *), rb_eRuntimeError);
gc_report(3, objspace, "heap_pages_expand_sorted: next_length: %"PRIdSIZE", size: %"PRIdSIZE"\n",
next_length, size);
if (heap_pages_sorted_length > 0) {
sorted = (struct heap_page **)realloc(heap_pages_sorted, size);
if (sorted) heap_pages_sorted = sorted;
}
else {
sorted = heap_pages_sorted = (struct heap_page **)malloc(size);
}
if (sorted == 0) {
rb_memerror();
}
heap_pages_sorted_length = next_length;
}
static void
heap_pages_expand_sorted(rb_objspace_t *objspace)
{
/* usually heap_allocatable_pages + heap_eden->total_pages == heap_pages_sorted_length
* because heap_allocatable_pages contains heap_tomb->total_pages (recycle heap_tomb pages).
* however, if there are pages which do not have empty slots, then try to create new pages
* so that the additional allocatable_pages counts (heap_tomb->total_pages) are added.
*/
size_t next_length = heap_allocatable_pages(objspace);
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
next_length += SIZE_POOL_EDEN_HEAP(size_pool)->total_pages;
next_length += SIZE_POOL_TOMB_HEAP(size_pool)->total_pages;
}
if (next_length > heap_pages_sorted_length) {
heap_pages_expand_sorted_to(objspace, next_length);
}
GC_ASSERT(heap_allocatable_pages(objspace) + heap_eden_total_pages(objspace) <= heap_pages_sorted_length);
GC_ASSERT(heap_allocated_pages <= heap_pages_sorted_length);
}
static void
size_pool_allocatable_pages_set(rb_objspace_t *objspace, rb_size_pool_t *size_pool, size_t s)
{
size_pool->allocatable_pages = s;
heap_pages_expand_sorted(objspace);
}
static inline void
heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
ASSERT_vm_locking();
RVALUE *p = (RVALUE *)obj;
asan_unpoison_object(obj, false);
asan_unlock_freelist(page);
p->as.free.flags = 0;
p->as.free.next = page->freelist;
page->freelist = p;
asan_lock_freelist(page);
if (RGENGC_CHECK_MODE &&
/* obj should belong to page */
!(page->start <= (uintptr_t)obj &&
(uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) &&
obj % BASE_SLOT_SIZE == 0)) {
rb_bug("heap_page_add_freeobj: %p is not rvalue.", (void *)p);
}
asan_poison_object(obj);
gc_report(3, objspace, "heap_page_add_freeobj: add %p to freelist\n", (void *)obj);
}
static inline void
heap_add_freepage(rb_heap_t *heap, struct heap_page *page)
{
asan_unlock_freelist(page);
GC_ASSERT(page->free_slots != 0);
GC_ASSERT(page->freelist != NULL);
page->free_next = heap->free_pages;
heap->free_pages = page;
RUBY_DEBUG_LOG("page:%p freelist:%p", (void *)page, (void *)page->freelist);
asan_lock_freelist(page);
}
#if GC_ENABLE_INCREMENTAL_MARK
static inline void
heap_add_poolpage(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
asan_unlock_freelist(page);
GC_ASSERT(page->free_slots != 0);
GC_ASSERT(page->freelist != NULL);
page->free_next = heap->pooled_pages;
heap->pooled_pages = page;
objspace->rincgc.pooled_slots += page->free_slots;
asan_lock_freelist(page);
}
#endif
static void
heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
ccan_list_del(&page->page_node);
heap->total_pages--;
heap->total_slots -= page->total_slots;
}
static void rb_aligned_free(void *ptr, size_t size);
static void
heap_page_body_free(struct heap_page_body *page_body)
{
GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
if (HEAP_PAGE_ALLOC_USE_MMAP) {
#ifdef HAVE_MMAP
GC_ASSERT(HEAP_PAGE_SIZE % sysconf(_SC_PAGE_SIZE) == 0);
if (munmap(page_body, HEAP_PAGE_SIZE)) {
rb_bug("heap_page_body_free: munmap failed");
}
#endif
}
else {
rb_aligned_free(page_body, HEAP_PAGE_SIZE);
}
}
static void
heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
{
heap_allocated_pages--;
page->size_pool->total_freed_pages++;
heap_page_body_free(GET_PAGE_BODY(page->start));
free(page);
}
static void
heap_pages_free_unused_pages(rb_objspace_t *objspace)
{
size_t i, j;
bool has_pages_in_tomb_heap = FALSE;
for (i = 0; i < SIZE_POOL_COUNT; i++) {
if (!ccan_list_empty(&SIZE_POOL_TOMB_HEAP(&size_pools[i])->pages)) {
has_pages_in_tomb_heap = TRUE;
break;
}
}
if (has_pages_in_tomb_heap) {
for (i = j = 1; j < heap_allocated_pages; i++) {
struct heap_page *page = heap_pages_sorted[i];
if (page->flags.in_tomb && page->free_slots == page->total_slots) {
heap_unlink_page(objspace, SIZE_POOL_TOMB_HEAP(page->size_pool), page);
heap_page_free(objspace, page);
}
else {
if (i != j) {
heap_pages_sorted[j] = page;
}
j++;
}
}
struct heap_page *hipage = heap_pages_sorted[heap_allocated_pages - 1];
uintptr_t himem = (uintptr_t)hipage->start + (hipage->total_slots * hipage->slot_size);
GC_ASSERT(himem <= heap_pages_himem);
heap_pages_himem = himem;
GC_ASSERT(j == heap_allocated_pages);
}
}
static struct heap_page_body *
heap_page_body_allocate(void)
{
struct heap_page_body *page_body;
if (HEAP_PAGE_ALLOC_USE_MMAP) {
#ifdef HAVE_MMAP
GC_ASSERT(HEAP_PAGE_ALIGN % sysconf(_SC_PAGE_SIZE) == 0);
char *ptr = mmap(NULL, HEAP_PAGE_ALIGN + HEAP_PAGE_SIZE,
PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (ptr == MAP_FAILED) {
return NULL;
}
char *aligned = ptr + HEAP_PAGE_ALIGN;
aligned -= ((VALUE)aligned & (HEAP_PAGE_ALIGN - 1));
GC_ASSERT(aligned > ptr);
GC_ASSERT(aligned <= ptr + HEAP_PAGE_ALIGN);
size_t start_out_of_range_size = aligned - ptr;
GC_ASSERT(start_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
if (start_out_of_range_size > 0) {
if (munmap(ptr, start_out_of_range_size)) {
rb_bug("heap_page_body_allocate: munmap failed for start");
}
}
size_t end_out_of_range_size = HEAP_PAGE_ALIGN - start_out_of_range_size;
GC_ASSERT(end_out_of_range_size % sysconf(_SC_PAGE_SIZE) == 0);
if (end_out_of_range_size > 0) {
if (munmap(aligned + HEAP_PAGE_SIZE, end_out_of_range_size)) {
rb_bug("heap_page_body_allocate: munmap failed for end");
}
}
page_body = (struct heap_page_body *)aligned;
#endif
}
else {
page_body = rb_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
}
GC_ASSERT((uintptr_t)page_body % HEAP_PAGE_ALIGN == 0);
return page_body;
}
static struct heap_page *
heap_page_allocate(rb_objspace_t *objspace, rb_size_pool_t *size_pool)
{
uintptr_t start, end, p;
struct heap_page *page;
uintptr_t hi, lo, mid;
size_t stride = size_pool->slot_size;
unsigned int limit = (unsigned int)((HEAP_PAGE_SIZE - sizeof(struct heap_page_header)))/(int)stride;
/* assign heap_page body (contains heap_page_header and RVALUEs) */
struct heap_page_body *page_body = heap_page_body_allocate();
if (page_body == 0) {
rb_memerror();
}
/* assign heap_page entry */
page = calloc1(sizeof(struct heap_page));
if (page == 0) {
heap_page_body_free(page_body);
rb_memerror();
}
/* adjust obj_limit (object number available in this page) */
start = (uintptr_t)((VALUE)page_body + sizeof(struct heap_page_header));
if (start % BASE_SLOT_SIZE != 0) {
int delta = BASE_SLOT_SIZE - (start % BASE_SLOT_SIZE);
start = start + delta;
GC_ASSERT(NUM_IN_PAGE(start) == 0 || NUM_IN_PAGE(start) == 1);
/* Find a num in page that is evenly divisible by `stride`.
* This is to ensure that objects are aligned with bit planes.
* In other words, ensure there are an even number of objects
* per bit plane. */
if (NUM_IN_PAGE(start) == 1) {
start += stride - BASE_SLOT_SIZE;
}
GC_ASSERT(NUM_IN_PAGE(start) * BASE_SLOT_SIZE % stride == 0);
limit = (HEAP_PAGE_SIZE - (int)(start - (uintptr_t)page_body))/(int)stride;
}
end = start + (limit * (int)stride);
/* setup heap_pages_sorted */
lo = 0;
hi = (uintptr_t)heap_allocated_pages;
while (lo < hi) {
struct heap_page *mid_page;
mid = (lo + hi) / 2;
mid_page = heap_pages_sorted[mid];
if ((uintptr_t)mid_page->start < start) {
lo = mid + 1;
}
else if ((uintptr_t)mid_page->start > start) {
hi = mid;
}
else {
rb_bug("same heap page is allocated: %p at %"PRIuVALUE, (void *)page_body, (VALUE)mid);
}
}
if (hi < (uintptr_t)heap_allocated_pages) {
MEMMOVE(&heap_pages_sorted[hi+1], &heap_pages_sorted[hi], struct heap_page_header*, heap_allocated_pages - hi);
}
heap_pages_sorted[hi] = page;
heap_allocated_pages++;
GC_ASSERT(heap_eden_total_pages(objspace) + heap_allocatable_pages(objspace) <= heap_pages_sorted_length);
GC_ASSERT(heap_eden_total_pages(objspace) + heap_tomb_total_pages(objspace) == heap_allocated_pages - 1);
GC_ASSERT(heap_allocated_pages <= heap_pages_sorted_length);
size_pool->total_allocated_pages++;
if (heap_allocated_pages > heap_pages_sorted_length) {
rb_bug("heap_page_allocate: allocated(%"PRIdSIZE") > sorted(%"PRIdSIZE")",
heap_allocated_pages, heap_pages_sorted_length);
}
if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
if (heap_pages_himem < end) heap_pages_himem = end;
page->start = start;
page->total_slots = limit;
page->slot_size = size_pool->slot_size;
page->size_pool = size_pool;
page_body->header.page = page;
for (p = start; p != end; p += stride) {
gc_report(3, objspace, "assign_heap_page: %p is added to freelist\n", (void *)p);
heap_page_add_freeobj(objspace, page, (VALUE)p);
}
page->free_slots = limit;
asan_lock_freelist(page);
return page;
}
static struct heap_page *
heap_page_resurrect(rb_objspace_t *objspace, rb_size_pool_t *size_pool)
{
struct heap_page *page = 0, *next;
ccan_list_for_each_safe(&SIZE_POOL_TOMB_HEAP(size_pool)->pages, page, next, page_node) {
asan_unlock_freelist(page);
if (page->freelist != NULL) {
heap_unlink_page(objspace, &size_pool->tomb_heap, page);
asan_lock_freelist(page);
return page;
}
}
return NULL;
}
static struct heap_page *
heap_page_create(rb_objspace_t *objspace, rb_size_pool_t *size_pool)
{
struct heap_page *page;
const char *method = "recycle";
size_pool->allocatable_pages--;
page = heap_page_resurrect(objspace, size_pool);
if (page == NULL) {
page = heap_page_allocate(objspace, size_pool);
method = "allocate";
}
if (0) fprintf(stderr, "heap_page_create: %s - %p, "
"heap_allocated_pages: %"PRIdSIZE", "
"heap_allocated_pages: %"PRIdSIZE", "
"tomb->total_pages: %"PRIdSIZE"\n",
method, (void *)page, heap_pages_sorted_length, heap_allocated_pages, SIZE_POOL_TOMB_HEAP(size_pool)->total_pages);
return page;
}
static void
heap_add_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap, struct heap_page *page)
{
/* Adding to eden heap during incremental sweeping is forbidden */
GC_ASSERT(!(heap == SIZE_POOL_EDEN_HEAP(size_pool) && heap->sweeping_page));
page->flags.in_tomb = (heap == SIZE_POOL_TOMB_HEAP(size_pool));
ccan_list_add_tail(&heap->pages, &page->page_node);
heap->total_pages++;
heap->total_slots += page->total_slots;
}
static void
heap_assign_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
{
struct heap_page *page = heap_page_create(objspace, size_pool);
heap_add_page(objspace, size_pool, heap, page);
heap_add_freepage(heap, page);
}
static void
heap_add_pages(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap, size_t add)
{
size_t i;
size_pool_allocatable_pages_set(objspace, size_pool, add);
for (i = 0; i < add; i++) {
heap_assign_page(objspace, size_pool, heap);
}
GC_ASSERT(size_pool->allocatable_pages == 0);
}
static size_t
heap_extend_pages(rb_objspace_t *objspace, rb_size_pool_t *size_pool, size_t free_slots, size_t total_slots, size_t used)
{
double goal_ratio = gc_params.heap_free_slots_goal_ratio;
size_t next_used;
if (goal_ratio == 0.0) {
next_used = (size_t)(used * gc_params.growth_factor);
}
else if (total_slots == 0) {
int multiple = size_pool->slot_size / BASE_SLOT_SIZE;
next_used = (gc_params.heap_init_slots * multiple) / HEAP_PAGE_OBJ_LIMIT;
}
else {
/* Find `f' where free_slots = f * total_slots * goal_ratio
* => f = (total_slots - free_slots) / ((1 - goal_ratio) * total_slots)
*/
double f = (double)(total_slots - free_slots) / ((1 - goal_ratio) * total_slots);
if (f > gc_params.growth_factor) f = gc_params.growth_factor;
if (f < 1.0) f = 1.1;
next_used = (size_t)(f * used);
if (0) {
fprintf(stderr,
"free_slots(%8"PRIuSIZE")/total_slots(%8"PRIuSIZE")=%1.2f,"
" G(%1.2f), f(%1.2f),"
" used(%8"PRIuSIZE") => next_used(%8"PRIuSIZE")\n",
free_slots, total_slots, free_slots/(double)total_slots,
goal_ratio, f, used, next_used);
}
}
if (gc_params.growth_max_slots > 0) {
size_t max_used = (size_t)(used + gc_params.growth_max_slots/HEAP_PAGE_OBJ_LIMIT);
if (next_used > max_used) next_used = max_used;
}
size_t extend_page_count = next_used - used;
/* Extend by at least 1 page. */
if (extend_page_count == 0) extend_page_count = 1;
return extend_page_count;
}
static int
heap_increment(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
{
if (size_pool->allocatable_pages > 0) {
gc_report(1, objspace, "heap_increment: heap_pages_sorted_length: %"PRIdSIZE", "
"heap_pages_inc: %"PRIdSIZE", heap->total_pages: %"PRIdSIZE"\n",
heap_pages_sorted_length, size_pool->allocatable_pages, heap->total_pages);
GC_ASSERT(heap_allocatable_pages(objspace) + heap_eden_total_pages(objspace) <= heap_pages_sorted_length);
GC_ASSERT(heap_allocated_pages <= heap_pages_sorted_length);
heap_assign_page(objspace, size_pool, heap);
return TRUE;
}
return FALSE;
}
static void
gc_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
{
/* Continue marking if in incremental marking. */
if (heap->free_pages == NULL && is_incremental_marking(objspace)) {
gc_marks_continue(objspace, size_pool, heap);
}
/* Continue sweeping if in lazy sweeping or the previous incremental
* marking finished and did not yield a free page. */
if (heap->free_pages == NULL && is_lazy_sweeping(objspace)) {
gc_sweep_continue(objspace, size_pool, heap);
}
}
static void
heap_prepare(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
{
GC_ASSERT(heap->free_pages == NULL);
/* Continue incremental marking or lazy sweeping, if in any of those steps. */
gc_continue(objspace, size_pool, heap);
/* If we still don't have a free page and not allowed to create a new page,
* we should start a new GC cycle. */
if (heap->free_pages == NULL &&
(will_be_incremental_marking(objspace) ||
(heap_increment(objspace, size_pool, heap) == FALSE))) {
if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
rb_memerror();
}
else {
/* Do steps of incremental marking or lazy sweeping if the GC run permits. */
gc_continue(objspace, size_pool, heap);
/* If we're not incremental marking (e.g. a minor GC) or finished
* sweeping and still don't have a free page, then
* gc_sweep_finish_size_pool should allow us to create a new page. */
if (heap->free_pages == NULL && !heap_increment(objspace, size_pool, heap)) {
if (objspace->rgengc.need_major_gc == GPR_FLAG_NONE) {
rb_bug("cannot create a new page after GC");
}
else { // Major GC is required, which will allow us to create new page
if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
rb_memerror();
}
else {
/* Do steps of incremental marking or lazy sweeping. */
gc_continue(objspace, size_pool, heap);
if (heap->free_pages == NULL &&
!heap_increment(objspace, size_pool, heap)) {
rb_bug("cannot create a new page after major GC");
}
}
}
}
}
}
GC_ASSERT(heap->free_pages != NULL);
}
void
rb_objspace_set_event_hook(const rb_event_flag_t event)
{
rb_objspace_t *objspace = &rb_objspace;
objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK;
objspace->flags.has_hook = (objspace->hook_events != 0);
}
static void
gc_event_hook_body(rb_execution_context_t *ec, rb_objspace_t *objspace, const rb_event_flag_t event, VALUE data)
{
const VALUE *pc = ec->cfp->pc;
if (pc && VM_FRAME_RUBYFRAME_P(ec->cfp)) {
/* increment PC because source line is calculated with PC-1 */
ec->cfp->pc++;
}
EXEC_EVENT_HOOK(ec, event, ec->cfp->self, 0, 0, 0, data);
ec->cfp->pc = pc;
}
#define gc_event_hook_available_p(objspace) ((objspace)->flags.has_hook)
#define gc_event_hook_needed_p(objspace, event) ((objspace)->hook_events & (event))
#define gc_event_hook_prep(objspace, event, data, prep) do { \
if (UNLIKELY(gc_event_hook_needed_p(objspace, event))) { \
prep; \
gc_event_hook_body(GET_EC(), (objspace), (event), (data)); \
} \
} while (0)
#define gc_event_hook(objspace, event, data) gc_event_hook_prep(objspace, event, data, (void)0)
static inline VALUE
newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, VALUE obj)
{
#if !__has_feature(memory_sanitizer)
GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
#endif
RVALUE *p = RANY(obj);
p->as.basic.flags = flags;
*((VALUE *)&p->as.basic.klass) = klass;
#if RACTOR_CHECK_MODE
rb_ractor_setup_belonging(obj);
#endif
#if RGENGC_CHECK_MODE
p->as.values.v1 = p->as.values.v2 = p->as.values.v3 = 0;
RB_VM_LOCK_ENTER_NO_BARRIER();
{
check_rvalue_consistency(obj);
GC_ASSERT(RVALUE_MARKED(obj) == FALSE);
GC_ASSERT(RVALUE_MARKING(obj) == FALSE);
GC_ASSERT(RVALUE_OLD_P(obj) == FALSE);
GC_ASSERT(RVALUE_WB_UNPROTECTED(obj) == FALSE);
if (flags & FL_PROMOTED1) {
if (RVALUE_AGE(obj) != 2) rb_bug("newobj: %s of age (%d) != 2.", obj_info(obj), RVALUE_AGE(obj));
}
else {
if (RVALUE_AGE(obj) > 0) rb_bug("newobj: %s of age (%d) > 0.", obj_info(obj), RVALUE_AGE(obj));
}
if (rgengc_remembered(objspace, (VALUE)obj)) rb_bug("newobj: %s is remembered.", obj_info(obj));
}
RB_VM_LOCK_LEAVE_NO_BARRIER();
#endif
if (UNLIKELY(wb_protected == FALSE)) {
ASSERT_vm_locking();
MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
}
// TODO: make it atomic, or ractor local
objspace->total_allocated_objects++;
#if RGENGC_PROFILE
if (wb_protected) {
objspace->profile.total_generated_normal_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.generated_normal_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
}
else {
objspace->profile.total_generated_shady_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.generated_shady_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
}
#endif
#if GC_DEBUG
RANY(obj)->file = rb_source_location_cstr(&RANY(obj)->line);
GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */
#endif
gc_report(5, objspace, "newobj: %s\n", obj_info(obj));
#if RGENGC_OLD_NEWOBJ_CHECK > 0
{
static int newobj_cnt = RGENGC_OLD_NEWOBJ_CHECK;
if (!is_incremental_marking(objspace) &&
flags & FL_WB_PROTECTED && /* do not promote WB unprotected objects */
! RB_TYPE_P(obj, T_ARRAY)) { /* array.c assumes that allocated objects are new */
if (--newobj_cnt == 0) {
newobj_cnt = RGENGC_OLD_NEWOBJ_CHECK;
gc_mark_set(objspace, obj);
RVALUE_AGE_SET_OLD(objspace, obj);
rb_gc_writebarrier_remember(obj);
}
}
}
#endif
// RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, obj_type_name(obj));
return obj;
}
size_t
rb_gc_obj_slot_size(VALUE obj)
{
return GET_HEAP_PAGE(obj)->slot_size;
}
static inline size_t
size_pool_slot_size(unsigned char pool_id)
{
GC_ASSERT(pool_id < SIZE_POOL_COUNT);
size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE;
#if RGENGC_CHECK_MODE
rb_objspace_t *objspace = &rb_objspace;
GC_ASSERT(size_pools[pool_id].slot_size == (short)slot_size);
#endif
return slot_size;
}
bool
rb_gc_size_allocatable_p(size_t size)
{
return size <= size_pool_slot_size(SIZE_POOL_COUNT - 1);
}
static inline VALUE
ractor_cache_allocate_slot(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache,
size_t size_pool_idx)
{
rb_ractor_newobj_size_pool_cache_t *size_pool_cache = &cache->size_pool_caches[size_pool_idx];
RVALUE *p = size_pool_cache->freelist;
#if GC_ENABLE_INCREMENTAL_MARK
if (is_incremental_marking(objspace)) {
// Not allowed to allocate without running an incremental marking step
if (cache->incremental_mark_step_allocated_slots >= INCREMENTAL_MARK_STEP_ALLOCATIONS) {
return Qfalse;
}
if (p) {
cache->incremental_mark_step_allocated_slots++;
}
}
#endif
if (p) {
VALUE obj = (VALUE)p;
MAYBE_UNUSED(const size_t) stride = size_pool_slot_size(size_pool_idx);
size_pool_cache->freelist = p->as.free.next;
#if USE_RVARGC
asan_unpoison_memory_region(p, stride, true);
#else
asan_unpoison_object(obj, true);
#endif
#if RGENGC_CHECK_MODE
GC_ASSERT(rb_gc_obj_slot_size(obj) == stride);
// zero clear
MEMZERO((char *)obj, char, stride);
#endif
return obj;
}
else {
return Qfalse;
}
}
static struct heap_page *
heap_next_free_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
{
ASSERT_vm_locking();
struct heap_page *page;
if (heap->free_pages == NULL) {
heap_prepare(objspace, size_pool, heap);
}
page = heap->free_pages;
heap->free_pages = page->free_next;
GC_ASSERT(page->free_slots != 0);
RUBY_DEBUG_LOG("page:%p freelist:%p cnt:%d", (void *)page, (void *)page->freelist, page->free_slots);
asan_unlock_freelist(page);
return page;
}
static inline void
ractor_cache_set_page(rb_ractor_newobj_cache_t *cache, size_t size_pool_idx,
struct heap_page *page)
{
gc_report(3, &rb_objspace, "ractor_set_cache: Using page %p\n", (void *)GET_PAGE_BODY(page->start));
rb_ractor_newobj_size_pool_cache_t *size_pool_cache = &cache->size_pool_caches[size_pool_idx];
GC_ASSERT(size_pool_cache->freelist == NULL);
GC_ASSERT(page->free_slots != 0);
GC_ASSERT(page->freelist != NULL);
size_pool_cache->using_page = page;
size_pool_cache->freelist = page->freelist;
page->free_slots = 0;
page->freelist = NULL;
asan_unpoison_object((VALUE)size_pool_cache->freelist, false);
GC_ASSERT(RB_TYPE_P((VALUE)size_pool_cache->freelist, T_NONE));
asan_poison_object((VALUE)size_pool_cache->freelist);
}
static inline VALUE
newobj_fill(VALUE obj, VALUE v1, VALUE v2, VALUE v3)
{
RVALUE *p = (RVALUE *)obj;
p->as.values.v1 = v1;
p->as.values.v2 = v2;
p->as.values.v3 = v3;
return obj;
}
static inline size_t
size_pool_idx_for_size(size_t size)
{
#if USE_RVARGC
size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE);
/* size_pool_idx is ceil(log2(slot_count)) */
size_t size_pool_idx = 64 - nlz_int64(slot_count - 1);
if (size_pool_idx >= SIZE_POOL_COUNT) {
rb_bug("size_pool_idx_for_size: allocation size too large");
}
#if RGENGC_CHECK_MODE
rb_objspace_t *objspace = &rb_objspace;
GC_ASSERT(size <= (size_t)size_pools[size_pool_idx].slot_size);
if (size_pool_idx > 0) GC_ASSERT(size > (size_t)size_pools[size_pool_idx - 1].slot_size);
#endif
return size_pool_idx;
#else
GC_ASSERT(size <= sizeof(RVALUE));
return 0;
#endif
}
static VALUE
newobj_alloc(rb_objspace_t *objspace, rb_ractor_t *cr, size_t size_pool_idx, bool vm_locked)
{
rb_size_pool_t *size_pool = &size_pools[size_pool_idx];
rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
rb_ractor_newobj_cache_t *cache = &cr->newobj_cache;
VALUE obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx);
if (UNLIKELY(obj == Qfalse)) {
unsigned int lev;
bool unlock_vm = false;
if (!vm_locked) {
RB_VM_LOCK_ENTER_CR_LEV(cr, &lev);
vm_locked = true;
unlock_vm = true;
}
{
ASSERT_vm_locking();
#if GC_ENABLE_INCREMENTAL_MARK
if (is_incremental_marking(objspace)) {
gc_marks_continue(objspace, size_pool, heap);
cache->incremental_mark_step_allocated_slots = 0;
// Retry allocation after resetting incremental_mark_step_allocated_slots
obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx);
}
#endif
if (obj == Qfalse) {
// Get next free page (possibly running GC)
struct heap_page *page = heap_next_free_page(objspace, size_pool, heap);
ractor_cache_set_page(cache, size_pool_idx, page);
// Retry allocation after moving to new page
obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx);
GC_ASSERT(obj != Qfalse);
}
}
if (unlock_vm) {
RB_VM_LOCK_LEAVE_CR_LEV(cr, &lev);
}
}
return obj;
}
ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr, int wb_protected, size_t size_pool_idx));
static inline VALUE
newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr, int wb_protected, size_t size_pool_idx)
{
VALUE obj;
unsigned int lev;
RB_VM_LOCK_ENTER_CR_LEV(cr, &lev);
{
if (UNLIKELY(during_gc || ruby_gc_stressful)) {
if (during_gc) {
dont_gc_on();
during_gc = 0;
rb_bug("object allocation during garbage collection phase");
}
if (ruby_gc_stressful) {
if (!garbage_collect(objspace, GPR_FLAG_NEWOBJ)) {
rb_memerror();
}
}
}
obj = newobj_alloc(objspace, cr, size_pool_idx, true);
newobj_init(klass, flags, wb_protected, objspace, obj);
gc_event_hook_prep(objspace, RUBY_INTERNAL_EVENT_NEWOBJ, obj, newobj_fill(obj, 0, 0, 0));
}
RB_VM_LOCK_LEAVE_CR_LEV(cr, &lev);
return obj;
}
NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags,
rb_objspace_t *objspace, rb_ractor_t *cr, size_t size_pool_idx));
NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags,
rb_objspace_t *objspace, rb_ractor_t *cr, size_t size_pool_idx));
static VALUE
newobj_slowpath_wb_protected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr, size_t size_pool_idx)
{
return newobj_slowpath(klass, flags, objspace, cr, TRUE, size_pool_idx);
}
static VALUE
newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_t *cr, size_t size_pool_idx)
{
return newobj_slowpath(klass, flags, objspace, cr, FALSE, size_pool_idx);
}
static inline VALUE
newobj_of0(VALUE klass, VALUE flags, int wb_protected, rb_ractor_t *cr, size_t alloc_size)
{
VALUE obj;
rb_objspace_t *objspace = &rb_objspace;
RB_DEBUG_COUNTER_INC(obj_newobj);
(void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected);
#if GC_DEBUG_STRESS_TO_CLASS
if (UNLIKELY(stress_to_class)) {
long i, cnt = RARRAY_LEN(stress_to_class);
for (i = 0; i < cnt; ++i) {
if (klass == RARRAY_AREF(stress_to_class, i)) rb_memerror();
}
}
#endif
size_t size_pool_idx = size_pool_idx_for_size(alloc_size);
if (!UNLIKELY(during_gc ||
ruby_gc_stressful ||
gc_event_hook_available_p(objspace)) &&
wb_protected) {
obj = newobj_alloc(objspace, cr, size_pool_idx, false);
newobj_init(klass, flags, wb_protected, objspace, obj);
}
else {
RB_DEBUG_COUNTER_INC(obj_newobj_slowpath);
obj = wb_protected ?
newobj_slowpath_wb_protected(klass, flags, objspace, cr, size_pool_idx) :
newobj_slowpath_wb_unprotected(klass, flags, objspace, cr, size_pool_idx);
}
return obj;
}
static inline VALUE
newobj_of(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected, size_t alloc_size)
{
VALUE obj = newobj_of0(klass, flags, wb_protected, GET_RACTOR(), alloc_size);
return newobj_fill(obj, v1, v2, v3);
}
static inline VALUE
newobj_of_cr(rb_ractor_t *cr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected, size_t alloc_size)
{
VALUE obj = newobj_of0(klass, flags, wb_protected, cr, alloc_size);
return newobj_fill(obj, v1, v2, v3);
}
VALUE
rb_wb_unprotected_newobj_of(VALUE klass, VALUE flags, size_t size)
{
GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
return newobj_of(klass, flags, 0, 0, 0, FALSE, size);
}
VALUE
rb_wb_protected_newobj_of(VALUE klass, VALUE flags, size_t size)
{
GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
return newobj_of(klass, flags, 0, 0, 0, TRUE, size);
}
VALUE
rb_ec_wb_protected_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flags, size_t size)
{
GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
return newobj_of_cr(rb_ec_ractor_ptr(ec), klass, flags, 0, 0, 0, TRUE, size);
}
/* for compatibility */
VALUE
rb_newobj(void)
{
return newobj_of(0, T_NONE, 0, 0, 0, FALSE, sizeof(RVALUE));
}
static size_t
rb_obj_embedded_size(uint32_t numiv)
{
return offsetof(struct RObject, as.ary) + (sizeof(VALUE) * numiv);
}
static VALUE
rb_class_instance_allocate_internal(VALUE klass, VALUE flags, bool wb_protected)
{
GC_ASSERT((flags & RUBY_T_MASK) == T_OBJECT);
GC_ASSERT(flags & ROBJECT_EMBED);
uint32_t index_tbl_num_entries = RCLASS_EXT(klass)->max_iv_count;
size_t size;
bool embed = true;
#if USE_RVARGC
size = rb_obj_embedded_size(index_tbl_num_entries);
if (!rb_gc_size_allocatable_p(size)) {
size = sizeof(struct RObject);
embed = false;
}
#else
size = sizeof(struct RObject);
if (index_tbl_num_entries > ROBJECT_EMBED_LEN_MAX) {
embed = false;
}
#endif
#if USE_RVARGC
VALUE obj = newobj_of(klass, flags, 0, 0, 0, wb_protected, size);
#else
VALUE obj = newobj_of(klass, flags, Qundef, Qundef, Qundef, wb_protected, size);
#endif
if (embed) {
#if USE_RVARGC
uint32_t capa = (uint32_t)((rb_gc_obj_slot_size(obj) - offsetof(struct RObject, as.ary)) / sizeof(VALUE));
GC_ASSERT(capa >= index_tbl_num_entries);
ROBJECT(obj)->numiv = capa;
for (size_t i = 0; i < capa; i++) {
ROBJECT(obj)->as.ary[i] = Qundef;
}
#endif
}
else {
rb_ensure_iv_list_size(obj, 0, index_tbl_num_entries);
}
return obj;
}
VALUE
rb_newobj_of(VALUE klass, VALUE flags)
{
if ((flags & RUBY_T_MASK) == T_OBJECT) {
return rb_class_instance_allocate_internal(klass, (flags | ROBJECT_EMBED) & ~FL_WB_PROTECTED, flags & FL_WB_PROTECTED);
}
else {
return newobj_of(klass, flags & ~FL_WB_PROTECTED, 0, 0, 0, flags & FL_WB_PROTECTED, sizeof(RVALUE));
}
}
#define UNEXPECTED_NODE(func) \
rb_bug(#func"(): GC does not handle T_NODE 0x%x(%p) 0x%"PRIxVALUE, \
BUILTIN_TYPE(obj), (void*)(obj), RBASIC(obj)->flags)
const char *
rb_imemo_name(enum imemo_type type)
{
// put no default case to get a warning if an imemo type is missing
switch (type) {
#define IMEMO_NAME(x) case imemo_##x: return #x;
IMEMO_NAME(env);
IMEMO_NAME(cref);
IMEMO_NAME(svar);
IMEMO_NAME(throw_data);
IMEMO_NAME(ifunc);
IMEMO_NAME(memo);
IMEMO_NAME(ment);
IMEMO_NAME(iseq);
IMEMO_NAME(tmpbuf);
IMEMO_NAME(ast);
IMEMO_NAME(parser_strterm);
IMEMO_NAME(callinfo);
IMEMO_NAME(callcache);
IMEMO_NAME(constcache);
#undef IMEMO_NAME
}
return "unknown";
}
#undef rb_imemo_new
VALUE
rb_imemo_new(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0)
{
size_t size = sizeof(RVALUE);
VALUE flags = T_IMEMO | (type << FL_USHIFT);
return newobj_of(v0, flags, v1, v2, v3, TRUE, size);
}
static VALUE
rb_imemo_tmpbuf_new(VALUE v1, VALUE v2, VALUE v3, VALUE v0)
{
size_t size = sizeof(RVALUE);
VALUE flags = T_IMEMO | (imemo_tmpbuf << FL_USHIFT);
return newobj_of(v0, flags, v1, v2, v3, FALSE, size);
}
static VALUE
rb_imemo_tmpbuf_auto_free_maybe_mark_buffer(void *buf, size_t cnt)
{
return rb_imemo_tmpbuf_new((VALUE)buf, 0, (VALUE)cnt, 0);
}
rb_imemo_tmpbuf_t *
rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt)
{
return (rb_imemo_tmpbuf_t *)rb_imemo_tmpbuf_new((VALUE)buf, (VALUE)old_heap, (VALUE)cnt, 0);
}
static size_t
imemo_memsize(VALUE obj)
{
size_t size = 0;
switch (imemo_type(obj)) {
case imemo_ment:
size += sizeof(RANY(obj)->as.imemo.ment.def);
break;
case imemo_iseq:
size += rb_iseq_memsize((rb_iseq_t *)obj);
break;
case imemo_env:
size += RANY(obj)->as.imemo.env.env_size * sizeof(VALUE);
break;
case imemo_tmpbuf:
size += RANY(obj)->as.imemo.alloc.cnt * sizeof(VALUE);
break;
case imemo_ast:
size += rb_ast_memsize(&RANY(obj)->as.imemo.ast);
break;
case imemo_cref:
case imemo_svar:
case imemo_throw_data:
case imemo_ifunc:
case imemo_memo:
case imemo_parser_strterm:
break;
default:
/* unreachable */
break;
}
return size;
}
#if IMEMO_DEBUG
VALUE
rb_imemo_new_debug(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0, const char *file, int line)
{
VALUE memo = rb_imemo_new(type, v1, v2, v3, v0);
fprintf(stderr, "memo %p (type: %d) @ %s:%d\n", (void *)memo, imemo_type(memo), file, line);
return memo;
}
#endif
VALUE
rb_class_allocate_instance(VALUE klass)
{
return rb_class_instance_allocate_internal(klass, T_OBJECT | ROBJECT_EMBED, RGENGC_WB_PROTECTED_OBJECT);
}
static inline void
rb_data_object_check(VALUE klass)
{
if (klass != rb_cObject && (rb_get_alloc_func(klass) == rb_class_allocate_instance)) {
rb_undef_alloc_func(klass);
rb_warn("undefining the allocator of T_DATA class %"PRIsVALUE, klass);
}
}
VALUE
rb_data_object_wrap(VALUE klass, void *datap, RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree)
{
RUBY_ASSERT_ALWAYS(dfree != (RUBY_DATA_FUNC)1);
if (klass) rb_data_object_check(klass);
return newobj_of(klass, T_DATA, (VALUE)dmark, (VALUE)dfree, (VALUE)datap, FALSE, sizeof(RVALUE));
}
VALUE
rb_data_object_zalloc(VALUE klass, size_t size, RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree)
{
VALUE obj = rb_data_object_wrap(klass, 0, dmark, dfree);
DATA_PTR(obj) = xcalloc(1, size);
return obj;
}
VALUE
rb_data_typed_object_wrap(VALUE klass, void *datap, const rb_data_type_t *type)
{
RBIMPL_NONNULL_ARG(type);
if (klass) rb_data_object_check(klass);
return newobj_of(klass, T_DATA, (VALUE)type, (VALUE)1, (VALUE)datap, type->flags & RUBY_FL_WB_PROTECTED, sizeof(RVALUE));
}
VALUE
rb_data_typed_object_zalloc(VALUE klass, size_t size, const rb_data_type_t *type)
{
VALUE obj = rb_data_typed_object_wrap(klass, 0, type);
DATA_PTR(obj) = xcalloc(1, size);
return obj;
}
size_t
rb_objspace_data_type_memsize(VALUE obj)
{
if (RTYPEDDATA_P(obj)) {
const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
const void *ptr = RTYPEDDATA_DATA(obj);
if (ptr && type->function.dsize) {
return type->function.dsize(ptr);
}
}
return 0;
}
const char *
rb_objspace_data_type_name(VALUE obj)
{
if (RTYPEDDATA_P(obj)) {
return RTYPEDDATA_TYPE(obj)->wrap_struct_name;
}
else {
return 0;
}
}
static int
ptr_in_page_body_p(const void *ptr, const void *memb)
{
struct heap_page *page = *(struct heap_page **)memb;
uintptr_t p_body = (uintptr_t)GET_PAGE_BODY(page->start);
if ((uintptr_t)ptr >= p_body) {
return (uintptr_t)ptr < (p_body + HEAP_PAGE_SIZE) ? 0 : 1;
}
else {
return -1;
}
}
PUREFUNC(static inline struct heap_page * heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr);)
static inline struct heap_page *
heap_page_for_ptr(rb_objspace_t *objspace, uintptr_t ptr)
{
struct heap_page **res;
if (ptr < (uintptr_t)heap_pages_lomem ||
ptr > (uintptr_t)heap_pages_himem) {
return NULL;
}
res = bsearch((void *)ptr, heap_pages_sorted,
(size_t)heap_allocated_pages, sizeof(struct heap_page *),
ptr_in_page_body_p);
if (res) {
return *res;
}
else {
return NULL;
}
}
PUREFUNC(static inline int is_pointer_to_heap(rb_objspace_t *objspace, void *ptr);)
static inline int
is_pointer_to_heap(rb_objspace_t *objspace, void *ptr)
{
register uintptr_t p = (uintptr_t)ptr;
register struct heap_page *page;
RB_DEBUG_COUNTER_INC(gc_isptr_trial);
if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE;
RB_DEBUG_COUNTER_INC(gc_isptr_range);
if (p % BASE_SLOT_SIZE != 0) return FALSE;
RB_DEBUG_COUNTER_INC(gc_isptr_align);
page = heap_page_for_ptr(objspace, (uintptr_t)ptr);
if (page) {
RB_DEBUG_COUNTER_INC(gc_isptr_maybe);
if (page->flags.in_tomb) {
return FALSE;
}
else {
if (p < page->start) return FALSE;
if (p >= page->start + (page->total_slots * page->slot_size)) return FALSE;
if ((NUM_IN_PAGE(p) * BASE_SLOT_SIZE) % page->slot_size != 0) return FALSE;
return TRUE;
}
}
return FALSE;
}
static enum rb_id_table_iterator_result
free_const_entry_i(VALUE value, void *data)
{
rb_const_entry_t *ce = (rb_const_entry_t *)value;
xfree(ce);
return ID_TABLE_CONTINUE;
}
void
rb_free_const_table(struct rb_id_table *tbl)
{
rb_id_table_foreach_values(tbl, free_const_entry_i, 0);
rb_id_table_free(tbl);
}
// alive: if false, target pointers can be freed already.
// To check it, we need objspace parameter.
static void
vm_ccs_free(struct rb_class_cc_entries *ccs, int alive, rb_objspace_t *objspace, VALUE klass)
{
if (ccs->entries) {
for (int i=0; i<ccs->len; i++) {
const struct rb_callcache *cc = ccs->entries[i].cc;
if (!alive) {
void *ptr = asan_unpoison_object_temporary((VALUE)cc);
// ccs can be free'ed.
if (is_pointer_to_heap(objspace, (void *)cc) &&
IMEMO_TYPE_P(cc, imemo_callcache) &&
cc->klass == klass) {
// OK. maybe target cc.
}
else {
if (ptr) {
asan_poison_object((VALUE)cc);
}
continue;
}
if (ptr) {
asan_poison_object((VALUE)cc);
}
}
vm_cc_invalidate(cc);
}
ruby_xfree(ccs->entries);
}
ruby_xfree(ccs);
}
void
rb_vm_ccs_free(struct rb_class_cc_entries *ccs)
{
RB_DEBUG_COUNTER_INC(ccs_free);
vm_ccs_free(ccs, TRUE, NULL, Qundef);
}
struct cc_tbl_i_data {
rb_objspace_t *objspace;
VALUE klass;
bool alive;
};
static enum rb_id_table_iterator_result
cc_table_mark_i(ID id, VALUE ccs_ptr, void *data_ptr)
{
struct cc_tbl_i_data *data = data_ptr;
struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_ptr;
VM_ASSERT(vm_ccs_p(ccs));
VM_ASSERT(id == ccs->cme->called_id);
if (METHOD_ENTRY_INVALIDATED(ccs->cme)) {
rb_vm_ccs_free(ccs);
return ID_TABLE_DELETE;
}
else {
gc_mark(data->objspace, (VALUE)ccs->cme);
for (int i=0; i<ccs->len; i++) {
VM_ASSERT(data->klass == ccs->entries[i].cc->klass);
VM_ASSERT(vm_cc_check_cme(ccs->entries[i].cc, ccs->cme));
gc_mark(data->objspace, (VALUE)ccs->entries[i].ci);
gc_mark(data->objspace, (VALUE)ccs->entries[i].cc);
}
return ID_TABLE_CONTINUE;
}
}
static void
cc_table_mark(rb_objspace_t *objspace, VALUE klass)
{
struct rb_id_table *cc_tbl = RCLASS_CC_TBL(klass);
if (cc_tbl) {
struct cc_tbl_i_data data = {
.objspace = objspace,
.klass = klass,
};
rb_id_table_foreach(cc_tbl, cc_table_mark_i, &data);
}
}
static enum rb_id_table_iterator_result
cc_table_free_i(VALUE ccs_ptr, void *data_ptr)
{
struct cc_tbl_i_data *data = data_ptr;
struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_ptr;
VM_ASSERT(vm_ccs_p(ccs));
vm_ccs_free(ccs, data->alive, data->objspace, data->klass);
return ID_TABLE_CONTINUE;
}
static void
cc_table_free(rb_objspace_t *objspace, VALUE klass, bool alive)
{
struct rb_id_table *cc_tbl = RCLASS_CC_TBL(klass);
if (cc_tbl) {
struct cc_tbl_i_data data = {
.objspace = objspace,
.klass = klass,
.alive = alive,
};
rb_id_table_foreach_values(cc_tbl, cc_table_free_i, &data);
rb_id_table_free(cc_tbl);
}
}
static enum rb_id_table_iterator_result
cvar_table_free_i(VALUE value, void * ctx)
{
xfree((void *) value);
return ID_TABLE_CONTINUE;
}
void
rb_cc_table_free(VALUE klass)
{
cc_table_free(&rb_objspace, klass, TRUE);
}
static inline void
make_zombie(rb_objspace_t *objspace, VALUE obj, void (*dfree)(void *), void *data)
{
struct RZombie *zombie = RZOMBIE(obj);
zombie->basic.flags = T_ZOMBIE | (zombie->basic.flags & FL_SEEN_OBJ_ID);
zombie->dfree = dfree;
zombie->data = data;
VALUE prev, next = heap_pages_deferred_final;
do {
zombie->next = prev = next;
next = RUBY_ATOMIC_VALUE_CAS(heap_pages_deferred_final, prev, obj);
} while (next != prev);
struct heap_page *page = GET_HEAP_PAGE(obj);
page->final_slots++;
heap_pages_final_slots++;
}
static inline void
make_io_zombie(rb_objspace_t *objspace, VALUE obj)
{
rb_io_t *fptr = RANY(obj)->as.file.fptr;
make_zombie(objspace, obj, rb_io_fptr_finalize_internal, fptr);
}
static void
obj_free_object_id(rb_objspace_t *objspace, VALUE obj)
{
ASSERT_vm_locking();
st_data_t o = (st_data_t)obj, id;
GC_ASSERT(FL_TEST(obj, FL_SEEN_OBJ_ID));
FL_UNSET(obj, FL_SEEN_OBJ_ID);
if (st_delete(objspace->obj_to_id_tbl, &o, &id)) {
GC_ASSERT(id);
st_delete(objspace->id_to_obj_tbl, &id, NULL);
}
else {
rb_bug("Object ID seen, but not in mapping table: %s\n", obj_info(obj));
}
}
static int
obj_free(rb_objspace_t *objspace, VALUE obj)
{
RB_DEBUG_COUNTER_INC(obj_free);
// RUBY_DEBUG_LOG("obj:%p (%s)", (void *)obj, obj_type_name(obj));
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_FREEOBJ, obj);
switch (BUILTIN_TYPE(obj)) {
case T_NIL:
case T_FIXNUM:
case T_TRUE:
case T_FALSE:
rb_bug("obj_free() called for broken object");
break;
default:
break;
}
if (FL_TEST(obj, FL_EXIVAR)) {
rb_free_generic_ivar((VALUE)obj);
FL_UNSET(obj, FL_EXIVAR);
}
if (FL_TEST(obj, FL_SEEN_OBJ_ID) && !FL_TEST(obj, FL_FINALIZE)) {
obj_free_object_id(objspace, obj);
}
if (RVALUE_WB_UNPROTECTED(obj)) CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
#if RGENGC_CHECK_MODE
#define CHECK(x) if (x(obj) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", obj_info(obj))
CHECK(RVALUE_WB_UNPROTECTED);
CHECK(RVALUE_MARKED);
CHECK(RVALUE_MARKING);
CHECK(RVALUE_UNCOLLECTIBLE);
#undef CHECK
#endif
switch (BUILTIN_TYPE(obj)) {
case T_OBJECT:
if (RANY(obj)->as.basic.flags & ROBJECT_EMBED) {
RB_DEBUG_COUNTER_INC(obj_obj_embed);
}
else if (ROBJ_TRANSIENT_P(obj)) {
RB_DEBUG_COUNTER_INC(obj_obj_transient);
}
else {
rb_shape_t *shape = rb_shape_get_shape_by_id(ROBJECT_SHAPE_ID(obj));
if (shape) {
VALUE klass = RBASIC_CLASS(obj);
// Increment max_iv_count if applicable, used to determine size pool allocation
uint32_t num_of_ivs = shape->iv_count;
if (RCLASS_EXT(klass)->max_iv_count < num_of_ivs) {
RCLASS_EXT(klass)->max_iv_count = num_of_ivs;
}
}
xfree(RANY(obj)->as.object.as.heap.ivptr);
RB_DEBUG_COUNTER_INC(obj_obj_ptr);
}
break;
case T_MODULE:
case T_CLASS:
rb_id_table_free(RCLASS_M_TBL(obj));
cc_table_free(objspace, obj, FALSE);
if (RCLASS_IV_TBL(obj)) {
st_free_table(RCLASS_IV_TBL(obj));
}
if (RCLASS_CONST_TBL(obj)) {
rb_free_const_table(RCLASS_CONST_TBL(obj));
}
if (RCLASS_CVC_TBL(obj)) {
rb_id_table_foreach_values(RCLASS_CVC_TBL(obj), cvar_table_free_i, NULL);
rb_id_table_free(RCLASS_CVC_TBL(obj));
}
rb_class_remove_subclass_head(obj);
rb_class_remove_from_module_subclasses(obj);
rb_class_remove_from_super_subclasses(obj);
if (FL_TEST_RAW(obj, RCLASS_SUPERCLASSES_INCLUDE_SELF)) {
xfree(RCLASS_SUPERCLASSES(obj));
}
#if SIZEOF_SERIAL_T != SIZEOF_VALUE && USE_RVARGC
xfree(RCLASS(obj)->class_serial_ptr);
#endif
#if !USE_RVARGC
if (RCLASS_EXT(obj))
xfree(RCLASS_EXT(obj));
#endif
(void)RB_DEBUG_COUNTER_INC_IF(obj_module_ptr, BUILTIN_TYPE(obj) == T_MODULE);
(void)RB_DEBUG_COUNTER_INC_IF(obj_class_ptr, BUILTIN_TYPE(obj) == T_CLASS);
break;
case T_STRING:
rb_str_free(obj);
break;
case T_ARRAY:
rb_ary_free(obj);
break;
case T_HASH:
#if USE_DEBUG_COUNTER
switch (RHASH_SIZE(obj)) {
case 0:
RB_DEBUG_COUNTER_INC(obj_hash_empty);
break;
case 1:
RB_DEBUG_COUNTER_INC(obj_hash_1);
break;
case 2:
RB_DEBUG_COUNTER_INC(obj_hash_2);
break;
case 3:
RB_DEBUG_COUNTER_INC(obj_hash_3);
break;
case 4:
RB_DEBUG_COUNTER_INC(obj_hash_4);
break;
case 5:
case 6:
case 7:
case 8:
RB_DEBUG_COUNTER_INC(obj_hash_5_8);
break;
default:
GC_ASSERT(RHASH_SIZE(obj) > 8);
RB_DEBUG_COUNTER_INC(obj_hash_g8);
}
if (RHASH_AR_TABLE_P(obj)) {
if (RHASH_AR_TABLE(obj) == NULL) {
RB_DEBUG_COUNTER_INC(obj_hash_null);
}
else {
RB_DEBUG_COUNTER_INC(obj_hash_ar);
}
}
else {
RB_DEBUG_COUNTER_INC(obj_hash_st);
}
#endif
if (/* RHASH_AR_TABLE_P(obj) */ !FL_TEST_RAW(obj, RHASH_ST_TABLE_FLAG)) {
struct ar_table_struct *tab = RHASH(obj)->as.ar;
if (tab) {
if (RHASH_TRANSIENT_P(obj)) {
RB_DEBUG_COUNTER_INC(obj_hash_transient);
}
else {
ruby_xfree(tab);
}
}
}
else {
GC_ASSERT(RHASH_ST_TABLE_P(obj));
st_free_table(RHASH(obj)->as.st);
}
break;
case T_REGEXP:
if (RANY(obj)->as.regexp.ptr) {
onig_free(RANY(obj)->as.regexp.ptr);
RB_DEBUG_COUNTER_INC(obj_regexp_ptr);
}
break;
case T_DATA:
if (DATA_PTR(obj)) {
int free_immediately = FALSE;
void (*dfree)(void *);
void *data = DATA_PTR(obj);
if (RTYPEDDATA_P(obj)) {
free_immediately = (RANY(obj)->as.typeddata.type->flags & RUBY_TYPED_FREE_IMMEDIATELY) != 0;
dfree = RANY(obj)->as.typeddata.type->function.dfree;
if (0 && free_immediately == 0) {
/* to expose non-free-immediate T_DATA */
fprintf(stderr, "not immediate -> %s\n", RANY(obj)->as.typeddata.type->wrap_struct_name);
}
}
else {
dfree = RANY(obj)->as.data.dfree;
}
if (dfree) {
if (dfree == RUBY_DEFAULT_FREE) {
xfree(data);
RB_DEBUG_COUNTER_INC(obj_data_xfree);
}
else if (free_immediately) {
(*dfree)(data);
RB_DEBUG_COUNTER_INC(obj_data_imm_free);
}
else {
make_zombie(objspace, obj, dfree, data);
RB_DEBUG_COUNTER_INC(obj_data_zombie);
return FALSE;
}
}
else {
RB_DEBUG_COUNTER_INC(obj_data_empty);
}
}
break;
case T_MATCH:
if (RANY(obj)->as.match.rmatch) {
struct rmatch *rm = RANY(obj)->as.match.rmatch;
#if USE_DEBUG_COUNTER
if (rm->regs.num_regs >= 8) {
RB_DEBUG_COUNTER_INC(obj_match_ge8);
}
else if (rm->regs.num_regs >= 4) {
RB_DEBUG_COUNTER_INC(obj_match_ge4);
}
else if (rm->regs.num_regs >= 1) {
RB_DEBUG_COUNTER_INC(obj_match_under4);
}
#endif
onig_region_free(&rm->regs, 0);
if (rm->char_offset)
xfree(rm->char_offset);
xfree(rm);
RB_DEBUG_COUNTER_INC(obj_match_ptr);
}
break;
case T_FILE:
if (RANY(obj)->as.file.fptr) {
make_io_zombie(objspace, obj);
RB_DEBUG_COUNTER_INC(obj_file_ptr);
return FALSE;
}
break;
case T_RATIONAL:
RB_DEBUG_COUNTER_INC(obj_rational);
break;
case T_COMPLEX:
RB_DEBUG_COUNTER_INC(obj_complex);
break;
case T_MOVED:
break;
case T_ICLASS:
/* Basically , T_ICLASS shares table with the module */
if (RICLASS_OWNS_M_TBL_P(obj)) {
/* Method table is not shared for origin iclasses of classes */
rb_id_table_free(RCLASS_M_TBL(obj));
}
if (RCLASS_CALLABLE_M_TBL(obj) != NULL) {
rb_id_table_free(RCLASS_CALLABLE_M_TBL(obj));
}
rb_class_remove_subclass_head(obj);
cc_table_free(objspace, obj, FALSE);
rb_class_remove_from_module_subclasses(obj);
rb_class_remove_from_super_subclasses(obj);
#if !USE_RVARGC
xfree(RCLASS_EXT(obj));
#endif
RB_DEBUG_COUNTER_INC(obj_iclass_ptr);
break;
case T_FLOAT:
RB_DEBUG_COUNTER_INC(obj_float);
break;
case T_BIGNUM:
if (!BIGNUM_EMBED_P(obj) && BIGNUM_DIGITS(obj)) {
xfree(BIGNUM_DIGITS(obj));
RB_DEBUG_COUNTER_INC(obj_bignum_ptr);
}
else {
RB_DEBUG_COUNTER_INC(obj_bignum_embed);
}
break;
case T_NODE:
UNEXPECTED_NODE(obj_free);
break;
case T_STRUCT:
if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) ||
RANY(obj)->as.rstruct.as.heap.ptr == NULL) {
RB_DEBUG_COUNTER_INC(obj_struct_embed);
}
else if (RSTRUCT_TRANSIENT_P(obj)) {
RB_DEBUG_COUNTER_INC(obj_struct_transient);
}
else {
xfree((void *)RANY(obj)->as.rstruct.as.heap.ptr);
RB_DEBUG_COUNTER_INC(obj_struct_ptr);
}
break;
case T_SYMBOL:
{
rb_gc_free_dsymbol(obj);
RB_DEBUG_COUNTER_INC(obj_symbol);
}
break;
case T_IMEMO:
switch (imemo_type(obj)) {
case imemo_ment:
rb_free_method_entry(&RANY(obj)->as.imemo.ment);
RB_DEBUG_COUNTER_INC(obj_imemo_ment);
break;
case imemo_iseq:
rb_iseq_free(&RANY(obj)->as.imemo.iseq);
RB_DEBUG_COUNTER_INC(obj_imemo_iseq);
break;
case imemo_env:
GC_ASSERT(VM_ENV_ESCAPED_P(RANY(obj)->as.imemo.env.ep));
xfree((VALUE *)RANY(obj)->as.imemo.env.env);
RB_DEBUG_COUNTER_INC(obj_imemo_env);
break;
case imemo_tmpbuf:
xfree(RANY(obj)->as.imemo.alloc.ptr);
RB_DEBUG_COUNTER_INC(obj_imemo_tmpbuf);
break;
case imemo_ast:
rb_ast_free(&RANY(obj)->as.imemo.ast);
RB_DEBUG_COUNTER_INC(obj_imemo_ast);
break;
case imemo_cref:
RB_DEBUG_COUNTER_INC(obj_imemo_cref);
break;
case imemo_svar:
RB_DEBUG_COUNTER_INC(obj_imemo_svar);
break;
case imemo_throw_data:
RB_DEBUG_COUNTER_INC(obj_imemo_throw_data);
break;
case imemo_ifunc:
RB_DEBUG_COUNTER_INC(obj_imemo_ifunc);
break;
case imemo_memo:
RB_DEBUG_COUNTER_INC(obj_imemo_memo);
break;
case imemo_parser_strterm:
RB_DEBUG_COUNTER_INC(obj_imemo_parser_strterm);
break;
case imemo_callinfo:
RB_DEBUG_COUNTER_INC(obj_imemo_callinfo);
break;
case imemo_callcache:
RB_DEBUG_COUNTER_INC(obj_imemo_callcache);
break;
case imemo_constcache:
RB_DEBUG_COUNTER_INC(obj_imemo_constcache);
break;
}
return TRUE;
default:
rb_bug("gc_sweep(): unknown data type 0x%x(%p) 0x%"PRIxVALUE,
BUILTIN_TYPE(obj), (void*)obj, RBASIC(obj)->flags);
}
if (FL_TEST(obj, FL_FINALIZE)) {
make_zombie(objspace, obj, 0, 0);
return FALSE;
}
else {
return TRUE;
}
}
#define OBJ_ID_INCREMENT (sizeof(RVALUE) / 2)
#define OBJ_ID_INITIAL (OBJ_ID_INCREMENT * 2)
static int
object_id_cmp(st_data_t x, st_data_t y)
{
if (RB_BIGNUM_TYPE_P(x)) {
return !rb_big_eql(x, y);
}
else {
return x != y;
}
}
static st_index_t
object_id_hash(st_data_t n)
{
if (RB_BIGNUM_TYPE_P(n)) {
return FIX2LONG(rb_big_hash(n));
}
else {
return st_numhash(n);
}
}
static const struct st_hash_type object_id_hash_type = {
object_id_cmp,
object_id_hash,
};
void
Init_heap(void)
{
rb_objspace_t *objspace = &rb_objspace;
#if defined(INIT_HEAP_PAGE_ALLOC_USE_MMAP)
/* Need to determine if we can use mmap at runtime. */
heap_page_alloc_use_mmap = INIT_HEAP_PAGE_ALLOC_USE_MMAP;
#endif
objspace->next_object_id = INT2FIX(OBJ_ID_INITIAL);
objspace->id_to_obj_tbl = st_init_table(&object_id_hash_type);
objspace->obj_to_id_tbl = st_init_numtable();
#if RGENGC_ESTIMATE_OLDMALLOC
objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
#endif
heap_add_pages(objspace, &size_pools[0], SIZE_POOL_EDEN_HEAP(&size_pools[0]), gc_params.heap_init_slots / HEAP_PAGE_OBJ_LIMIT);
/* Give other size pools allocatable pages. */
for (int i = 1; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
int multiple = size_pool->slot_size / BASE_SLOT_SIZE;
size_pool->allocatable_pages = gc_params.heap_init_slots * multiple / HEAP_PAGE_OBJ_LIMIT;
}
heap_pages_expand_sorted(objspace);
init_mark_stack(&objspace->mark_stack);
objspace->profile.invoke_time = getrusage_time();
finalizer_table = st_init_numtable();
}
void
Init_gc_stress(void)
{
rb_objspace_t *objspace = &rb_objspace;
gc_stress_set(objspace, ruby_initial_gc_stress);
}
typedef int each_obj_callback(void *, void *, size_t, void *);
static void objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data, bool protected);
static void objspace_reachable_objects_from_root(rb_objspace_t *, void (func)(const char *, VALUE, void *), void *);
struct each_obj_data {
rb_objspace_t *objspace;
bool reenable_incremental;
each_obj_callback *callback;
void *data;
struct heap_page **pages[SIZE_POOL_COUNT];
size_t pages_counts[SIZE_POOL_COUNT];
};
static VALUE
objspace_each_objects_ensure(VALUE arg)
{
struct each_obj_data *data = (struct each_obj_data *)arg;
rb_objspace_t *objspace = data->objspace;
/* Reenable incremental GC */
if (data->reenable_incremental) {
objspace->flags.dont_incremental = FALSE;
}
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
struct heap_page **pages = data->pages[i];
/* pages could be NULL if an error was raised during setup (e.g.
* malloc failed due to out of memory). */
if (pages) {
free(pages);
}
}
return Qnil;
}
static VALUE
objspace_each_objects_try(VALUE arg)
{
struct each_obj_data *data = (struct each_obj_data *)arg;
rb_objspace_t *objspace = data->objspace;
/* Copy pages from all size_pools to their respective buffers. */
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
size_t size = size_mul_or_raise(SIZE_POOL_EDEN_HEAP(size_pool)->total_pages, sizeof(struct heap_page *), rb_eRuntimeError);
struct heap_page **pages = malloc(size);
if (!pages) rb_memerror();
/* Set up pages buffer by iterating over all pages in the current eden
* heap. This will be a snapshot of the state of the heap before we
* call the callback over each page that exists in this buffer. Thus it
* is safe for the callback to allocate objects without possibly entering
* an infinite loop. */
struct heap_page *page = 0;
size_t pages_count = 0;
ccan_list_for_each(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, page, page_node) {
pages[pages_count] = page;
pages_count++;
}
data->pages[i] = pages;
data->pages_counts[i] = pages_count;
GC_ASSERT(pages_count == SIZE_POOL_EDEN_HEAP(size_pool)->total_pages);
}
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
size_t pages_count = data->pages_counts[i];
struct heap_page **pages = data->pages[i];
struct heap_page *page = ccan_list_top(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, struct heap_page, page_node);
for (size_t i = 0; i < pages_count; i++) {
/* If we have reached the end of the linked list then there are no
* more pages, so break. */
if (page == NULL) break;
/* If this page does not match the one in the buffer, then move to
* the next page in the buffer. */
if (pages[i] != page) continue;
uintptr_t pstart = (uintptr_t)page->start;
uintptr_t pend = pstart + (page->total_slots * size_pool->slot_size);
if (!__asan_region_is_poisoned((void *)pstart, pend - pstart) &&
(*data->callback)((void *)pstart, (void *)pend, size_pool->slot_size, data->data)) {
break;
}
page = ccan_list_next(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, page, page_node);
}
}
return Qnil;
}
/*
* rb_objspace_each_objects() is special C API to walk through
* Ruby object space. This C API is too difficult to use it.
* To be frank, you should not use it. Or you need to read the
* source code of this function and understand what this function does.
*
* 'callback' will be called several times (the number of heap page,
* at current implementation) with:
* vstart: a pointer to the first living object of the heap_page.
* vend: a pointer to next to the valid heap_page area.
* stride: a distance to next VALUE.
*
* If callback() returns non-zero, the iteration will be stopped.
*
* This is a sample callback code to iterate liveness objects:
*
* int
* sample_callback(void *vstart, void *vend, int stride, void *data) {
* VALUE v = (VALUE)vstart;
* for (; v != (VALUE)vend; v += stride) {
* if (RBASIC(v)->flags) { // liveness check
* // do something with live object 'v'
* }
* return 0; // continue to iteration
* }
*
* Note: 'vstart' is not a top of heap_page. This point the first
* living object to grasp at least one object to avoid GC issue.
* This means that you can not walk through all Ruby object page
* including freed object page.
*
* Note: On this implementation, 'stride' is the same as sizeof(RVALUE).
* However, there are possibilities to pass variable values with
* 'stride' with some reasons. You must use stride instead of
* use some constant value in the iteration.
*/
void
rb_objspace_each_objects(each_obj_callback *callback, void *data)
{
objspace_each_objects(&rb_objspace, callback, data, TRUE);
}
static void
objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data, bool protected)
{
/* Disable incremental GC */
bool reenable_incremental = FALSE;
if (protected) {
reenable_incremental = !objspace->flags.dont_incremental;
gc_rest(objspace);
objspace->flags.dont_incremental = TRUE;
}
struct each_obj_data each_obj_data = {
.objspace = objspace,
.reenable_incremental = reenable_incremental,
.callback = callback,
.data = data,
.pages = {NULL},
.pages_counts = {0},
};
rb_ensure(objspace_each_objects_try, (VALUE)&each_obj_data,
objspace_each_objects_ensure, (VALUE)&each_obj_data);
}
void
rb_objspace_each_objects_without_setup(each_obj_callback *callback, void *data)
{
objspace_each_objects(&rb_objspace, callback, data, FALSE);
}
struct os_each_struct {
size_t num;
VALUE of;
};
static int
internal_object_p(VALUE obj)
{
RVALUE *p = (RVALUE *)obj;
void *ptr = asan_unpoison_object_temporary(obj);
bool used_p = p->as.basic.flags;
if (used_p) {
switch (BUILTIN_TYPE(obj)) {
case T_NODE:
UNEXPECTED_NODE(internal_object_p);
break;
case T_NONE:
case T_MOVED:
case T_IMEMO:
case T_ICLASS:
case T_ZOMBIE:
break;
case T_CLASS:
if (!p->as.basic.klass) break;
if (FL_TEST(obj, FL_SINGLETON)) {
return rb_singleton_class_internal_p(obj);
}
return 0;
default:
if (!p->as.basic.klass) break;
return 0;
}
}
if (ptr || ! used_p) {
asan_poison_object(obj);
}
return 1;
}
int
rb_objspace_internal_object_p(VALUE obj)
{
return internal_object_p(obj);
}
static int
os_obj_of_i(void *vstart, void *vend, size_t stride, void *data)
{
struct os_each_struct *oes = (struct os_each_struct *)data;
VALUE v = (VALUE)vstart;
for (; v != (VALUE)vend; v += stride) {
if (!internal_object_p(v)) {
if (!oes->of || rb_obj_is_kind_of(v, oes->of)) {
if (!rb_multi_ractor_p() || rb_ractor_shareable_p(v)) {
rb_yield(v);
oes->num++;
}
}
}
}
return 0;
}
static VALUE
os_obj_of(VALUE of)
{
struct os_each_struct oes;
oes.num = 0;
oes.of = of;
rb_objspace_each_objects(os_obj_of_i, &oes);
return SIZET2NUM(oes.num);
}
/*
* call-seq:
* ObjectSpace.each_object([module]) {|obj| ... } -> integer
* ObjectSpace.each_object([module]) -> an_enumerator
*
* Calls the block once for each living, nonimmediate object in this
* Ruby process. If <i>module</i> is specified, calls the block
* for only those classes or modules that match (or are a subclass of)
* <i>module</i>. Returns the number of objects found. Immediate
* objects (<code>Fixnum</code>s, <code>Symbol</code>s
* <code>true</code>, <code>false</code>, and <code>nil</code>) are
* never returned. In the example below, #each_object returns both
* the numbers we defined and several constants defined in the Math
* module.
*
* If no block is given, an enumerator is returned instead.
*
* a = 102.7
* b = 95 # Won't be returned
* c = 12345678987654321
* count = ObjectSpace.each_object(Numeric) {|x| p x }
* puts "Total count: #{count}"
*
* <em>produces:</em>
*
* 12345678987654321
* 102.7
* 2.71828182845905
* 3.14159265358979
* 2.22044604925031e-16
* 1.7976931348623157e+308
* 2.2250738585072e-308
* Total count: 7
*
*/
static VALUE
os_each_obj(int argc, VALUE *argv, VALUE os)
{
VALUE of;
of = (!rb_check_arity(argc, 0, 1) ? 0 : argv[0]);
RETURN_ENUMERATOR(os, 1, &of);
return os_obj_of(of);
}
/*
* call-seq:
* ObjectSpace.undefine_finalizer(obj)
*
* Removes all finalizers for <i>obj</i>.
*
*/
static VALUE
undefine_final(VALUE os, VALUE obj)
{
return rb_undefine_finalizer(obj);
}
VALUE
rb_undefine_finalizer(VALUE obj)
{
rb_objspace_t *objspace = &rb_objspace;
st_data_t data = obj;
rb_check_frozen(obj);
st_delete(finalizer_table, &data, 0);
FL_UNSET(obj, FL_FINALIZE);
return obj;
}
static void
should_be_callable(VALUE block)
{
if (!rb_obj_respond_to(block, idCall, TRUE)) {
rb_raise(rb_eArgError, "wrong type argument %"PRIsVALUE" (should be callable)",
rb_obj_class(block));
}
}
static void
should_be_finalizable(VALUE obj)
{
if (!FL_ABLE(obj)) {
rb_raise(rb_eArgError, "cannot define finalizer for %s",
rb_obj_classname(obj));
}
rb_check_frozen(obj);
}
/*
* call-seq:
* ObjectSpace.define_finalizer(obj, aProc=proc())
*
* Adds <i>aProc</i> as a finalizer, to be called after <i>obj</i>
* was destroyed. The object ID of the <i>obj</i> will be passed
* as an argument to <i>aProc</i>. If <i>aProc</i> is a lambda or
* method, make sure it can be called with a single argument.
*
* The return value is an array <code>[0, aProc]</code>.
*
* The two recommended patterns are to either create the finaliser proc
* in a non-instance method where it can safely capture the needed state,
* or to use a custom callable object that stores the needed state
* explicitly as instance variables.
*
* class Foo
* def initialize(data_needed_for_finalization)
* ObjectSpace.define_finalizer(self, self.class.create_finalizer(data_needed_for_finalization))
* end
*
* def self.create_finalizer(data_needed_for_finalization)
* proc {
* puts "finalizing #{data_needed_for_finalization}"
* }
* end
* end
*
* class Bar
* class Remover
* def initialize(data_needed_for_finalization)
* @data_needed_for_finalization = data_needed_for_finalization
* end
*
* def call(id)
* puts "finalizing #{@data_needed_for_finalization}"
* end
* end
*
* def initialize(data_needed_for_finalization)
* ObjectSpace.define_finalizer(self, Remover.new(data_needed_for_finalization))
* end
* end
*
* Note that if your finalizer references the object to be
* finalized it will never be run on GC, although it will still be
* run at exit. You will get a warning if you capture the object
* to be finalized as the receiver of the finalizer.
*
* class CapturesSelf
* def initialize(name)
* ObjectSpace.define_finalizer(self, proc {
* # this finalizer will only be run on exit
* puts "finalizing #{name}"
* })
* end
* end
*
* Also note that finalization can be unpredictable and is never guaranteed
* to be run except on exit.
*/
static VALUE
define_final(int argc, VALUE *argv, VALUE os)
{
VALUE obj, block;
rb_scan_args(argc, argv, "11", &obj, &block);
should_be_finalizable(obj);
if (argc == 1) {
block = rb_block_proc();
}
else {
should_be_callable(block);
}
if (rb_callable_receiver(block) == obj) {
rb_warn("finalizer references object to be finalized");
}
return define_final0(obj, block);
}
static VALUE
define_final0(VALUE obj, VALUE block)
{
rb_objspace_t *objspace = &rb_objspace;
VALUE table;
st_data_t data;
RBASIC(obj)->flags |= FL_FINALIZE;
if (st_lookup(finalizer_table, obj, &data)) {
table = (VALUE)data;
/* avoid duplicate block, table is usually small */
{
long len = RARRAY_LEN(table);
long i;
for (i = 0; i < len; i++) {
VALUE recv = RARRAY_AREF(table, i);
if (rb_equal(recv, block)) {
block = recv;
goto end;
}
}
}
rb_ary_push(table, block);
}
else {
table = rb_ary_new3(1, block);
RBASIC_CLEAR_CLASS(table);
st_add_direct(finalizer_table, obj, table);
}
end:
block = rb_ary_new3(2, INT2FIX(0), block);
OBJ_FREEZE(block);
return block;
}
VALUE
rb_define_finalizer(VALUE obj, VALUE block)
{
should_be_finalizable(obj);
should_be_callable(block);
return define_final0(obj, block);
}
void
rb_gc_copy_finalizer(VALUE dest, VALUE obj)
{
rb_objspace_t *objspace = &rb_objspace;
VALUE table;
st_data_t data;
if (!FL_TEST(obj, FL_FINALIZE)) return;
if (st_lookup(finalizer_table, obj, &data)) {
table = (VALUE)data;
st_insert(finalizer_table, dest, table);
}
FL_SET(dest, FL_FINALIZE);
}
static VALUE
run_single_final(VALUE cmd, VALUE objid)
{
return rb_check_funcall(cmd, idCall, 1, &objid);
}
static void
warn_exception_in_finalizer(rb_execution_context_t *ec, VALUE final)
{
if (final != Qundef && !NIL_P(ruby_verbose)) {
VALUE errinfo = ec->errinfo;
rb_warn("Exception in finalizer %+"PRIsVALUE, final);
rb_ec_error_print(ec, errinfo);
}
}
static void
run_finalizer(rb_objspace_t *objspace, VALUE obj, VALUE table)
{
long i;
enum ruby_tag_type state;
volatile struct {
VALUE errinfo;
VALUE objid;
VALUE final;
rb_control_frame_t *cfp;
long finished;
} saved;
rb_execution_context_t * volatile ec = GET_EC();
#define RESTORE_FINALIZER() (\
ec->cfp = saved.cfp, \
ec->errinfo = saved.errinfo)
saved.errinfo = ec->errinfo;
saved.objid = rb_obj_id(obj);
saved.cfp = ec->cfp;
saved.finished = 0;
saved.final = Qundef;
EC_PUSH_TAG(ec);
state = EC_EXEC_TAG();
if (state != TAG_NONE) {
++saved.finished; /* skip failed finalizer */
warn_exception_in_finalizer(ec, ATOMIC_VALUE_EXCHANGE(saved.final, Qundef));
}
for (i = saved.finished;
RESTORE_FINALIZER(), i<RARRAY_LEN(table);
saved.finished = ++i) {
run_single_final(saved.final = RARRAY_AREF(table, i), saved.objid);
}
EC_POP_TAG();
#undef RESTORE_FINALIZER
}
static void
run_final(rb_objspace_t *objspace, VALUE zombie)
{
st_data_t key, table;
if (RZOMBIE(zombie)->dfree) {
RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data);
}
key = (st_data_t)zombie;
if (st_delete(finalizer_table, &key, &table)) {
run_finalizer(objspace, zombie, (VALUE)table);
}
}
static void
finalize_list(rb_objspace_t *objspace, VALUE zombie)
{
while (zombie) {
VALUE next_zombie;
struct heap_page *page;
asan_unpoison_object(zombie, false);
next_zombie = RZOMBIE(zombie)->next;
page = GET_HEAP_PAGE(zombie);
run_final(objspace, zombie);
RB_VM_LOCK_ENTER();
{
GC_ASSERT(BUILTIN_TYPE(zombie) == T_ZOMBIE);
if (FL_TEST(zombie, FL_SEEN_OBJ_ID)) {
obj_free_object_id(objspace, zombie);
}
GC_ASSERT(heap_pages_final_slots > 0);
GC_ASSERT(page->final_slots > 0);
heap_pages_final_slots--;
page->final_slots--;
page->free_slots++;
heap_page_add_freeobj(objspace, page, zombie);
objspace->profile.total_freed_objects++;
}
RB_VM_LOCK_LEAVE();
zombie = next_zombie;
}
}
static void
finalize_deferred_heap_pages(rb_objspace_t *objspace)
{
VALUE zombie;
while ((zombie = ATOMIC_VALUE_EXCHANGE(heap_pages_deferred_final, 0)) != 0) {
finalize_list(objspace, zombie);
}
}
static void
finalize_deferred(rb_objspace_t *objspace)
{
rb_execution_context_t *ec = GET_EC();
ec->interrupt_mask |= PENDING_INTERRUPT_MASK;
finalize_deferred_heap_pages(objspace);
ec->interrupt_mask &= ~PENDING_INTERRUPT_MASK;
}
static void
gc_finalize_deferred(void *dmy)
{
rb_objspace_t *objspace = dmy;
if (ATOMIC_EXCHANGE(finalizing, 1)) return;
finalize_deferred(objspace);
ATOMIC_SET(finalizing, 0);
}
static void
gc_finalize_deferred_register(rb_objspace_t *objspace)
{
if (rb_postponed_job_register_one(0, gc_finalize_deferred, objspace) == 0) {
rb_bug("gc_finalize_deferred_register: can't register finalizer.");
}
}
struct force_finalize_list {
VALUE obj;
VALUE table;
struct force_finalize_list *next;
};
static int
force_chain_object(st_data_t key, st_data_t val, st_data_t arg)
{
struct force_finalize_list **prev = (struct force_finalize_list **)arg;
struct force_finalize_list *curr = ALLOC(struct force_finalize_list);
curr->obj = key;
curr->table = val;
curr->next = *prev;
*prev = curr;
return ST_CONTINUE;
}
bool rb_obj_is_main_ractor(VALUE gv);
void
rb_objspace_call_finalizer(rb_objspace_t *objspace)
{
size_t i;
#if RGENGC_CHECK_MODE >= 2
gc_verify_internal_consistency(objspace);
#endif
gc_rest(objspace);
if (ATOMIC_EXCHANGE(finalizing, 1)) return;
/* run finalizers */
finalize_deferred(objspace);
GC_ASSERT(heap_pages_deferred_final == 0);
gc_rest(objspace);
/* prohibit incremental GC */
objspace->flags.dont_incremental = 1;
/* force to run finalizer */
while (finalizer_table->num_entries) {
struct force_finalize_list *list = 0;
st_foreach(finalizer_table, force_chain_object, (st_data_t)&list);
while (list) {
struct force_finalize_list *curr = list;
st_data_t obj = (st_data_t)curr->obj;
run_finalizer(objspace, curr->obj, curr->table);
st_delete(finalizer_table, &obj, 0);
list = curr->next;
xfree(curr);
}
}
/* prohibit GC because force T_DATA finalizers can break an object graph consistency */
dont_gc_on();
/* running data/file finalizers are part of garbage collection */
unsigned int lock_lev;
gc_enter(objspace, gc_enter_event_finalizer, &lock_lev);
/* run data/file object's finalizers */
for (i = 0; i < heap_allocated_pages; i++) {
struct heap_page *page = heap_pages_sorted[i];
short stride = page->slot_size;
uintptr_t p = (uintptr_t)page->start;
uintptr_t pend = p + page->total_slots * stride;
for (; p < pend; p += stride) {
VALUE vp = (VALUE)p;
void *poisoned = asan_unpoison_object_temporary(vp);
switch (BUILTIN_TYPE(vp)) {
case T_DATA:
if (!DATA_PTR(p) || !RANY(p)->as.data.dfree) break;
if (rb_obj_is_thread(vp)) break;
if (rb_obj_is_mutex(vp)) break;
if (rb_obj_is_fiber(vp)) break;
if (rb_obj_is_main_ractor(vp)) break;
if (RTYPEDDATA_P(vp)) {
RDATA(p)->dfree = RANY(p)->as.typeddata.type->function.dfree;
}
RANY(p)->as.free.flags = 0;
if (RANY(p)->as.data.dfree == RUBY_DEFAULT_FREE) {
xfree(DATA_PTR(p));
}
else if (RANY(p)->as.data.dfree) {
make_zombie(objspace, vp, RANY(p)->as.data.dfree, RANY(p)->as.data.data);
}
break;
case T_FILE:
if (RANY(p)->as.file.fptr) {
make_io_zombie(objspace, vp);
}
break;
default:
break;
}
if (poisoned) {
GC_ASSERT(BUILTIN_TYPE(vp) == T_NONE);
asan_poison_object(vp);
}
}
}
gc_exit(objspace, gc_enter_event_finalizer, &lock_lev);
finalize_deferred_heap_pages(objspace);
st_free_table(finalizer_table);
finalizer_table = 0;
ATOMIC_SET(finalizing, 0);
}
static inline int
is_swept_object(rb_objspace_t *objspace, VALUE ptr)
{
struct heap_page *page = GET_HEAP_PAGE(ptr);
return page->flags.before_sweep ? FALSE : TRUE;
}
/* garbage objects will be collected soon. */
static inline int
is_garbage_object(rb_objspace_t *objspace, VALUE ptr)
{
if (!is_lazy_sweeping(objspace) ||
is_swept_object(objspace, ptr) ||
MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(ptr), ptr)) {
return FALSE;
}
else {
return TRUE;
}
}
static inline int
is_live_object(rb_objspace_t *objspace, VALUE ptr)
{
switch (BUILTIN_TYPE(ptr)) {
case T_NONE:
case T_MOVED:
case T_ZOMBIE:
return FALSE;
default:
break;
}
if (!is_garbage_object(objspace, ptr)) {
return TRUE;
}
else {
return FALSE;
}
}
static inline int
is_markable_object(rb_objspace_t *objspace, VALUE obj)
{
if (rb_special_const_p(obj)) return FALSE; /* special const is not markable */
check_rvalue_consistency(obj);
return TRUE;
}
int
rb_objspace_markable_object_p(VALUE obj)
{
rb_objspace_t *objspace = &rb_objspace;
return is_markable_object(objspace, obj) && is_live_object(objspace, obj);
}
int
rb_objspace_garbage_object_p(VALUE obj)
{
rb_objspace_t *objspace = &rb_objspace;
return is_garbage_object(objspace, obj);
}
static VALUE
id2ref_obj_tbl(rb_objspace_t *objspace, VALUE objid)
{
VALUE orig;
if (st_lookup(objspace->id_to_obj_tbl, objid, &orig)) {
return orig;
}
else {
return Qundef;
}
}
/*
* call-seq:
* ObjectSpace._id2ref(object_id) -> an_object
*
* Converts an object id to a reference to the object. May not be
* called on an object id passed as a parameter to a finalizer.
*
* s = "I am a string" #=> "I am a string"
* r = ObjectSpace._id2ref(s.object_id) #=> "I am a string"
* r == s #=> true
*
* On multi-ractor mode, if the object is not shareable, it raises
* RangeError.
*/
static VALUE
id2ref(VALUE objid)
{
#if SIZEOF_LONG == SIZEOF_VOIDP
#define NUM2PTR(x) NUM2ULONG(x)
#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
#define NUM2PTR(x) NUM2ULL(x)
#endif
rb_objspace_t *objspace = &rb_objspace;
VALUE ptr;
VALUE orig;
void *p0;
objid = rb_to_int(objid);
if (FIXNUM_P(objid) || rb_big_size(objid) <= SIZEOF_VOIDP) {
ptr = NUM2PTR(objid);
if (ptr == Qtrue) return Qtrue;
if (ptr == Qfalse) return Qfalse;
if (NIL_P(ptr)) return Qnil;
if (FIXNUM_P(ptr)) return (VALUE)ptr;
if (FLONUM_P(ptr)) return (VALUE)ptr;
ptr = obj_id_to_ref(objid);
if ((ptr % sizeof(RVALUE)) == (4 << 2)) {
ID symid = ptr / sizeof(RVALUE);
p0 = (void *)ptr;
if (!rb_static_id_valid_p(symid))
rb_raise(rb_eRangeError, "%p is not symbol id value", p0);
return ID2SYM(symid);
}
}
if ((orig = id2ref_obj_tbl(objspace, objid)) != Qundef &&
is_live_object(objspace, orig)) {
if (!rb_multi_ractor_p() || rb_ractor_shareable_p(orig)) {
return orig;
}
else {
rb_raise(rb_eRangeError, "%+"PRIsVALUE" is id of the unshareable object on multi-ractor", rb_int2str(objid, 10));
}
}
if (rb_int_ge(objid, objspace->next_object_id)) {
rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not id value", rb_int2str(objid, 10));
}
else {
rb_raise(rb_eRangeError, "%+"PRIsVALUE" is recycled object", rb_int2str(objid, 10));
}
}
static VALUE
os_id2ref(VALUE os, VALUE objid)
{
return id2ref(objid);
}
static VALUE
rb_find_object_id(VALUE obj, VALUE (*get_heap_object_id)(VALUE))
{
if (STATIC_SYM_P(obj)) {
return (SYM2ID(obj) * sizeof(RVALUE) + (4 << 2)) | FIXNUM_FLAG;
}
else if (FLONUM_P(obj)) {
#if SIZEOF_LONG == SIZEOF_VOIDP
return LONG2NUM((SIGNED_VALUE)obj);
#else
return LL2NUM((SIGNED_VALUE)obj);
#endif
}
else if (SPECIAL_CONST_P(obj)) {
return LONG2NUM((SIGNED_VALUE)obj);
}
return get_heap_object_id(obj);
}
static VALUE
cached_object_id(VALUE obj)
{
VALUE id;
rb_objspace_t *objspace = &rb_objspace;
RB_VM_LOCK_ENTER();
if (st_lookup(objspace->obj_to_id_tbl, (st_data_t)obj, &id)) {
GC_ASSERT(FL_TEST(obj, FL_SEEN_OBJ_ID));
}
else {
GC_ASSERT(!FL_TEST(obj, FL_SEEN_OBJ_ID));
id = objspace->next_object_id;
objspace->next_object_id = rb_int_plus(id, INT2FIX(OBJ_ID_INCREMENT));
VALUE already_disabled = rb_gc_disable_no_rest();
st_insert(objspace->obj_to_id_tbl, (st_data_t)obj, (st_data_t)id);
st_insert(objspace->id_to_obj_tbl, (st_data_t)id, (st_data_t)obj);
if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
FL_SET(obj, FL_SEEN_OBJ_ID);
}
RB_VM_LOCK_LEAVE();
return id;
}
static VALUE
nonspecial_obj_id_(VALUE obj)
{
return nonspecial_obj_id(obj);
}
VALUE
rb_memory_id(VALUE obj)
{
return rb_find_object_id(obj, nonspecial_obj_id_);
}
/*
* Document-method: __id__
* Document-method: object_id
*
* call-seq:
* obj.__id__ -> integer
* obj.object_id -> integer
*
* Returns an integer identifier for +obj+.
*
* The same number will be returned on all calls to +object_id+ for a given
* object, and no two active objects will share an id.
*
* Note: that some objects of builtin classes are reused for optimization.
* This is the case for immediate values and frozen string literals.
*
* BasicObject implements +__id__+, Kernel implements +object_id+.
*
* Immediate values are not passed by reference but are passed by value:
* +nil+, +true+, +false+, Fixnums, Symbols, and some Floats.
*
* Object.new.object_id == Object.new.object_id # => false
* (21 * 2).object_id == (21 * 2).object_id # => true
* "hello".object_id == "hello".object_id # => false
* "hi".freeze.object_id == "hi".freeze.object_id # => true
*/
VALUE
rb_obj_id(VALUE obj)
{
/*
* 32-bit VALUE space
* MSB ------------------------ LSB
* false 00000000000000000000000000000000
* true 00000000000000000000000000000010
* nil 00000000000000000000000000000100
* undef 00000000000000000000000000000110
* symbol ssssssssssssssssssssssss00001110
* object oooooooooooooooooooooooooooooo00 = 0 (mod sizeof(RVALUE))
* fixnum fffffffffffffffffffffffffffffff1
*
* object_id space
* LSB
* false 00000000000000000000000000000000
* true 00000000000000000000000000000010
* nil 00000000000000000000000000000100
* undef 00000000000000000000000000000110
* symbol 000SSSSSSSSSSSSSSSSSSSSSSSSSSS0 S...S % A = 4 (S...S = s...s * A + 4)
* object oooooooooooooooooooooooooooooo0 o...o % A = 0
* fixnum fffffffffffffffffffffffffffffff1 bignum if required
*
* where A = sizeof(RVALUE)/4
*
* sizeof(RVALUE) is
* 20 if 32-bit, double is 4-byte aligned
* 24 if 32-bit, double is 8-byte aligned
* 40 if 64-bit
*/
return rb_find_object_id(obj, cached_object_id);
}
static enum rb_id_table_iterator_result
cc_table_memsize_i(VALUE ccs_ptr, void *data_ptr)
{
size_t *total_size = data_ptr;
struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_ptr;
*total_size += sizeof(*ccs);
*total_size += sizeof(ccs->entries[0]) * ccs->capa;
return ID_TABLE_CONTINUE;
}
static size_t
cc_table_memsize(struct rb_id_table *cc_table)
{
size_t total = rb_id_table_memsize(cc_table);
rb_id_table_foreach_values(cc_table, cc_table_memsize_i, &total);
return total;
}
static size_t
obj_memsize_of(VALUE obj, int use_all_types)
{
size_t size = 0;
if (SPECIAL_CONST_P(obj)) {
return 0;
}
if (FL_TEST(obj, FL_EXIVAR)) {
size += rb_generic_ivar_memsize(obj);
}
switch (BUILTIN_TYPE(obj)) {
case T_OBJECT:
if (!(RBASIC(obj)->flags & ROBJECT_EMBED)) {
size += ROBJECT_NUMIV(obj) * sizeof(VALUE);
}
break;
case T_MODULE:
case T_CLASS:
if (RCLASS_EXT(obj)) {
if (RCLASS_M_TBL(obj)) {
size += rb_id_table_memsize(RCLASS_M_TBL(obj));
}
if (RCLASS_IV_TBL(obj)) {
size += st_memsize(RCLASS_IV_TBL(obj));
}
if (RCLASS_CVC_TBL(obj)) {
size += rb_id_table_memsize(RCLASS_CVC_TBL(obj));
}
if (RCLASS_EXT(obj)->iv_tbl) {
size += st_memsize(RCLASS_EXT(obj)->iv_tbl);
}
if (RCLASS_EXT(obj)->const_tbl) {
size += rb_id_table_memsize(RCLASS_EXT(obj)->const_tbl);
}
if (RCLASS_CC_TBL(obj)) {
size += cc_table_memsize(RCLASS_CC_TBL(obj));
}
if (FL_TEST_RAW(obj, RCLASS_SUPERCLASSES_INCLUDE_SELF)) {
size += (RCLASS_SUPERCLASS_DEPTH(obj) + 1) * sizeof(VALUE);
}
#if !USE_RVARGC
size += sizeof(rb_classext_t);
#endif
}
break;
case T_ICLASS:
if (RICLASS_OWNS_M_TBL_P(obj)) {
if (RCLASS_M_TBL(obj)) {
size += rb_id_table_memsize(RCLASS_M_TBL(obj));
}
}
if (RCLASS_EXT(obj) && RCLASS_CC_TBL(obj)) {
size += cc_table_memsize(RCLASS_CC_TBL(obj));
}
break;
case T_STRING:
size += rb_str_memsize(obj);
break;
case T_ARRAY:
size += rb_ary_memsize(obj);
break;
case T_HASH:
if (RHASH_AR_TABLE_P(obj)) {
if (RHASH_AR_TABLE(obj) != NULL) {
size_t rb_hash_ar_table_size(void);
size += rb_hash_ar_table_size();
}
}
else {
VM_ASSERT(RHASH_ST_TABLE(obj) != NULL);
size += st_memsize(RHASH_ST_TABLE(obj));
}
break;
case T_REGEXP:
if (RREGEXP_PTR(obj)) {
size += onig_memsize(RREGEXP_PTR(obj));
}
break;
case T_DATA:
if (use_all_types) size += rb_objspace_data_type_memsize(obj);
break;
case T_MATCH:
if (RMATCH(obj)->rmatch) {
struct rmatch *rm = RMATCH(obj)->rmatch;
size += onig_region_memsize(&rm->regs);
size += sizeof(struct rmatch_offset) * rm->char_offset_num_allocated;
size += sizeof(struct rmatch);
}
break;
case T_FILE:
if (RFILE(obj)->fptr) {
size += rb_io_memsize(RFILE(obj)->fptr);
}
break;
case T_RATIONAL:
case T_COMPLEX:
break;
case T_IMEMO:
size += imemo_memsize(obj);
break;
case T_FLOAT:
case T_SYMBOL:
break;
case T_BIGNUM:
if (!(RBASIC(obj)->flags & BIGNUM_EMBED_FLAG) && BIGNUM_DIGITS(obj)) {
size += BIGNUM_LEN(obj) * sizeof(BDIGIT);
}
break;
case T_NODE:
UNEXPECTED_NODE(obj_memsize_of);
break;
case T_STRUCT:
if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) == 0 &&
RSTRUCT(obj)->as.heap.ptr) {
size += sizeof(VALUE) * RSTRUCT_LEN(obj);
}
break;
case T_ZOMBIE:
case T_MOVED:
break;
default:
rb_bug("objspace/memsize_of(): unknown data type 0x%x(%p)",
BUILTIN_TYPE(obj), (void*)obj);
}
return size + GET_HEAP_PAGE(obj)->slot_size;
}
size_t
rb_obj_memsize_of(VALUE obj)
{
return obj_memsize_of(obj, TRUE);
}
static int
set_zero(st_data_t key, st_data_t val, st_data_t arg)
{
VALUE k = (VALUE)key;
VALUE hash = (VALUE)arg;
rb_hash_aset(hash, k, INT2FIX(0));
return ST_CONTINUE;
}
static VALUE
type_sym(size_t type)
{
switch (type) {
#define COUNT_TYPE(t) case (t): return ID2SYM(rb_intern(#t)); break;
COUNT_TYPE(T_NONE);
COUNT_TYPE(T_OBJECT);
COUNT_TYPE(T_CLASS);
COUNT_TYPE(T_MODULE);
COUNT_TYPE(T_FLOAT);
COUNT_TYPE(T_STRING);
COUNT_TYPE(T_REGEXP);
COUNT_TYPE(T_ARRAY);
COUNT_TYPE(T_HASH);
COUNT_TYPE(T_STRUCT);
COUNT_TYPE(T_BIGNUM);
COUNT_TYPE(T_FILE);
COUNT_TYPE(T_DATA);
COUNT_TYPE(T_MATCH);
COUNT_TYPE(T_COMPLEX);
COUNT_TYPE(T_RATIONAL);
COUNT_TYPE(T_NIL);
COUNT_TYPE(T_TRUE);
COUNT_TYPE(T_FALSE);
COUNT_TYPE(T_SYMBOL);
COUNT_TYPE(T_FIXNUM);
COUNT_TYPE(T_IMEMO);
COUNT_TYPE(T_UNDEF);
COUNT_TYPE(T_NODE);
COUNT_TYPE(T_ICLASS);
COUNT_TYPE(T_ZOMBIE);
COUNT_TYPE(T_MOVED);
#undef COUNT_TYPE
default: return SIZET2NUM(type); break;
}
}
/*
* call-seq:
* ObjectSpace.count_objects([result_hash]) -> hash
*
* Counts all objects grouped by type.
*
* It returns a hash, such as:
* {
* :TOTAL=>10000,
* :FREE=>3011,
* :T_OBJECT=>6,
* :T_CLASS=>404,
* # ...
* }
*
* The contents of the returned hash are implementation specific.
* It may be changed in future.
*
* The keys starting with +:T_+ means live objects.
* For example, +:T_ARRAY+ is the number of arrays.
* +:FREE+ means object slots which is not used now.
* +:TOTAL+ means sum of above.
*
* If the optional argument +result_hash+ is given,
* it is overwritten and returned. This is intended to avoid probe effect.
*
* h = {}
* ObjectSpace.count_objects(h)
* puts h
* # => { :TOTAL=>10000, :T_CLASS=>158280, :T_MODULE=>20672, :T_STRING=>527249 }
*
* This method is only expected to work on C Ruby.
*
*/
static VALUE
count_objects(int argc, VALUE *argv, VALUE os)
{
rb_objspace_t *objspace = &rb_objspace;
size_t counts[T_MASK+1];
size_t freed = 0;
size_t total = 0;
size_t i;
VALUE hash = Qnil;
if (rb_check_arity(argc, 0, 1) == 1) {
hash = argv[0];
if (!RB_TYPE_P(hash, T_HASH))
rb_raise(rb_eTypeError, "non-hash given");
}
for (i = 0; i <= T_MASK; i++) {
counts[i] = 0;
}
for (i = 0; i < heap_allocated_pages; i++) {
struct heap_page *page = heap_pages_sorted[i];
short stride = page->slot_size;
uintptr_t p = (uintptr_t)page->start;
uintptr_t pend = p + page->total_slots * stride;
for (;p < pend; p += stride) {
VALUE vp = (VALUE)p;
GC_ASSERT((NUM_IN_PAGE(vp) * BASE_SLOT_SIZE) % page->slot_size == 0);
void *poisoned = asan_unpoison_object_temporary(vp);
if (RANY(p)->as.basic.flags) {
counts[BUILTIN_TYPE(vp)]++;
}
else {
freed++;
}
if (poisoned) {
GC_ASSERT(BUILTIN_TYPE(vp) == T_NONE);
asan_poison_object(vp);
}
}
total += page->total_slots;
}
if (NIL_P(hash)) {
hash = rb_hash_new();
}
else if (!RHASH_EMPTY_P(hash)) {
rb_hash_stlike_foreach(hash, set_zero, hash);
}
rb_hash_aset(hash, ID2SYM(rb_intern("TOTAL")), SIZET2NUM(total));
rb_hash_aset(hash, ID2SYM(rb_intern("FREE")), SIZET2NUM(freed));
for (i = 0; i <= T_MASK; i++) {
VALUE type = type_sym(i);
if (counts[i])
rb_hash_aset(hash, type, SIZET2NUM(counts[i]));
}
return hash;
}
/*
------------------------ Garbage Collection ------------------------
*/
/* Sweeping */
static size_t
objspace_available_slots(rb_objspace_t *objspace)
{
size_t total_slots = 0;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
total_slots += SIZE_POOL_EDEN_HEAP(size_pool)->total_slots;
total_slots += SIZE_POOL_TOMB_HEAP(size_pool)->total_slots;
}
return total_slots;
}
static size_t
objspace_live_slots(rb_objspace_t *objspace)
{
return (objspace->total_allocated_objects - objspace->profile.total_freed_objects) - heap_pages_final_slots;
}
static size_t
objspace_free_slots(rb_objspace_t *objspace)
{
return objspace_available_slots(objspace) - objspace_live_slots(objspace) - heap_pages_final_slots;
}
static void
gc_setup_mark_bits(struct heap_page *page)
{
/* copy oldgen bitmap to mark bitmap */
memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_PAGE_BITMAP_SIZE);
}
static int gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj);
static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, size_t slot_size);
#if defined(_WIN32)
enum {HEAP_PAGE_LOCK = PAGE_NOACCESS, HEAP_PAGE_UNLOCK = PAGE_READWRITE};
static BOOL
protect_page_body(struct heap_page_body *body, DWORD protect)
{
DWORD old_protect;
return VirtualProtect(body, HEAP_PAGE_SIZE, protect, &old_protect) != 0;
}
#else
enum {HEAP_PAGE_LOCK = PROT_NONE, HEAP_PAGE_UNLOCK = PROT_READ | PROT_WRITE};
#define protect_page_body(body, protect) !mprotect((body), HEAP_PAGE_SIZE, (protect))
#endif
static void
lock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
{
if (!protect_page_body(body, HEAP_PAGE_LOCK)) {
rb_bug("Couldn't protect page %p, errno: %s", (void *)body, strerror(errno));
}
else {
gc_report(5, objspace, "Protecting page in move %p\n", (void *)body);
}
}
static void
unlock_page_body(rb_objspace_t *objspace, struct heap_page_body *body)
{
if (!protect_page_body(body, HEAP_PAGE_UNLOCK)) {
rb_bug("Couldn't unprotect page %p, errno: %s", (void *)body, strerror(errno));
}
else {
gc_report(5, objspace, "Unprotecting page in move %p\n", (void *)body);
}
}
static bool
try_move(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *free_page, VALUE src)
{
struct heap_page *src_page = GET_HEAP_PAGE(src);
if (!free_page) {
return false;
}
/* We should return true if either src is successfully moved, or src is
* unmoveable. A false return will cause the sweeping cursor to be
* incremented to the next page, and src will attempt to move again */
if (gc_is_moveable_obj(objspace, src)) {
GC_ASSERT(MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(src), src));
VALUE dest = (VALUE)free_page->freelist;
asan_unpoison_object(dest, false);
if (!dest) {
/* if we can't get something from the freelist then the page must be
* full */
return false;
}
free_page->freelist = RANY(dest)->as.free.next;
GC_ASSERT(RB_BUILTIN_TYPE(dest) == T_NONE);
if (src_page->slot_size > free_page->slot_size) {
objspace->rcompactor.moved_down_count_table[BUILTIN_TYPE(src)]++;
}
else if (free_page->slot_size > src_page->slot_size) {
objspace->rcompactor.moved_up_count_table[BUILTIN_TYPE(src)]++;
}
objspace->rcompactor.moved_count_table[BUILTIN_TYPE(src)]++;
objspace->rcompactor.total_moved++;
gc_move(objspace, src, dest, src_page->slot_size, free_page->slot_size);
gc_pin(objspace, src);
free_page->free_slots--;
}
return true;
}
static void
gc_unprotect_pages(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *cursor = heap->compact_cursor;
while (cursor) {
unlock_page_body(objspace, GET_PAGE_BODY(cursor->start));
cursor = ccan_list_next(&heap->pages, cursor, page_node);
}
}
static void gc_update_references(rb_objspace_t * objspace);
static void invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page);
#ifndef GC_CAN_COMPILE_COMPACTION
#if defined(__wasi__) /* WebAssembly doesn't support signals */
# define GC_CAN_COMPILE_COMPACTION 0
#else
# define GC_CAN_COMPILE_COMPACTION 1
#endif
#endif
#if defined(__MINGW32__) || defined(_WIN32)
# define GC_COMPACTION_SUPPORTED 1
#else
/* If not MinGW, Windows, or does not have mmap, we cannot use mprotect for
* the read barrier, so we must disable compaction. */
# define GC_COMPACTION_SUPPORTED (GC_CAN_COMPILE_COMPACTION && HEAP_PAGE_ALLOC_USE_MMAP)
#endif
#if GC_CAN_COMPILE_COMPACTION
static void
read_barrier_handler(uintptr_t original_address)
{
VALUE obj;
rb_objspace_t * objspace = &rb_objspace;
/* Calculate address aligned to slots. */
uintptr_t address = original_address - (original_address % BASE_SLOT_SIZE);
obj = (VALUE)address;
struct heap_page_body *page_body = GET_PAGE_BODY(obj);
/* If the page_body is NULL, then mprotect cannot handle it and will crash
* with "Cannot allocate memory". */
if (page_body == NULL) {
rb_bug("read_barrier_handler: segmentation fault at %p", (void *)original_address);
}
RB_VM_LOCK_ENTER();
{
unlock_page_body(objspace, page_body);
objspace->profile.read_barrier_faults++;
invalidate_moved_page(objspace, GET_HEAP_PAGE(obj));
}
RB_VM_LOCK_LEAVE();
}
#endif
#if !GC_CAN_COMPILE_COMPACTION
static void
uninstall_handlers(void)
{
/* no-op */
}
static void
install_handlers(void)
{
/* no-op */
}
#elif defined(_WIN32)
static LPTOP_LEVEL_EXCEPTION_FILTER old_handler;
typedef void (*signal_handler)(int);
static signal_handler old_sigsegv_handler;
static LONG WINAPI
read_barrier_signal(EXCEPTION_POINTERS * info)
{
/* EXCEPTION_ACCESS_VIOLATION is what's raised by access to protected pages */
if (info->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
/* > The second array element specifies the virtual address of the inaccessible data.
* https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_record
*
* Use this address to invalidate the page */
read_barrier_handler((uintptr_t)info->ExceptionRecord->ExceptionInformation[1]);
return EXCEPTION_CONTINUE_EXECUTION;
}
else {
return EXCEPTION_CONTINUE_SEARCH;
}
}
static void
uninstall_handlers(void)
{
signal(SIGSEGV, old_sigsegv_handler);
SetUnhandledExceptionFilter(old_handler);
}
static void
install_handlers(void)
{
/* Remove SEGV handler so that the Unhandled Exception Filter handles it */
old_sigsegv_handler = signal(SIGSEGV, NULL);
/* Unhandled Exception Filter has access to the violation address similar
* to si_addr from sigaction */
old_handler = SetUnhandledExceptionFilter(read_barrier_signal);
}
#else
static struct sigaction old_sigbus_handler;
static struct sigaction old_sigsegv_handler;
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
static exception_mask_t old_exception_masks[32];
static mach_port_t old_exception_ports[32];
static exception_behavior_t old_exception_behaviors[32];
static thread_state_flavor_t old_exception_flavors[32];
static mach_msg_type_number_t old_exception_count;
static void
disable_mach_bad_access_exc(void)
{
old_exception_count = sizeof(old_exception_masks) / sizeof(old_exception_masks[0]);
task_swap_exception_ports(
mach_task_self(), EXC_MASK_BAD_ACCESS,
MACH_PORT_NULL, EXCEPTION_DEFAULT, 0,
old_exception_masks, &old_exception_count,
old_exception_ports, old_exception_behaviors, old_exception_flavors
);
}
static void
restore_mach_bad_access_exc(void)
{
for (mach_msg_type_number_t i = 0; i < old_exception_count; i++) {
task_set_exception_ports(
mach_task_self(),
old_exception_masks[i], old_exception_ports[i],
old_exception_behaviors[i], old_exception_flavors[i]
);
}
}
#endif
static void
read_barrier_signal(int sig, siginfo_t * info, void * data)
{
// setup SEGV/BUS handlers for errors
struct sigaction prev_sigbus, prev_sigsegv;
sigaction(SIGBUS, &old_sigbus_handler, &prev_sigbus);
sigaction(SIGSEGV, &old_sigsegv_handler, &prev_sigsegv);
// enable SIGBUS/SEGV
sigset_t set, prev_set;
sigemptyset(&set);
sigaddset(&set, SIGBUS);
sigaddset(&set, SIGSEGV);
sigprocmask(SIG_UNBLOCK, &set, &prev_set);
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
disable_mach_bad_access_exc();
#endif
// run handler
read_barrier_handler((uintptr_t)info->si_addr);
// reset SEGV/BUS handlers
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
restore_mach_bad_access_exc();
#endif
sigaction(SIGBUS, &prev_sigbus, NULL);
sigaction(SIGSEGV, &prev_sigsegv, NULL);
sigprocmask(SIG_SETMASK, &prev_set, NULL);
}
static void
uninstall_handlers(void)
{
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
restore_mach_bad_access_exc();
#endif
sigaction(SIGBUS, &old_sigbus_handler, NULL);
sigaction(SIGSEGV, &old_sigsegv_handler, NULL);
}
static void
install_handlers(void)
{
struct sigaction action;
memset(&action, 0, sizeof(struct sigaction));
sigemptyset(&action.sa_mask);
action.sa_sigaction = read_barrier_signal;
action.sa_flags = SA_SIGINFO | SA_ONSTACK;
sigaction(SIGBUS, &action, &old_sigbus_handler);
sigaction(SIGSEGV, &action, &old_sigsegv_handler);
#ifdef HAVE_MACH_TASK_EXCEPTION_PORTS
disable_mach_bad_access_exc();
#endif
}
#endif
static void
revert_stack_objects(VALUE stack_obj, void *ctx)
{
rb_objspace_t * objspace = (rb_objspace_t*)ctx;
if (BUILTIN_TYPE(stack_obj) == T_MOVED) {
/* For now we'll revert the whole page if the object made it to the
* stack. I think we can change this to move just the one object
* back though */
invalidate_moved_page(objspace, GET_HEAP_PAGE(stack_obj));
}
}
static void
revert_machine_stack_references(rb_objspace_t *objspace, VALUE v)
{
if (is_pointer_to_heap(objspace, (void *)v)) {
if (BUILTIN_TYPE(v) == T_MOVED) {
/* For now we'll revert the whole page if the object made it to the
* stack. I think we can change this to move just the one object
* back though */
invalidate_moved_page(objspace, GET_HEAP_PAGE(v));
}
}
}
static void each_machine_stack_value(const rb_execution_context_t *ec, void (*cb)(rb_objspace_t *, VALUE));
static void
check_stack_for_moved(rb_objspace_t *objspace)
{
rb_execution_context_t *ec = GET_EC();
rb_vm_t *vm = rb_ec_vm_ptr(ec);
rb_vm_each_stack_value(vm, revert_stack_objects, (void*)objspace);
each_machine_stack_value(ec, revert_machine_stack_references);
}
static void gc_mode_transition(rb_objspace_t *objspace, enum gc_mode mode);
static void
gc_compact_finish(rb_objspace_t *objspace)
{
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
gc_unprotect_pages(objspace, heap);
}
uninstall_handlers();
/* The mutator is allowed to run during incremental sweeping. T_MOVED
* objects can get pushed on the stack and when the compaction process
* finishes up, it may remove the read barrier before anything has a
* chance to read from the T_MOVED address. To fix this, we scan the stack
* then revert any moved objects that made it to the stack. */
check_stack_for_moved(objspace);
gc_update_references(objspace);
objspace->profile.compact_count++;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
heap->compact_cursor = NULL;
heap->free_pages = NULL;
heap->compact_cursor_index = 0;
}
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->moved_objects = objspace->rcompactor.total_moved - record->moved_objects;
}
objspace->flags.during_compacting = FALSE;
}
struct gc_sweep_context {
struct heap_page *page;
int final_slots;
int freed_slots;
int empty_slots;
};
static inline void
gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct gc_sweep_context *ctx)
{
struct heap_page * sweep_page = ctx->page;
short slot_size = sweep_page->slot_size;
short slot_bits = slot_size / BASE_SLOT_SIZE;
GC_ASSERT(slot_bits > 0);
do {
VALUE vp = (VALUE)p;
GC_ASSERT(vp % BASE_SLOT_SIZE == 0);
asan_unpoison_object(vp, false);
if (bitset & 1) {
switch (BUILTIN_TYPE(vp)) {
default: /* majority case */
gc_report(2, objspace, "page_sweep: free %p\n", (void *)p);
#if RGENGC_CHECK_MODE
if (!is_full_marking(objspace)) {
if (RVALUE_OLD_P(vp)) rb_bug("page_sweep: %p - old while minor GC.", (void *)p);
if (rgengc_remembered_sweep(objspace, vp)) rb_bug("page_sweep: %p - remembered.", (void *)p);
}
#endif
if (obj_free(objspace, vp)) {
// always add free slots back to the swept pages freelist,
// so that if we're comapacting, we can re-use the slots
(void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, BASE_SLOT_SIZE);
heap_page_add_freeobj(objspace, sweep_page, vp);
gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info(vp));
ctx->freed_slots++;
}
else {
ctx->final_slots++;
}
break;
case T_MOVED:
if (objspace->flags.during_compacting) {
/* The sweep cursor shouldn't have made it to any
* T_MOVED slots while the compact flag is enabled.
* The sweep cursor and compact cursor move in
* opposite directions, and when they meet references will
* get updated and "during_compacting" should get disabled */
rb_bug("T_MOVED shouldn't be seen until compaction is finished\n");
}
gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info(vp));
ctx->empty_slots++;
heap_page_add_freeobj(objspace, sweep_page, vp);
break;
case T_ZOMBIE:
/* already counted */
break;
case T_NONE:
ctx->empty_slots++; /* already freed */
break;
}
}
p += slot_size;
bitset >>= slot_bits;
} while (bitset);
}
static inline void
gc_sweep_page(rb_objspace_t *objspace, rb_heap_t *heap, struct gc_sweep_context *ctx)
{
struct heap_page *sweep_page = ctx->page;
GC_ASSERT(SIZE_POOL_EDEN_HEAP(sweep_page->size_pool) == heap);
uintptr_t p;
bits_t *bits, bitset;
gc_report(2, objspace, "page_sweep: start.\n");
#if RGENGC_CHECK_MODE
if (!objspace->flags.immediate_sweep) {
GC_ASSERT(sweep_page->flags.before_sweep == TRUE);
}
#endif
sweep_page->flags.before_sweep = FALSE;
sweep_page->free_slots = 0;
p = (uintptr_t)sweep_page->start;
bits = sweep_page->mark_bits;
int page_rvalue_count = sweep_page->total_slots * (sweep_page->slot_size / BASE_SLOT_SIZE);
int out_of_range_bits = (NUM_IN_PAGE(p) + page_rvalue_count) % BITS_BITLENGTH;
if (out_of_range_bits != 0) { // sizeof(RVALUE) == 64
bits[BITMAP_INDEX(p) + page_rvalue_count / BITS_BITLENGTH] |= ~(((bits_t)1 << out_of_range_bits) - 1);
}
/* The last bitmap plane may not be used if the last plane does not
* have enough space for the slot_size. In that case, the last plane must
* be skipped since none of the bits will be set. */
int bitmap_plane_count = CEILDIV(NUM_IN_PAGE(p) + page_rvalue_count, BITS_BITLENGTH);
GC_ASSERT(bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT - 1 ||
bitmap_plane_count == HEAP_PAGE_BITMAP_LIMIT);
// Skip out of range slots at the head of the page
bitset = ~bits[0];
bitset >>= NUM_IN_PAGE(p);
if (bitset) {
gc_sweep_plane(objspace, heap, p, bitset, ctx);
}
p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
for (int i = 1; i < bitmap_plane_count; i++) {
bitset = ~bits[i];
if (bitset) {
gc_sweep_plane(objspace, heap, p, bitset, ctx);
}
p += BITS_BITLENGTH * BASE_SLOT_SIZE;
}
if (!heap->compact_cursor) {
gc_setup_mark_bits(sweep_page);
}
#if GC_PROFILE_MORE_DETAIL
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->removing_objects += ctx->final_slots + ctx->freed_slots;
record->empty_objects += ctx->empty_slots;
}
#endif
if (0) fprintf(stderr, "gc_sweep_page(%"PRIdSIZE"): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n",
rb_gc_count(),
sweep_page->total_slots,
ctx->freed_slots, ctx->empty_slots, ctx->final_slots);
sweep_page->free_slots += ctx->freed_slots + ctx->empty_slots;
objspace->profile.total_freed_objects += ctx->freed_slots;
if (heap_pages_deferred_final && !finalizing) {
rb_thread_t *th = GET_THREAD();
if (th) {
gc_finalize_deferred_register(objspace);
}
}
#if RGENGC_CHECK_MODE
short freelist_len = 0;
asan_unlock_freelist(sweep_page);
RVALUE *ptr = sweep_page->freelist;
while (ptr) {
freelist_len++;
ptr = ptr->as.free.next;
}
asan_lock_freelist(sweep_page);
if (freelist_len != sweep_page->free_slots) {
rb_bug("inconsistent freelist length: expected %d but was %d", sweep_page->free_slots, freelist_len);
}
#endif
gc_report(2, objspace, "page_sweep: end.\n");
}
#if !USE_RVARGC
/* allocate additional minimum page to work */
static void
gc_heap_prepare_minimum_pages(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
{
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
if (!heap->free_pages && heap_increment(objspace, size_pool, heap) == FALSE) {
/* there is no free after page_sweep() */
size_pool_allocatable_pages_set(objspace, size_pool, 1);
if (!heap_increment(objspace, size_pool, heap)) { /* can't allocate additional free objects */
rb_memerror();
}
}
}
}
#endif
static const char *
gc_mode_name(enum gc_mode mode)
{
switch (mode) {
case gc_mode_none: return "none";
case gc_mode_marking: return "marking";
case gc_mode_sweeping: return "sweeping";
case gc_mode_compacting: return "compacting";
default: rb_bug("gc_mode_name: unknown mode: %d", (int)mode);
}
}
static void
gc_mode_transition(rb_objspace_t *objspace, enum gc_mode mode)
{
#if RGENGC_CHECK_MODE
enum gc_mode prev_mode = gc_mode(objspace);
switch (prev_mode) {
case gc_mode_none: GC_ASSERT(mode == gc_mode_marking); break;
case gc_mode_marking: GC_ASSERT(mode == gc_mode_sweeping); break;
case gc_mode_sweeping: GC_ASSERT(mode == gc_mode_none || mode == gc_mode_compacting); break;
case gc_mode_compacting: GC_ASSERT(mode == gc_mode_none); break;
}
#endif
if (0) fprintf(stderr, "gc_mode_transition: %s->%s\n", gc_mode_name(gc_mode(objspace)), gc_mode_name(mode));
gc_mode_set(objspace, mode);
}
static void
heap_page_freelist_append(struct heap_page *page, RVALUE *freelist)
{
if (freelist) {
asan_unlock_freelist(page);
if (page->freelist) {
RVALUE *p = page->freelist;
asan_unpoison_object((VALUE)p, false);
while (p->as.free.next) {
RVALUE *prev = p;
p = p->as.free.next;
asan_poison_object((VALUE)prev);
asan_unpoison_object((VALUE)p, false);
}
p->as.free.next = freelist;
asan_poison_object((VALUE)p);
}
else {
page->freelist = freelist;
}
asan_lock_freelist(page);
}
}
static void
gc_sweep_start_heap(rb_objspace_t *objspace, rb_heap_t *heap)
{
heap->sweeping_page = ccan_list_top(&heap->pages, struct heap_page, page_node);
heap->free_pages = NULL;
#if GC_ENABLE_INCREMENTAL_MARK
heap->pooled_pages = NULL;
#endif
if (!objspace->flags.immediate_sweep) {
struct heap_page *page = NULL;
ccan_list_for_each(&heap->pages, page, page_node) {
page->flags.before_sweep = TRUE;
}
}
}
#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 4
__attribute__((noinline))
#endif
static void
gc_sweep_start(rb_objspace_t *objspace)
{
gc_mode_transition(objspace, gc_mode_sweeping);
#if GC_ENABLE_INCREMENTAL_MARK
objspace->rincgc.pooled_slots = 0;
#endif
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
gc_sweep_start_heap(objspace, heap);
#if USE_RVARGC
/* We should call gc_sweep_finish_size_pool for size pools with no pages. */
if (heap->sweeping_page == NULL) {
GC_ASSERT(heap->total_pages == 0);
GC_ASSERT(heap->total_slots == 0);
gc_sweep_finish_size_pool(objspace, size_pool);
}
#endif
}
rb_ractor_t *r = NULL;
ccan_list_for_each(&GET_VM()->ractor.set, r, vmlr_node) {
rb_gc_ractor_newobj_cache_clear(&r->newobj_cache);
}
}
#if USE_RVARGC
static void
gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool)
{
rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
size_t total_slots = heap->total_slots + SIZE_POOL_TOMB_HEAP(size_pool)->total_slots;
size_t total_pages = heap->total_pages + SIZE_POOL_TOMB_HEAP(size_pool)->total_pages;
size_t swept_slots = size_pool->freed_slots + size_pool->empty_slots;
size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio);
/* Some size pools may have very few pages (or even no pages). These size pools
* should still have allocatable pages. */
if (min_free_slots < gc_params.heap_init_slots) {
min_free_slots = gc_params.heap_init_slots;
}
/* If we don't have enough slots and we have pages on the tomb heap, move
* pages from the tomb heap to the eden heap. This may prevent page
* creation thrashing (frequently allocating and deallocting pages) and
* GC thrashing (running GC more frequently than required). */
struct heap_page *resurrected_page;
while (swept_slots < min_free_slots &&
(resurrected_page = heap_page_resurrect(objspace, size_pool))) {
swept_slots += resurrected_page->free_slots;
heap_add_page(objspace, size_pool, heap, resurrected_page);
heap_add_freepage(heap, resurrected_page);
}
if (swept_slots < min_free_slots) {
bool grow_heap = is_full_marking(objspace);
if (!is_full_marking(objspace)) {
/* The heap is a growth heap if it freed more slots than had empty
* slots and used up all of its allocatable pages. */
bool is_growth_heap = (size_pool->empty_slots == 0 ||
size_pool->freed_slots > size_pool->empty_slots) &&
size_pool->allocatable_pages == 0;
if (objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
grow_heap = TRUE;
}
else if (is_growth_heap) { /* Only growth heaps are allowed to start a major GC. */
objspace->rgengc.need_major_gc |= GPR_FLAG_MAJOR_BY_NOFREE;
size_pool->force_major_gc_count++;
}
}
if (grow_heap) {
size_t extend_page_count = heap_extend_pages(objspace, size_pool, swept_slots, total_slots, total_pages);
if (extend_page_count > size_pool->allocatable_pages) {
size_pool_allocatable_pages_set(objspace, size_pool, extend_page_count);
}
}
}
}
#endif
static void
gc_sweep_finish(rb_objspace_t *objspace)
{
gc_report(1, objspace, "gc_sweep_finish\n");
gc_prof_set_heap_info(objspace);
heap_pages_free_unused_pages(objspace);
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
/* if heap_pages has unused pages, then assign them to increment */
size_t tomb_pages = SIZE_POOL_TOMB_HEAP(size_pool)->total_pages;
if (size_pool->allocatable_pages < tomb_pages) {
size_pool->allocatable_pages = tomb_pages;
}
#if USE_RVARGC
size_pool->freed_slots = 0;
size_pool->empty_slots = 0;
#if GC_ENABLE_INCREMENTAL_MARK
if (!will_be_incremental_marking(objspace)) {
rb_heap_t *eden_heap = SIZE_POOL_EDEN_HEAP(size_pool);
struct heap_page *end_page = eden_heap->free_pages;
if (end_page) {
while (end_page->free_next) end_page = end_page->free_next;
end_page->free_next = eden_heap->pooled_pages;
}
else {
eden_heap->free_pages = eden_heap->pooled_pages;
}
eden_heap->pooled_pages = NULL;
objspace->rincgc.pooled_slots = 0;
}
#endif
#endif
}
heap_pages_expand_sorted(objspace);
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_SWEEP, 0);
gc_mode_transition(objspace, gc_mode_none);
}
static int
gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
{
struct heap_page *sweep_page = heap->sweeping_page;
int unlink_limit = GC_SWEEP_PAGES_FREEABLE_PER_STEP;
#if GC_ENABLE_INCREMENTAL_MARK
int swept_slots = 0;
#if USE_RVARGC
bool need_pool = TRUE;
#else
int need_pool = will_be_incremental_marking(objspace) ? TRUE : FALSE;
#endif
gc_report(2, objspace, "gc_sweep_step (need_pool: %d)\n", need_pool);
#else
gc_report(2, objspace, "gc_sweep_step\n");
#endif
if (sweep_page == NULL) return FALSE;
#if GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_start(objspace);
#endif
do {
RUBY_DEBUG_LOG("sweep_page:%p", (void *)sweep_page);
struct gc_sweep_context ctx = {
.page = sweep_page,
.final_slots = 0,
.freed_slots = 0,
.empty_slots = 0,
};
gc_sweep_page(objspace, heap, &ctx);
int free_slots = ctx.freed_slots + ctx.empty_slots;
heap->sweeping_page = ccan_list_next(&heap->pages, sweep_page, page_node);
if (sweep_page->final_slots + free_slots == sweep_page->total_slots &&
heap_pages_freeable_pages > 0 &&
unlink_limit > 0) {
heap_pages_freeable_pages--;
unlink_limit--;
/* there are no living objects -> move this page to tomb heap */
heap_unlink_page(objspace, heap, sweep_page);
heap_add_page(objspace, size_pool, SIZE_POOL_TOMB_HEAP(size_pool), sweep_page);
}
else if (free_slots > 0) {
#if USE_RVARGC
size_pool->freed_slots += ctx.freed_slots;
size_pool->empty_slots += ctx.empty_slots;
#endif
#if GC_ENABLE_INCREMENTAL_MARK
if (need_pool) {
heap_add_poolpage(objspace, heap, sweep_page);
need_pool = FALSE;
}
else {
heap_add_freepage(heap, sweep_page);
swept_slots += free_slots;
if (swept_slots > GC_INCREMENTAL_SWEEP_SLOT_COUNT) {
break;
}
}
#else
heap_add_freepage(heap, sweep_page);
break;
#endif
}
else {
sweep_page->free_next = NULL;
}
} while ((sweep_page = heap->sweeping_page));
if (!heap->sweeping_page) {
#if USE_RVARGC
gc_sweep_finish_size_pool(objspace, size_pool);
#endif
if (!has_sweeping_pages(objspace)) {
gc_sweep_finish(objspace);
}
}
#if GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_stop(objspace);
#endif
return heap->free_pages != NULL;
}
static void
gc_sweep_rest(rb_objspace_t *objspace)
{
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
while (SIZE_POOL_EDEN_HEAP(size_pool)->sweeping_page) {
gc_sweep_step(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool));
}
}
}
static void
gc_sweep_continue(rb_objspace_t *objspace, rb_size_pool_t *sweep_size_pool, rb_heap_t *heap)
{
GC_ASSERT(dont_gc_val() == FALSE);
if (!GC_ENABLE_LAZY_SWEEP) return;
unsigned int lock_lev;
gc_enter(objspace, gc_enter_event_sweep_continue, &lock_lev);
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
if (!gc_sweep_step(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool))) {
#if USE_RVARGC
/* sweep_size_pool requires a free slot but sweeping did not yield any. */
if (size_pool == sweep_size_pool) {
if (size_pool->allocatable_pages > 0) {
heap_increment(objspace, size_pool, heap);
}
else {
/* Not allowed to create a new page so finish sweeping. */
gc_sweep_rest(objspace);
break;
}
}
#endif
}
}
gc_exit(objspace, gc_enter_event_sweep_continue, &lock_lev);
}
static void
invalidate_moved_plane(rb_objspace_t *objspace, struct heap_page *page, uintptr_t p, bits_t bitset)
{
if (bitset) {
do {
if (bitset & 1) {
VALUE forwarding_object = (VALUE)p;
VALUE object;
if (BUILTIN_TYPE(forwarding_object) == T_MOVED) {
GC_ASSERT(MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object));
GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(forwarding_object), forwarding_object));
CLEAR_IN_BITMAP(GET_HEAP_PINNED_BITS(forwarding_object), forwarding_object);
object = rb_gc_location(forwarding_object);
gc_move(objspace, object, forwarding_object, GET_HEAP_PAGE(object)->slot_size, page->slot_size);
/* forwarding_object is now our actual object, and "object"
* is the free slot for the original page */
struct heap_page *orig_page = GET_HEAP_PAGE(object);
orig_page->free_slots++;
heap_page_add_freeobj(objspace, orig_page, object);
GC_ASSERT(MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(forwarding_object), forwarding_object));
GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_MOVED);
GC_ASSERT(BUILTIN_TYPE(forwarding_object) != T_NONE);
}
}
p += BASE_SLOT_SIZE;
bitset >>= 1;
} while (bitset);
}
}
static void
invalidate_moved_page(rb_objspace_t *objspace, struct heap_page *page)
{
int i;
bits_t *mark_bits, *pin_bits;
bits_t bitset;
mark_bits = page->mark_bits;
pin_bits = page->pinned_bits;
uintptr_t p = page->start;
// Skip out of range slots at the head of the page
bitset = pin_bits[0] & ~mark_bits[0];
bitset >>= NUM_IN_PAGE(p);
invalidate_moved_plane(objspace, page, p, bitset);
p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
for (i=1; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
/* Moved objects are pinned but never marked. We reuse the pin bits
* to indicate there is a moved object in this slot. */
bitset = pin_bits[i] & ~mark_bits[i];
invalidate_moved_plane(objspace, page, p, bitset);
p += BITS_BITLENGTH * BASE_SLOT_SIZE;
}
}
static void
gc_compact_start(rb_objspace_t *objspace)
{
struct heap_page *page = NULL;
gc_mode_transition(objspace, gc_mode_compacting);
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(&size_pools[i]);
ccan_list_for_each(&heap->pages, page, page_node) {
page->flags.before_sweep = TRUE;
}
heap->compact_cursor = ccan_list_tail(&heap->pages, struct heap_page, page_node);
heap->compact_cursor_index = 0;
}
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->moved_objects = objspace->rcompactor.total_moved;
}
memset(objspace->rcompactor.considered_count_table, 0, T_MASK * sizeof(size_t));
memset(objspace->rcompactor.moved_count_table, 0, T_MASK * sizeof(size_t));
memset(objspace->rcompactor.moved_up_count_table, 0, T_MASK * sizeof(size_t));
memset(objspace->rcompactor.moved_down_count_table, 0, T_MASK * sizeof(size_t));
/* Set up read barrier for pages containing MOVED objects */
install_handlers();
}
static void gc_sweep_compact(rb_objspace_t *objspace);
static void
gc_sweep(rb_objspace_t *objspace)
{
const unsigned int immediate_sweep = objspace->flags.immediate_sweep;
gc_report(1, objspace, "gc_sweep: immediate: %d\n", immediate_sweep);
gc_sweep_start(objspace);
if (objspace->flags.during_compacting) {
gc_sweep_compact(objspace);
}
if (immediate_sweep) {
#if !GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_start(objspace);
#endif
gc_sweep_rest(objspace);
#if !GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_stop(objspace);
#endif
}
else {
/* Sweep every size pool. */
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
gc_sweep_step(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool));
}
}
#if !USE_RVARGC
rb_size_pool_t *size_pool = &size_pools[0];
gc_heap_prepare_minimum_pages(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool));
#endif
}
/* Marking - Marking stack */
static stack_chunk_t *
stack_chunk_alloc(void)
{
stack_chunk_t *res;
res = malloc(sizeof(stack_chunk_t));
if (!res)
rb_memerror();
return res;
}
static inline int
is_mark_stack_empty(mark_stack_t *stack)
{
return stack->chunk == NULL;
}
static size_t
mark_stack_size(mark_stack_t *stack)
{
size_t size = stack->index;
stack_chunk_t *chunk = stack->chunk ? stack->chunk->next : NULL;
while (chunk) {
size += stack->limit;
chunk = chunk->next;
}
return size;
}
static void
add_stack_chunk_cache(mark_stack_t *stack, stack_chunk_t *chunk)
{
chunk->next = stack->cache;
stack->cache = chunk;
stack->cache_size++;
}
static void
shrink_stack_chunk_cache(mark_stack_t *stack)
{
stack_chunk_t *chunk;
if (stack->unused_cache_size > (stack->cache_size/2)) {
chunk = stack->cache;
stack->cache = stack->cache->next;
stack->cache_size--;
free(chunk);
}
stack->unused_cache_size = stack->cache_size;
}
static void
push_mark_stack_chunk(mark_stack_t *stack)
{
stack_chunk_t *next;
GC_ASSERT(stack->index == stack->limit);
if (stack->cache_size > 0) {
next = stack->cache;
stack->cache = stack->cache->next;
stack->cache_size--;
if (stack->unused_cache_size > stack->cache_size)
stack->unused_cache_size = stack->cache_size;
}
else {
next = stack_chunk_alloc();
}
next->next = stack->chunk;
stack->chunk = next;
stack->index = 0;
}
static void
pop_mark_stack_chunk(mark_stack_t *stack)
{
stack_chunk_t *prev;
prev = stack->chunk->next;
GC_ASSERT(stack->index == 0);
add_stack_chunk_cache(stack, stack->chunk);
stack->chunk = prev;
stack->index = stack->limit;
}
static void
mark_stack_chunk_list_free(stack_chunk_t *chunk)
{
stack_chunk_t *next = NULL;
while (chunk != NULL) {
next = chunk->next;
free(chunk);
chunk = next;
}
}
static void
free_stack_chunks(mark_stack_t *stack)
{
mark_stack_chunk_list_free(stack->chunk);
}
static void
mark_stack_free_cache(mark_stack_t *stack)
{
mark_stack_chunk_list_free(stack->cache);
stack->cache_size = 0;
stack->unused_cache_size = 0;
}
static void
push_mark_stack(mark_stack_t *stack, VALUE data)
{
VALUE obj = data;
switch (BUILTIN_TYPE(obj)) {
case T_OBJECT:
case T_CLASS:
case T_MODULE:
case T_FLOAT:
case T_STRING:
case T_REGEXP:
case T_ARRAY:
case T_HASH:
case T_STRUCT:
case T_BIGNUM:
case T_FILE:
case T_DATA:
case T_MATCH:
case T_COMPLEX:
case T_RATIONAL:
case T_TRUE:
case T_FALSE:
case T_SYMBOL:
case T_IMEMO:
case T_ICLASS:
if (stack->index == stack->limit) {
push_mark_stack_chunk(stack);
}
stack->chunk->data[stack->index++] = data;
return;
case T_NONE:
case T_NIL:
case T_FIXNUM:
case T_MOVED:
case T_ZOMBIE:
case T_UNDEF:
case T_MASK:
rb_bug("push_mark_stack() called for broken object");
break;
case T_NODE:
UNEXPECTED_NODE(push_mark_stack);
break;
}
rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s",
BUILTIN_TYPE(obj), (void *)data,
is_pointer_to_heap(&rb_objspace, (void *)data) ? "corrupted object" : "non object");
}
static int
pop_mark_stack(mark_stack_t *stack, VALUE *data)
{
if (is_mark_stack_empty(stack)) {
return FALSE;
}
if (stack->index == 1) {
*data = stack->chunk->data[--stack->index];
pop_mark_stack_chunk(stack);
}
else {
*data = stack->chunk->data[--stack->index];
}
return TRUE;
}
static void
init_mark_stack(mark_stack_t *stack)
{
int i;
MEMZERO(stack, mark_stack_t, 1);
stack->index = stack->limit = STACK_CHUNK_SIZE;
for (i=0; i < 4; i++) {
add_stack_chunk_cache(stack, stack_chunk_alloc());
}
stack->unused_cache_size = stack->cache_size;
}
/* Marking */
#define SET_STACK_END SET_MACHINE_STACK_END(&ec->machine.stack_end)
#define STACK_START (ec->machine.stack_start)
#define STACK_END (ec->machine.stack_end)
#define STACK_LEVEL_MAX (ec->machine.stack_maxsize/sizeof(VALUE))
#if STACK_GROW_DIRECTION < 0
# define STACK_LENGTH (size_t)(STACK_START - STACK_END)
#elif STACK_GROW_DIRECTION > 0
# define STACK_LENGTH (size_t)(STACK_END - STACK_START + 1)
#else
# define STACK_LENGTH ((STACK_END < STACK_START) ? (size_t)(STACK_START - STACK_END) \
: (size_t)(STACK_END - STACK_START + 1))
#endif
#if !STACK_GROW_DIRECTION
int ruby_stack_grow_direction;
int
ruby_get_stack_grow_direction(volatile VALUE *addr)
{
VALUE *end;
SET_MACHINE_STACK_END(&end);
if (end > addr) return ruby_stack_grow_direction = 1;
return ruby_stack_grow_direction = -1;
}
#endif
size_t
ruby_stack_length(VALUE **p)
{
rb_execution_context_t *ec = GET_EC();
SET_STACK_END;
if (p) *p = STACK_UPPER(STACK_END, STACK_START, STACK_END);
return STACK_LENGTH;
}
#define PREVENT_STACK_OVERFLOW 1
#ifndef PREVENT_STACK_OVERFLOW
#if !(defined(POSIX_SIGNAL) && defined(SIGSEGV) && defined(HAVE_SIGALTSTACK))
# define PREVENT_STACK_OVERFLOW 1
#else
# define PREVENT_STACK_OVERFLOW 0
#endif
#endif
#if PREVENT_STACK_OVERFLOW && !defined(__EMSCRIPTEN__)
static int
stack_check(rb_execution_context_t *ec, int water_mark)
{
SET_STACK_END;
size_t length = STACK_LENGTH;
size_t maximum_length = STACK_LEVEL_MAX - water_mark;
return length > maximum_length;
}
#else
#define stack_check(ec, water_mark) FALSE
#endif
#define STACKFRAME_FOR_CALL_CFUNC 2048
MJIT_FUNC_EXPORTED int
rb_ec_stack_check(rb_execution_context_t *ec)
{
return stack_check(ec, STACKFRAME_FOR_CALL_CFUNC);
}
int
ruby_stack_check(void)
{
return stack_check(GET_EC(), STACKFRAME_FOR_CALL_CFUNC);
}
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void each_location(rb_objspace_t *objspace, register const VALUE *x, register long n, void (*cb)(rb_objspace_t *, VALUE)));
static void
each_location(rb_objspace_t *objspace, register const VALUE *x, register long n, void (*cb)(rb_objspace_t *, VALUE))
{
VALUE v;
while (n--) {
v = *x;
cb(objspace, v);
x++;
}
}
static void
gc_mark_locations(rb_objspace_t *objspace, const VALUE *start, const VALUE *end, void (*cb)(rb_objspace_t *, VALUE))
{
long n;
if (end <= start) return;
n = end - start;
each_location(objspace, start, n, cb);
}
void
rb_gc_mark_locations(const VALUE *start, const VALUE *end)
{
gc_mark_locations(&rb_objspace, start, end, gc_mark_maybe);
}
static void
gc_mark_values(rb_objspace_t *objspace, long n, const VALUE *values)
{
long i;
for (i=0; i<n; i++) {
gc_mark(objspace, values[i]);
}
}
void
rb_gc_mark_values(long n, const VALUE *values)
{
long i;
rb_objspace_t *objspace = &rb_objspace;
for (i=0; i<n; i++) {
gc_mark_and_pin(objspace, values[i]);
}
}
static void
gc_mark_stack_values(rb_objspace_t *objspace, long n, const VALUE *values)
{
long i;
for (i=0; i<n; i++) {
if (is_markable_object(objspace, values[i])) {
gc_mark_and_pin(objspace, values[i]);
}
}
}
void
rb_gc_mark_vm_stack_values(long n, const VALUE *values)
{
rb_objspace_t *objspace = &rb_objspace;
gc_mark_stack_values(objspace, n, values);
}
static int
mark_value(st_data_t key, st_data_t value, st_data_t data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
gc_mark(objspace, (VALUE)value);
return ST_CONTINUE;
}
static int
mark_value_pin(st_data_t key, st_data_t value, st_data_t data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
gc_mark_and_pin(objspace, (VALUE)value);
return ST_CONTINUE;
}
static void
mark_tbl_no_pin(rb_objspace_t *objspace, st_table *tbl)
{
if (!tbl || tbl->num_entries == 0) return;
st_foreach(tbl, mark_value, (st_data_t)objspace);
}
static void
mark_tbl(rb_objspace_t *objspace, st_table *tbl)
{
if (!tbl || tbl->num_entries == 0) return;
st_foreach(tbl, mark_value_pin, (st_data_t)objspace);
}
static int
mark_key(st_data_t key, st_data_t value, st_data_t data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
gc_mark_and_pin(objspace, (VALUE)key);
return ST_CONTINUE;
}
static void
mark_set(rb_objspace_t *objspace, st_table *tbl)
{
if (!tbl) return;
st_foreach(tbl, mark_key, (st_data_t)objspace);
}
static int
pin_value(st_data_t key, st_data_t value, st_data_t data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
gc_mark_and_pin(objspace, (VALUE)value);
return ST_CONTINUE;
}
static void
mark_finalizer_tbl(rb_objspace_t *objspace, st_table *tbl)
{
if (!tbl) return;
st_foreach(tbl, pin_value, (st_data_t)objspace);
}
void
rb_mark_set(st_table *tbl)
{
mark_set(&rb_objspace, tbl);
}
static int
mark_keyvalue(st_data_t key, st_data_t value, st_data_t data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
gc_mark(objspace, (VALUE)key);
gc_mark(objspace, (VALUE)value);
return ST_CONTINUE;
}
static int
pin_key_pin_value(st_data_t key, st_data_t value, st_data_t data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
gc_mark_and_pin(objspace, (VALUE)key);
gc_mark_and_pin(objspace, (VALUE)value);
return ST_CONTINUE;
}
static int
pin_key_mark_value(st_data_t key, st_data_t value, st_data_t data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
gc_mark_and_pin(objspace, (VALUE)key);
gc_mark(objspace, (VALUE)value);
return ST_CONTINUE;
}
static void
mark_hash(rb_objspace_t *objspace, VALUE hash)
{
if (rb_hash_compare_by_id_p(hash)) {
rb_hash_stlike_foreach(hash, pin_key_mark_value, (st_data_t)objspace);
}
else {
rb_hash_stlike_foreach(hash, mark_keyvalue, (st_data_t)objspace);
}
if (RHASH_AR_TABLE_P(hash)) {
if (LIKELY(during_gc) && RHASH_TRANSIENT_P(hash)) {
rb_transient_heap_mark(hash, RHASH_AR_TABLE(hash));
}
}
else {
VM_ASSERT(!RHASH_TRANSIENT_P(hash));
}
gc_mark(objspace, RHASH(hash)->ifnone);
}
static void
mark_st(rb_objspace_t *objspace, st_table *tbl)
{
if (!tbl) return;
st_foreach(tbl, pin_key_pin_value, (st_data_t)objspace);
}
void
rb_mark_hash(st_table *tbl)
{
mark_st(&rb_objspace, tbl);
}
static void
mark_method_entry(rb_objspace_t *objspace, const rb_method_entry_t *me)
{
const rb_method_definition_t *def = me->def;
gc_mark(objspace, me->owner);
gc_mark(objspace, me->defined_class);
if (def) {
switch (def->type) {
case VM_METHOD_TYPE_ISEQ:
if (def->body.iseq.iseqptr) gc_mark(objspace, (VALUE)def->body.iseq.iseqptr);
gc_mark(objspace, (VALUE)def->body.iseq.cref);
if (def->iseq_overload && me->defined_class) {
// it can be a key of "overloaded_cme" table
// so it should be pinned.
gc_mark_and_pin(objspace, (VALUE)me);
}
break;
case VM_METHOD_TYPE_ATTRSET:
case VM_METHOD_TYPE_IVAR:
gc_mark(objspace, def->body.attr.location);
break;
case VM_METHOD_TYPE_BMETHOD:
gc_mark(objspace, def->body.bmethod.proc);
if (def->body.bmethod.hooks) rb_hook_list_mark(def->body.bmethod.hooks);
break;
case VM_METHOD_TYPE_ALIAS:
gc_mark(objspace, (VALUE)def->body.alias.original_me);
return;
case VM_METHOD_TYPE_REFINED:
gc_mark(objspace, (VALUE)def->body.refined.orig_me);
gc_mark(objspace, (VALUE)def->body.refined.owner);
break;
case VM_METHOD_TYPE_CFUNC:
case VM_METHOD_TYPE_ZSUPER:
case VM_METHOD_TYPE_MISSING:
case VM_METHOD_TYPE_OPTIMIZED:
case VM_METHOD_TYPE_UNDEF:
case VM_METHOD_TYPE_NOTIMPLEMENTED:
break;
}
}
}
static enum rb_id_table_iterator_result
mark_method_entry_i(VALUE me, void *data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
gc_mark(objspace, me);
return ID_TABLE_CONTINUE;
}
static void
mark_m_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
{
if (tbl) {
rb_id_table_foreach_values(tbl, mark_method_entry_i, objspace);
}
}
static enum rb_id_table_iterator_result
mark_const_entry_i(VALUE value, void *data)
{
const rb_const_entry_t *ce = (const rb_const_entry_t *)value;
rb_objspace_t *objspace = data;
gc_mark(objspace, ce->value);
gc_mark(objspace, ce->file);
return ID_TABLE_CONTINUE;
}
static void
mark_const_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
{
if (!tbl) return;
rb_id_table_foreach_values(tbl, mark_const_entry_i, objspace);
}
#if STACK_GROW_DIRECTION < 0
#define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_END, (end) = STACK_START)
#elif STACK_GROW_DIRECTION > 0
#define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_START, (end) = STACK_END+(appendix))
#else
#define GET_STACK_BOUNDS(start, end, appendix) \
((STACK_END < STACK_START) ? \
((start) = STACK_END, (end) = STACK_START) : ((start) = STACK_START, (end) = STACK_END+(appendix)))
#endif
static void each_stack_location(rb_objspace_t *objspace, const rb_execution_context_t *ec,
const VALUE *stack_start, const VALUE *stack_end, void (*cb)(rb_objspace_t *, VALUE));
#if defined(__wasm__)
static VALUE *rb_stack_range_tmp[2];
static void
rb_mark_locations(void *begin, void *end)
{
rb_stack_range_tmp[0] = begin;
rb_stack_range_tmp[1] = end;
}
# if defined(__EMSCRIPTEN__)
static void
mark_current_machine_context(rb_objspace_t *objspace, rb_execution_context_t *ec)
{
emscripten_scan_stack(rb_mark_locations);
each_stack_location(objspace, ec, rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe);
emscripten_scan_registers(rb_mark_locations);
each_stack_location(objspace, ec, rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe);
}
# else // use Asyncify version
static void
mark_current_machine_context(rb_objspace_t *objspace, rb_execution_context_t *ec)
{
rb_wasm_scan_stack(rb_mark_locations);
each_stack_location(objspace, ec, rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe);
rb_wasm_scan_locals(rb_mark_locations);
each_stack_location(objspace, ec, rb_stack_range_tmp[0], rb_stack_range_tmp[1], gc_mark_maybe);
}
# endif
#else // !defined(__wasm__)
static void
mark_current_machine_context(rb_objspace_t *objspace, rb_execution_context_t *ec)
{
union {
rb_jmp_buf j;
VALUE v[sizeof(rb_jmp_buf) / (sizeof(VALUE))];
} save_regs_gc_mark;
VALUE *stack_start, *stack_end;
FLUSH_REGISTER_WINDOWS;
memset(&save_regs_gc_mark, 0, sizeof(save_regs_gc_mark));
/* This assumes that all registers are saved into the jmp_buf (and stack) */
rb_setjmp(save_regs_gc_mark.j);
/* SET_STACK_END must be called in this function because
* the stack frame of this function may contain
* callee save registers and they should be marked. */
SET_STACK_END;
GET_STACK_BOUNDS(stack_start, stack_end, 1);
each_location(objspace, save_regs_gc_mark.v, numberof(save_regs_gc_mark.v), gc_mark_maybe);
each_stack_location(objspace, ec, stack_start, stack_end, gc_mark_maybe);
}
#endif
static void
each_machine_stack_value(const rb_execution_context_t *ec, void (*cb)(rb_objspace_t *, VALUE))
{
rb_objspace_t *objspace = &rb_objspace;
VALUE *stack_start, *stack_end;
GET_STACK_BOUNDS(stack_start, stack_end, 0);
each_stack_location(objspace, ec, stack_start, stack_end, cb);
}
void
rb_gc_mark_machine_stack(const rb_execution_context_t *ec)
{
each_machine_stack_value(ec, gc_mark_maybe);
}
static void
each_stack_location(rb_objspace_t *objspace, const rb_execution_context_t *ec,
const VALUE *stack_start, const VALUE *stack_end, void (*cb)(rb_objspace_t *, VALUE))
{
gc_mark_locations(objspace, stack_start, stack_end, cb);
#if defined(__mc68000__)
gc_mark_locations(objspace,
(VALUE*)((char*)stack_start + 2),
(VALUE*)((char*)stack_end - 2), cb);
#endif
}
void
rb_mark_tbl(st_table *tbl)
{
mark_tbl(&rb_objspace, tbl);
}
void
rb_mark_tbl_no_pin(st_table *tbl)
{
mark_tbl_no_pin(&rb_objspace, tbl);
}
static void
gc_mark_maybe(rb_objspace_t *objspace, VALUE obj)
{
(void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj));
if (is_pointer_to_heap(objspace, (void *)obj)) {
void *ptr = asan_unpoison_object_temporary(obj);
/* Garbage can live on the stack, so do not mark or pin */
switch (BUILTIN_TYPE(obj)) {
case T_ZOMBIE:
case T_NONE:
break;
default:
gc_mark_and_pin(objspace, obj);
break;
}
if (ptr) {
GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
asan_poison_object(obj);
}
}
}
void
rb_gc_mark_maybe(VALUE obj)
{
gc_mark_maybe(&rb_objspace, obj);
}
static inline int
gc_mark_set(rb_objspace_t *objspace, VALUE obj)
{
ASSERT_vm_locking();
if (RVALUE_MARKED(obj)) return 0;
MARK_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj);
return 1;
}
static int
gc_remember_unprotected(rb_objspace_t *objspace, VALUE obj)
{
struct heap_page *page = GET_HEAP_PAGE(obj);
bits_t *uncollectible_bits = &page->uncollectible_bits[0];
if (!MARKED_IN_BITMAP(uncollectible_bits, obj)) {
page->flags.has_uncollectible_shady_objects = TRUE;
MARK_IN_BITMAP(uncollectible_bits, obj);
objspace->rgengc.uncollectible_wb_unprotected_objects++;
#if RGENGC_PROFILE > 0
objspace->profile.total_remembered_shady_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.remembered_shady_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
#endif
return TRUE;
}
else {
return FALSE;
}
}
static void
rgengc_check_relation(rb_objspace_t *objspace, VALUE obj)
{
const VALUE old_parent = objspace->rgengc.parent_object;
if (old_parent) { /* parent object is old */
if (RVALUE_WB_UNPROTECTED(obj)) {
if (gc_remember_unprotected(objspace, obj)) {
gc_report(2, objspace, "relation: (O->S) %s -> %s\n", obj_info(old_parent), obj_info(obj));
}
}
else {
if (!RVALUE_OLD_P(obj)) {
if (RVALUE_MARKED(obj)) {
/* An object pointed from an OLD object should be OLD. */
gc_report(2, objspace, "relation: (O->unmarked Y) %s -> %s\n", obj_info(old_parent), obj_info(obj));
RVALUE_AGE_SET_OLD(objspace, obj);
if (is_incremental_marking(objspace)) {
if (!RVALUE_MARKING(obj)) {
gc_grey(objspace, obj);
}
}
else {
rgengc_remember(objspace, obj);
}
}
else {
gc_report(2, objspace, "relation: (O->Y) %s -> %s\n", obj_info(old_parent), obj_info(obj));
RVALUE_AGE_SET_CANDIDATE(objspace, obj);
}
}
}
}
GC_ASSERT(old_parent == objspace->rgengc.parent_object);
}
static void
gc_grey(rb_objspace_t *objspace, VALUE obj)
{
#if RGENGC_CHECK_MODE
if (RVALUE_MARKED(obj) == FALSE) rb_bug("gc_grey: %s is not marked.", obj_info(obj));
if (RVALUE_MARKING(obj) == TRUE) rb_bug("gc_grey: %s is marking/remembered.", obj_info(obj));
#endif
#if GC_ENABLE_INCREMENTAL_MARK
if (is_incremental_marking(objspace)) {
MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
}
#endif
push_mark_stack(&objspace->mark_stack, obj);
}
static void
gc_aging(rb_objspace_t *objspace, VALUE obj)
{
struct heap_page *page = GET_HEAP_PAGE(obj);
GC_ASSERT(RVALUE_MARKING(obj) == FALSE);
check_rvalue_consistency(obj);
if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) {
if (!RVALUE_OLD_P(obj)) {
gc_report(3, objspace, "gc_aging: YOUNG: %s\n", obj_info(obj));
RVALUE_AGE_INC(objspace, obj);
}
else if (is_full_marking(objspace)) {
GC_ASSERT(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE);
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, page, obj);
}
}
check_rvalue_consistency(obj);
objspace->marked_slots++;
}
NOINLINE(static void gc_mark_ptr(rb_objspace_t *objspace, VALUE obj));
static void reachable_objects_from_callback(VALUE obj);
static void
gc_mark_ptr(rb_objspace_t *objspace, VALUE obj)
{
if (LIKELY(during_gc)) {
rgengc_check_relation(objspace, obj);
if (!gc_mark_set(objspace, obj)) return; /* already marked */
if (0) { // for debug GC marking miss
if (objspace->rgengc.parent_object) {
RUBY_DEBUG_LOG("%p (%s) parent:%p (%s)",
(void *)obj, obj_type_name(obj),
(void *)objspace->rgengc.parent_object, obj_type_name(objspace->rgengc.parent_object));
}
else {
RUBY_DEBUG_LOG("%p (%s)", (void *)obj, obj_type_name(obj));
}
}
if (UNLIKELY(RB_TYPE_P(obj, T_NONE))) {
rp(obj);
rb_bug("try to mark T_NONE object"); /* check here will help debugging */
}
gc_aging(objspace, obj);
gc_grey(objspace, obj);
}
else {
reachable_objects_from_callback(obj);
}
}
static inline void
gc_pin(rb_objspace_t *objspace, VALUE obj)
{
GC_ASSERT(is_markable_object(objspace, obj));
if (UNLIKELY(objspace->flags.during_compacting)) {
if (LIKELY(during_gc)) {
MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj);
}
}
}
static inline void
gc_mark_and_pin(rb_objspace_t *objspace, VALUE obj)
{
if (!is_markable_object(objspace, obj)) return;
gc_pin(objspace, obj);
gc_mark_ptr(objspace, obj);
}
static inline void
gc_mark(rb_objspace_t *objspace, VALUE obj)
{
if (!is_markable_object(objspace, obj)) return;
gc_mark_ptr(objspace, obj);
}
void
rb_gc_mark_movable(VALUE ptr)
{
gc_mark(&rb_objspace, ptr);
}
void
rb_gc_mark(VALUE ptr)
{
gc_mark_and_pin(&rb_objspace, ptr);
}
/* CAUTION: THIS FUNCTION ENABLE *ONLY BEFORE* SWEEPING.
* This function is only for GC_END_MARK timing.
*/
int
rb_objspace_marked_object_p(VALUE obj)
{
return RVALUE_MARKED(obj) ? TRUE : FALSE;
}
static inline void
gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj)
{
if (RVALUE_OLD_P(obj)) {
objspace->rgengc.parent_object = obj;
}
else {
objspace->rgengc.parent_object = Qfalse;
}
}
static void
gc_mark_imemo(rb_objspace_t *objspace, VALUE obj)
{
switch (imemo_type(obj)) {
case imemo_env:
{
const rb_env_t *env = (const rb_env_t *)obj;
if (LIKELY(env->ep)) {
// just after newobj() can be NULL here.
GC_ASSERT(env->ep[VM_ENV_DATA_INDEX_ENV] == obj);
GC_ASSERT(VM_ENV_ESCAPED_P(env->ep));
gc_mark_values(objspace, (long)env->env_size, env->env);
VM_ENV_FLAGS_SET(env->ep, VM_ENV_FLAG_WB_REQUIRED);
gc_mark(objspace, (VALUE)rb_vm_env_prev_env(env));
gc_mark(objspace, (VALUE)env->iseq);
}
}
return;
case imemo_cref:
gc_mark(objspace, RANY(obj)->as.imemo.cref.klass_or_self);
gc_mark(objspace, (VALUE)RANY(obj)->as.imemo.cref.next);
gc_mark(objspace, RANY(obj)->as.imemo.cref.refinements);
return;
case imemo_svar:
gc_mark(objspace, RANY(obj)->as.imemo.svar.cref_or_me);
gc_mark(objspace, RANY(obj)->as.imemo.svar.lastline);
gc_mark(objspace, RANY(obj)->as.imemo.svar.backref);
gc_mark(objspace, RANY(obj)->as.imemo.svar.others);
return;
case imemo_throw_data:
gc_mark(objspace, RANY(obj)->as.imemo.throw_data.throw_obj);
return;
case imemo_ifunc:
gc_mark_maybe(objspace, (VALUE)RANY(obj)->as.imemo.ifunc.data);
return;
case imemo_memo:
gc_mark(objspace, RANY(obj)->as.imemo.memo.v1);
gc_mark(objspace, RANY(obj)->as.imemo.memo.v2);
gc_mark_maybe(objspace, RANY(obj)->as.imemo.memo.u3.value);
return;
case imemo_ment:
mark_method_entry(objspace, &RANY(obj)->as.imemo.ment);
return;
case imemo_iseq:
rb_iseq_mark((rb_iseq_t *)obj);
return;
case imemo_tmpbuf:
{
const rb_imemo_tmpbuf_t *m = &RANY(obj)->as.imemo.alloc;
do {
rb_gc_mark_locations(m->ptr, m->ptr + m->cnt);
} while ((m = m->next) != NULL);
}
return;
case imemo_ast:
rb_ast_mark(&RANY(obj)->as.imemo.ast);
return;
case imemo_parser_strterm:
rb_strterm_mark(obj);
return;
case imemo_callinfo:
return;
case imemo_callcache:
{
const struct rb_callcache *cc = (const struct rb_callcache *)obj;
// should not mark klass here
gc_mark(objspace, (VALUE)vm_cc_cme(cc));
}
return;
case imemo_constcache:
{
const struct iseq_inline_constant_cache_entry *ice = (struct iseq_inline_constant_cache_entry *)obj;
gc_mark(objspace, ice->value);
}
return;
#if VM_CHECK_MODE > 0
default:
VM_UNREACHABLE(gc_mark_imemo);
#endif
}
}
static void
gc_mark_children(rb_objspace_t *objspace, VALUE obj)
{
register RVALUE *any = RANY(obj);
gc_mark_set_parent(objspace, obj);
if (FL_TEST(obj, FL_EXIVAR)) {
rb_mark_generic_ivar(obj);
}
switch (BUILTIN_TYPE(obj)) {
case T_FLOAT:
case T_BIGNUM:
case T_SYMBOL:
/* Not immediates, but does not have references and singleton
* class */
return;
case T_NIL:
case T_FIXNUM:
rb_bug("rb_gc_mark() called for broken object");
break;
case T_NODE:
UNEXPECTED_NODE(rb_gc_mark);
break;
case T_IMEMO:
gc_mark_imemo(objspace, obj);
return;
default:
break;
}
gc_mark(objspace, any->as.basic.klass);
switch (BUILTIN_TYPE(obj)) {
case T_CLASS:
case T_MODULE:
if (RCLASS_SUPER(obj)) {
gc_mark(objspace, RCLASS_SUPER(obj));
}
if (!RCLASS_EXT(obj)) break;
mark_m_tbl(objspace, RCLASS_M_TBL(obj));
cc_table_mark(objspace, obj);
mark_tbl_no_pin(objspace, RCLASS_IV_TBL(obj));
mark_const_tbl(objspace, RCLASS_CONST_TBL(obj));
break;
case T_ICLASS:
if (RICLASS_OWNS_M_TBL_P(obj)) {
mark_m_tbl(objspace, RCLASS_M_TBL(obj));
}
if (RCLASS_SUPER(obj)) {
gc_mark(objspace, RCLASS_SUPER(obj));
}
if (!RCLASS_EXT(obj)) break;
if (RCLASS_INCLUDER(obj)) {
gc_mark(objspace, RCLASS_INCLUDER(obj));
}
mark_m_tbl(objspace, RCLASS_CALLABLE_M_TBL(obj));
cc_table_mark(objspace, obj);
break;
case T_ARRAY:
if (ARY_SHARED_P(obj)) {
VALUE root = ARY_SHARED_ROOT(obj);
gc_mark(objspace, root);
}
else {
long i, len = RARRAY_LEN(obj);
const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(obj);
for (i=0; i < len; i++) {
gc_mark(objspace, ptr[i]);
}
if (LIKELY(during_gc)) {
if (!ARY_EMBED_P(obj) && RARRAY_TRANSIENT_P(obj)) {
rb_transient_heap_mark(obj, ptr);
}
}
}
break;
case T_HASH:
mark_hash(objspace, obj);
break;
case T_STRING:
if (STR_SHARED_P(obj)) {
gc_mark(objspace, any->as.string.as.heap.aux.shared);
}
break;
case T_DATA:
{
void *const ptr = DATA_PTR(obj);
if (ptr) {
RUBY_DATA_FUNC mark_func = RTYPEDDATA_P(obj) ?
any->as.typeddata.type->function.dmark :
any->as.data.dmark;
if (mark_func) (*mark_func)(ptr);
}
}
break;
case T_OBJECT:
{
const VALUE * const ptr = ROBJECT_IVPTR(obj);
uint32_t i, len = ROBJECT_IV_COUNT(obj);
for (i = 0; i < len; i++) {
gc_mark(objspace, ptr[i]);
}
if (LIKELY(during_gc) &&
ROBJ_TRANSIENT_P(obj)) {
rb_transient_heap_mark(obj, ptr);
}
}
break;
case T_FILE:
if (any->as.file.fptr) {
gc_mark(objspace, any->as.file.fptr->self);
gc_mark(objspace, any->as.file.fptr->pathv);
gc_mark(objspace, any->as.file.fptr->tied_io_for_writing);
gc_mark(objspace, any->as.file.fptr->writeconv_asciicompat);
gc_mark(objspace, any->as.file.fptr->writeconv_pre_ecopts);
gc_mark(objspace, any->as.file.fptr->encs.ecopts);
gc_mark(objspace, any->as.file.fptr->write_lock);
gc_mark(objspace, any->as.file.fptr->timeout);
}
break;
case T_REGEXP:
gc_mark(objspace, any->as.regexp.src);
break;
case T_MATCH:
gc_mark(objspace, any->as.match.regexp);
if (any->as.match.str) {
gc_mark(objspace, any->as.match.str);
}
break;
case T_RATIONAL:
gc_mark(objspace, any->as.rational.num);
gc_mark(objspace, any->as.rational.den);
break;
case T_COMPLEX:
gc_mark(objspace, any->as.complex.real);
gc_mark(objspace, any->as.complex.imag);
break;
case T_STRUCT:
{
long i;
const long len = RSTRUCT_LEN(obj);
const VALUE * const ptr = RSTRUCT_CONST_PTR(obj);
for (i=0; i<len; i++) {
gc_mark(objspace, ptr[i]);
}
if (LIKELY(during_gc) &&
RSTRUCT_TRANSIENT_P(obj)) {
rb_transient_heap_mark(obj, ptr);
}
}
break;
default:
#if GC_DEBUG
rb_gcdebug_print_obj_condition((VALUE)obj);
#endif
if (BUILTIN_TYPE(obj) == T_MOVED) rb_bug("rb_gc_mark(): %p is T_MOVED", (void *)obj);
if (BUILTIN_TYPE(obj) == T_NONE) rb_bug("rb_gc_mark(): %p is T_NONE", (void *)obj);
if (BUILTIN_TYPE(obj) == T_ZOMBIE) rb_bug("rb_gc_mark(): %p is T_ZOMBIE", (void *)obj);
rb_bug("rb_gc_mark(): unknown data type 0x%x(%p) %s",
BUILTIN_TYPE(obj), (void *)any,
is_pointer_to_heap(objspace, any) ? "corrupted object" : "non object");
}
}
/**
* incremental: 0 -> not incremental (do all)
* incremental: n -> mark at most `n' objects
*/
static inline int
gc_mark_stacked_objects(rb_objspace_t *objspace, int incremental, size_t count)
{
mark_stack_t *mstack = &objspace->mark_stack;
VALUE obj;
#if GC_ENABLE_INCREMENTAL_MARK
size_t marked_slots_at_the_beginning = objspace->marked_slots;
size_t popped_count = 0;
#endif
while (pop_mark_stack(mstack, &obj)) {
if (obj == Qundef) continue; /* skip */
if (RGENGC_CHECK_MODE && !RVALUE_MARKED(obj)) {
rb_bug("gc_mark_stacked_objects: %s is not marked.", obj_info(obj));
}
gc_mark_children(objspace, obj);
#if GC_ENABLE_INCREMENTAL_MARK
if (incremental) {
if (RGENGC_CHECK_MODE && !RVALUE_MARKING(obj)) {
rb_bug("gc_mark_stacked_objects: incremental, but marking bit is 0");
}
CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
popped_count++;
if (popped_count + (objspace->marked_slots - marked_slots_at_the_beginning) > count) {
break;
}
}
else {
/* just ignore marking bits */
}
#endif
}
if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
if (is_mark_stack_empty(mstack)) {
shrink_stack_chunk_cache(mstack);
return TRUE;
}
else {
return FALSE;
}
}
static int
gc_mark_stacked_objects_incremental(rb_objspace_t *objspace, size_t count)
{
return gc_mark_stacked_objects(objspace, TRUE, count);
}
static int
gc_mark_stacked_objects_all(rb_objspace_t *objspace)
{
return gc_mark_stacked_objects(objspace, FALSE, 0);
}
#if PRINT_ROOT_TICKS
#define MAX_TICKS 0x100
static tick_t mark_ticks[MAX_TICKS];
static const char *mark_ticks_categories[MAX_TICKS];
static void
show_mark_ticks(void)
{
int i;
fprintf(stderr, "mark ticks result:\n");
for (i=0; i<MAX_TICKS; i++) {
const char *category = mark_ticks_categories[i];
if (category) {
fprintf(stderr, "%s\t%8lu\n", category, (unsigned long)mark_ticks[i]);
}
else {
break;
}
}
}
#endif /* PRINT_ROOT_TICKS */
static void
gc_mark_roots(rb_objspace_t *objspace, const char **categoryp)
{
struct gc_list *list;
rb_execution_context_t *ec = GET_EC();
rb_vm_t *vm = rb_ec_vm_ptr(ec);
#if PRINT_ROOT_TICKS
tick_t start_tick = tick();
int tick_count = 0;
const char *prev_category = 0;
if (mark_ticks_categories[0] == 0) {
atexit(show_mark_ticks);
}
#endif
if (categoryp) *categoryp = "xxx";
objspace->rgengc.parent_object = Qfalse;
#if PRINT_ROOT_TICKS
#define MARK_CHECKPOINT_PRINT_TICK(category) do { \
if (prev_category) { \
tick_t t = tick(); \
mark_ticks[tick_count] = t - start_tick; \
mark_ticks_categories[tick_count] = prev_category; \
tick_count++; \
} \
prev_category = category; \
start_tick = tick(); \
} while (0)
#else /* PRINT_ROOT_TICKS */
#define MARK_CHECKPOINT_PRINT_TICK(category)
#endif
#define MARK_CHECKPOINT(category) do { \
if (categoryp) *categoryp = category; \
MARK_CHECKPOINT_PRINT_TICK(category); \
} while (0)
MARK_CHECKPOINT("vm");
SET_STACK_END;
rb_vm_mark(vm);
if (vm->self) gc_mark(objspace, vm->self);
MARK_CHECKPOINT("finalizers");
mark_finalizer_tbl(objspace, finalizer_table);
MARK_CHECKPOINT("machine_context");
mark_current_machine_context(objspace, ec);
/* mark protected global variables */
MARK_CHECKPOINT("global_list");
for (list = global_list; list; list = list->next) {
gc_mark_maybe(objspace, *list->varptr);
}
MARK_CHECKPOINT("end_proc");
rb_mark_end_proc();
MARK_CHECKPOINT("global_tbl");
rb_gc_mark_global_tbl();
MARK_CHECKPOINT("object_id");
rb_gc_mark(objspace->next_object_id);
mark_tbl_no_pin(objspace, objspace->obj_to_id_tbl); /* Only mark ids */
if (stress_to_class) rb_gc_mark(stress_to_class);
MARK_CHECKPOINT("finish");
#undef MARK_CHECKPOINT
}
#if RGENGC_CHECK_MODE >= 4
#define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01)
#define IS_ROOTSIG(obj) ((VALUE)(obj) & 0x01)
#define GET_ROOTSIG(obj) ((const char *)((VALUE)(obj) >> 1))
struct reflist {
VALUE *list;
int pos;
int size;
};
static struct reflist *
reflist_create(VALUE obj)
{
struct reflist *refs = xmalloc(sizeof(struct reflist));
refs->size = 1;
refs->list = ALLOC_N(VALUE, refs->size);
refs->list[0] = obj;
refs->pos = 1;
return refs;
}
static void
reflist_destruct(struct reflist *refs)
{
xfree(refs->list);
xfree(refs);
}
static void
reflist_add(struct reflist *refs, VALUE obj)
{
if (refs->pos == refs->size) {
refs->size *= 2;
SIZED_REALLOC_N(refs->list, VALUE, refs->size, refs->size/2);
}
refs->list[refs->pos++] = obj;
}
static void
reflist_dump(struct reflist *refs)
{
int i;
for (i=0; i<refs->pos; i++) {
VALUE obj = refs->list[i];
if (IS_ROOTSIG(obj)) { /* root */
fprintf(stderr, "<root@%s>", GET_ROOTSIG(obj));
}
else {
fprintf(stderr, "<%s>", obj_info(obj));
}
if (i+1 < refs->pos) fprintf(stderr, ", ");
}
}
static int
reflist_referred_from_machine_context(struct reflist *refs)
{
int i;
for (i=0; i<refs->pos; i++) {
VALUE obj = refs->list[i];
if (IS_ROOTSIG(obj) && strcmp(GET_ROOTSIG(obj), "machine_context") == 0) return 1;
}
return 0;
}
struct allrefs {
rb_objspace_t *objspace;
/* a -> obj1
* b -> obj1
* c -> obj1
* c -> obj2
* d -> obj3
* #=> {obj1 => [a, b, c], obj2 => [c, d]}
*/
struct st_table *references;
const char *category;
VALUE root_obj;
mark_stack_t mark_stack;
};
static int
allrefs_add(struct allrefs *data, VALUE obj)
{
struct reflist *refs;
st_data_t r;
if (st_lookup(data->references, obj, &r)) {
refs = (struct reflist *)r;
reflist_add(refs, data->root_obj);
return 0;
}
else {
refs = reflist_create(data->root_obj);
st_insert(data->references, obj, (st_data_t)refs);
return 1;
}
}
static void
allrefs_i(VALUE obj, void *ptr)
{
struct allrefs *data = (struct allrefs *)ptr;
if (allrefs_add(data, obj)) {
push_mark_stack(&data->mark_stack, obj);
}
}
static void
allrefs_roots_i(VALUE obj, void *ptr)
{
struct allrefs *data = (struct allrefs *)ptr;
if (strlen(data->category) == 0) rb_bug("!!!");
data->root_obj = MAKE_ROOTSIG(data->category);
if (allrefs_add(data, obj)) {
push_mark_stack(&data->mark_stack, obj);
}
}
#define PUSH_MARK_FUNC_DATA(v) do { \
struct gc_mark_func_data_struct *prev_mark_func_data = GET_RACTOR()->mfd; \
GET_RACTOR()->mfd = (v);
#define POP_MARK_FUNC_DATA() GET_RACTOR()->mfd = prev_mark_func_data;} while (0)
static st_table *
objspace_allrefs(rb_objspace_t *objspace)
{
struct allrefs data;
struct gc_mark_func_data_struct mfd;
VALUE obj;
int prev_dont_gc = dont_gc_val();
dont_gc_on();
data.objspace = objspace;
data.references = st_init_numtable();
init_mark_stack(&data.mark_stack);
mfd.mark_func = allrefs_roots_i;
mfd.data = &data;
/* traverse root objects */
PUSH_MARK_FUNC_DATA(&mfd);
GET_RACTOR()->mfd = &mfd;
gc_mark_roots(objspace, &data.category);
POP_MARK_FUNC_DATA();
/* traverse rest objects reachable from root objects */
while (pop_mark_stack(&data.mark_stack, &obj)) {
rb_objspace_reachable_objects_from(data.root_obj = obj, allrefs_i, &data);
}
free_stack_chunks(&data.mark_stack);
dont_gc_set(prev_dont_gc);
return data.references;
}
static int
objspace_allrefs_destruct_i(st_data_t key, st_data_t value, st_data_t ptr)
{
struct reflist *refs = (struct reflist *)value;
reflist_destruct(refs);
return ST_CONTINUE;
}
static void
objspace_allrefs_destruct(struct st_table *refs)
{
st_foreach(refs, objspace_allrefs_destruct_i, 0);
st_free_table(refs);
}
#if RGENGC_CHECK_MODE >= 5
static int
allrefs_dump_i(st_data_t k, st_data_t v, st_data_t ptr)
{
VALUE obj = (VALUE)k;
struct reflist *refs = (struct reflist *)v;
fprintf(stderr, "[allrefs_dump_i] %s <- ", obj_info(obj));
reflist_dump(refs);
fprintf(stderr, "\n");
return ST_CONTINUE;
}
static void
allrefs_dump(rb_objspace_t *objspace)
{
VALUE size = objspace->rgengc.allrefs_table->num_entries;
fprintf(stderr, "[all refs] (size: %"PRIuVALUE")\n", size);
st_foreach(objspace->rgengc.allrefs_table, allrefs_dump_i, 0);
}
#endif
static int
gc_check_after_marks_i(st_data_t k, st_data_t v, st_data_t ptr)
{
VALUE obj = k;
struct reflist *refs = (struct reflist *)v;
rb_objspace_t *objspace = (rb_objspace_t *)ptr;
/* object should be marked or oldgen */
if (!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj)) {
fprintf(stderr, "gc_check_after_marks_i: %s is not marked and not oldgen.\n", obj_info(obj));
fprintf(stderr, "gc_check_after_marks_i: %p is referred from ", (void *)obj);
reflist_dump(refs);
if (reflist_referred_from_machine_context(refs)) {
fprintf(stderr, " (marked from machine stack).\n");
/* marked from machine context can be false positive */
}
else {
objspace->rgengc.error_count++;
fprintf(stderr, "\n");
}
}
return ST_CONTINUE;
}
static void
gc_marks_check(rb_objspace_t *objspace, st_foreach_callback_func *checker_func, const char *checker_name)
{
size_t saved_malloc_increase = objspace->malloc_params.increase;
#if RGENGC_ESTIMATE_OLDMALLOC
size_t saved_oldmalloc_increase = objspace->rgengc.oldmalloc_increase;
#endif
VALUE already_disabled = rb_objspace_gc_disable(objspace);
objspace->rgengc.allrefs_table = objspace_allrefs(objspace);
if (checker_func) {
st_foreach(objspace->rgengc.allrefs_table, checker_func, (st_data_t)objspace);
}
if (objspace->rgengc.error_count > 0) {
#if RGENGC_CHECK_MODE >= 5
allrefs_dump(objspace);
#endif
if (checker_name) rb_bug("%s: GC has problem.", checker_name);
}
objspace_allrefs_destruct(objspace->rgengc.allrefs_table);
objspace->rgengc.allrefs_table = 0;
if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
objspace->malloc_params.increase = saved_malloc_increase;
#if RGENGC_ESTIMATE_OLDMALLOC
objspace->rgengc.oldmalloc_increase = saved_oldmalloc_increase;
#endif
}
#endif /* RGENGC_CHECK_MODE >= 4 */
struct verify_internal_consistency_struct {
rb_objspace_t *objspace;
int err_count;
size_t live_object_count;
size_t zombie_object_count;
VALUE parent;
size_t old_object_count;
size_t remembered_shady_count;
};
static void
check_generation_i(const VALUE child, void *ptr)
{
struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
const VALUE parent = data->parent;
if (RGENGC_CHECK_MODE) GC_ASSERT(RVALUE_OLD_P(parent));
if (!RVALUE_OLD_P(child)) {
if (!RVALUE_REMEMBERED(parent) &&
!RVALUE_REMEMBERED(child) &&
!RVALUE_UNCOLLECTIBLE(child)) {
fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (O->Y) %s -> %s\n", obj_info(parent), obj_info(child));
data->err_count++;
}
}
}
static void
check_color_i(const VALUE child, void *ptr)
{
struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
const VALUE parent = data->parent;
if (!RVALUE_WB_UNPROTECTED(parent) && RVALUE_WHITE_P(child)) {
fprintf(stderr, "verify_internal_consistency_reachable_i: WB miss (B->W) - %s -> %s\n",
obj_info(parent), obj_info(child));
data->err_count++;
}
}
static void
check_children_i(const VALUE child, void *ptr)
{
struct verify_internal_consistency_struct *data = (struct verify_internal_consistency_struct *)ptr;
if (check_rvalue_consistency_force(child, FALSE) != 0) {
fprintf(stderr, "check_children_i: %s has error (referenced from %s)",
obj_info(child), obj_info(data->parent));
rb_print_backtrace(); /* C backtrace will help to debug */
data->err_count++;
}
}
static int
verify_internal_consistency_i(void *page_start, void *page_end, size_t stride,
struct verify_internal_consistency_struct *data)
{
VALUE obj;
rb_objspace_t *objspace = data->objspace;
for (obj = (VALUE)page_start; obj != (VALUE)page_end; obj += stride) {
void *poisoned = asan_unpoison_object_temporary(obj);
if (is_live_object(objspace, obj)) {
/* count objects */
data->live_object_count++;
data->parent = obj;
/* Normally, we don't expect T_MOVED objects to be in the heap.
* But they can stay alive on the stack, */
if (!gc_object_moved_p(objspace, obj)) {
/* moved slots don't have children */
rb_objspace_reachable_objects_from(obj, check_children_i, (void *)data);
}
/* check health of children */
if (RVALUE_OLD_P(obj)) data->old_object_count++;
if (RVALUE_WB_UNPROTECTED(obj) && RVALUE_UNCOLLECTIBLE(obj)) data->remembered_shady_count++;
if (!is_marking(objspace) && RVALUE_OLD_P(obj)) {
/* reachable objects from an oldgen object should be old or (young with remember) */
data->parent = obj;
rb_objspace_reachable_objects_from(obj, check_generation_i, (void *)data);
}
if (is_incremental_marking(objspace)) {
if (RVALUE_BLACK_P(obj)) {
/* reachable objects from black objects should be black or grey objects */
data->parent = obj;
rb_objspace_reachable_objects_from(obj, check_color_i, (void *)data);
}
}
}
else {
if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
GC_ASSERT((RBASIC(obj)->flags & ~FL_SEEN_OBJ_ID) == T_ZOMBIE);
data->zombie_object_count++;
}
}
if (poisoned) {
GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
asan_poison_object(obj);
}
}
return 0;
}
static int
gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
unsigned int has_remembered_shady = FALSE;
unsigned int has_remembered_old = FALSE;
int remembered_old_objects = 0;
int free_objects = 0;
int zombie_objects = 0;
short slot_size = page->slot_size;
uintptr_t start = (uintptr_t)page->start;
uintptr_t end = start + page->total_slots * slot_size;
for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
VALUE val = (VALUE)ptr;
void *poisoned = asan_unpoison_object_temporary(val);
enum ruby_value_type type = BUILTIN_TYPE(val);
if (type == T_NONE) free_objects++;
if (type == T_ZOMBIE) zombie_objects++;
if (RVALUE_PAGE_UNCOLLECTIBLE(page, val) && RVALUE_PAGE_WB_UNPROTECTED(page, val)) {
has_remembered_shady = TRUE;
}
if (RVALUE_PAGE_MARKING(page, val)) {
has_remembered_old = TRUE;
remembered_old_objects++;
}
if (poisoned) {
GC_ASSERT(BUILTIN_TYPE(val) == T_NONE);
asan_poison_object(val);
}
}
if (!is_incremental_marking(objspace) &&
page->flags.has_remembered_objects == FALSE && has_remembered_old == TRUE) {
for (uintptr_t ptr = start; ptr < end; ptr += slot_size) {
VALUE val = (VALUE)ptr;
if (RVALUE_PAGE_MARKING(page, val)) {
fprintf(stderr, "marking -> %s\n", obj_info(val));
}
}
rb_bug("page %p's has_remembered_objects should be false, but there are remembered old objects (%d). %s",
(void *)page, remembered_old_objects, obj ? obj_info(obj) : "");
}
if (page->flags.has_uncollectible_shady_objects == FALSE && has_remembered_shady == TRUE) {
rb_bug("page %p's has_remembered_shady should be false, but there are remembered shady objects. %s",
(void *)page, obj ? obj_info(obj) : "");
}
if (0) {
/* free_slots may not equal to free_objects */
if (page->free_slots != free_objects) {
rb_bug("page %p's free_slots should be %d, but %d\n", (void *)page, page->free_slots, free_objects);
}
}
if (page->final_slots != zombie_objects) {
rb_bug("page %p's final_slots should be %d, but %d\n", (void *)page, page->final_slots, zombie_objects);
}
return remembered_old_objects;
}
static int
gc_verify_heap_pages_(rb_objspace_t *objspace, struct ccan_list_head *head)
{
int remembered_old_objects = 0;
struct heap_page *page = 0;
ccan_list_for_each(head, page, page_node) {
asan_unlock_freelist(page);
RVALUE *p = page->freelist;
while (p) {
VALUE vp = (VALUE)p;
VALUE prev = vp;
asan_unpoison_object(vp, false);
if (BUILTIN_TYPE(vp) != T_NONE) {
fprintf(stderr, "freelist slot expected to be T_NONE but was: %s\n", obj_info(vp));
}
p = p->as.free.next;
asan_poison_object(prev);
}
asan_lock_freelist(page);
if (page->flags.has_remembered_objects == FALSE) {
remembered_old_objects += gc_verify_heap_page(objspace, page, Qfalse);
}
}
return remembered_old_objects;
}
static int
gc_verify_heap_pages(rb_objspace_t *objspace)
{
int remembered_old_objects = 0;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
remembered_old_objects += gc_verify_heap_pages_(objspace, &(SIZE_POOL_EDEN_HEAP(&size_pools[i])->pages));
remembered_old_objects += gc_verify_heap_pages_(objspace, &(SIZE_POOL_TOMB_HEAP(&size_pools[i])->pages));
}
return remembered_old_objects;
}
/*
* call-seq:
* GC.verify_internal_consistency -> nil
*
* Verify internal consistency.
*
* This method is implementation specific.
* Now this method checks generational consistency
* if RGenGC is supported.
*/
static VALUE
gc_verify_internal_consistency_m(VALUE dummy)
{
gc_verify_internal_consistency(&rb_objspace);
return Qnil;
}
static void
gc_verify_internal_consistency_(rb_objspace_t *objspace)
{
struct verify_internal_consistency_struct data = {0};
data.objspace = objspace;
gc_report(5, objspace, "gc_verify_internal_consistency: start\n");
/* check relations */
for (size_t i = 0; i < heap_allocated_pages; i++) {
struct heap_page *page = heap_pages_sorted[i];
short slot_size = page->slot_size;
uintptr_t start = (uintptr_t)page->start;
uintptr_t end = start + page->total_slots * slot_size;
verify_internal_consistency_i((void *)start, (void *)end, slot_size, &data);
}
if (data.err_count != 0) {
#if RGENGC_CHECK_MODE >= 5
objspace->rgengc.error_count = data.err_count;
gc_marks_check(objspace, NULL, NULL);
allrefs_dump(objspace);
#endif
rb_bug("gc_verify_internal_consistency: found internal inconsistency.");
}
/* check heap_page status */
gc_verify_heap_pages(objspace);
/* check counters */
if (!is_lazy_sweeping(objspace) &&
!finalizing &&
ruby_single_main_ractor != NULL) {
if (objspace_live_slots(objspace) != data.live_object_count) {
fprintf(stderr, "heap_pages_final_slots: %"PRIdSIZE", "
"objspace->profile.total_freed_objects: %"PRIdSIZE"\n",
heap_pages_final_slots, objspace->profile.total_freed_objects);
rb_bug("inconsistent live slot number: expect %"PRIuSIZE", but %"PRIuSIZE".",
objspace_live_slots(objspace), data.live_object_count);
}
}
if (!is_marking(objspace)) {
if (objspace->rgengc.old_objects != data.old_object_count) {
rb_bug("inconsistent old slot number: expect %"PRIuSIZE", but %"PRIuSIZE".",
objspace->rgengc.old_objects, data.old_object_count);
}
if (objspace->rgengc.uncollectible_wb_unprotected_objects != data.remembered_shady_count) {
rb_bug("inconsistent number of wb unprotected objects: expect %"PRIuSIZE", but %"PRIuSIZE".",
objspace->rgengc.uncollectible_wb_unprotected_objects, data.remembered_shady_count);
}
}
if (!finalizing) {
size_t list_count = 0;
{
VALUE z = heap_pages_deferred_final;
while (z) {
list_count++;
z = RZOMBIE(z)->next;
}
}
if (heap_pages_final_slots != data.zombie_object_count ||
heap_pages_final_slots != list_count) {
rb_bug("inconsistent finalizing object count:\n"
" expect %"PRIuSIZE"\n"
" but %"PRIuSIZE" zombies\n"
" heap_pages_deferred_final list has %"PRIuSIZE" items.",
heap_pages_final_slots,
data.zombie_object_count,
list_count);
}
}
gc_report(5, objspace, "gc_verify_internal_consistency: OK\n");
}
static void
gc_verify_internal_consistency(rb_objspace_t *objspace)
{
RB_VM_LOCK_ENTER();
{
rb_vm_barrier(); // stop other ractors
unsigned int prev_during_gc = during_gc;
during_gc = FALSE; // stop gc here
{
gc_verify_internal_consistency_(objspace);
}
during_gc = prev_during_gc;
}
RB_VM_LOCK_LEAVE();
}
void
rb_gc_verify_internal_consistency(void)
{
gc_verify_internal_consistency(&rb_objspace);
}
static VALUE
gc_verify_transient_heap_internal_consistency(VALUE dmy)
{
rb_transient_heap_verify();
return Qnil;
}
#if GC_ENABLE_INCREMENTAL_MARK
static void
heap_move_pooled_pages_to_free_pages(rb_heap_t *heap)
{
if (heap->pooled_pages) {
if (heap->free_pages) {
struct heap_page *free_pages_tail = heap->free_pages;
while (free_pages_tail->free_next) {
free_pages_tail = free_pages_tail->free_next;
}
free_pages_tail->free_next = heap->pooled_pages;
}
else {
heap->free_pages = heap->pooled_pages;
}
heap->pooled_pages = NULL;
}
}
#endif
/* marks */
static void
gc_marks_start(rb_objspace_t *objspace, int full_mark)
{
/* start marking */
gc_report(1, objspace, "gc_marks_start: (%s)\n", full_mark ? "full" : "minor");
gc_mode_transition(objspace, gc_mode_marking);
if (full_mark) {
#if GC_ENABLE_INCREMENTAL_MARK
size_t incremental_marking_steps = (objspace->rincgc.pooled_slots / INCREMENTAL_MARK_STEP_ALLOCATIONS) + 1;
objspace->rincgc.step_slots = (objspace->marked_slots * 2) / incremental_marking_steps;
if (0) fprintf(stderr, "objspace->marked_slots: %"PRIdSIZE", "
"objspace->rincgc.pooled_page_num: %"PRIdSIZE", "
"objspace->rincgc.step_slots: %"PRIdSIZE", \n",
objspace->marked_slots, objspace->rincgc.pooled_slots, objspace->rincgc.step_slots);
#endif
objspace->flags.during_minor_gc = FALSE;
if (ruby_enable_autocompact) {
objspace->flags.during_compacting |= TRUE;
}
objspace->profile.major_gc_count++;
objspace->rgengc.uncollectible_wb_unprotected_objects = 0;
objspace->rgengc.old_objects = 0;
objspace->rgengc.last_major_gc = objspace->profile.count;
objspace->marked_slots = 0;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
rgengc_mark_and_rememberset_clear(objspace, heap);
heap_move_pooled_pages_to_free_pages(heap);
}
}
else {
objspace->flags.during_minor_gc = TRUE;
objspace->marked_slots =
objspace->rgengc.old_objects + objspace->rgengc.uncollectible_wb_unprotected_objects; /* uncollectible objects are marked already */
objspace->profile.minor_gc_count++;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rgengc_rememberset_mark(objspace, SIZE_POOL_EDEN_HEAP(&size_pools[i]));
}
}
gc_mark_roots(objspace, NULL);
gc_report(1, objspace, "gc_marks_start: (%s) end, stack in %"PRIdSIZE"\n",
full_mark ? "full" : "minor", mark_stack_size(&objspace->mark_stack));
}
#if GC_ENABLE_INCREMENTAL_MARK
static inline void
gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bits)
{
if (bits) {
do {
if (bits & 1) {
gc_report(2, objspace, "gc_marks_wb_unprotected_objects: marked shady: %s\n", obj_info((VALUE)p));
GC_ASSERT(RVALUE_WB_UNPROTECTED((VALUE)p));
GC_ASSERT(RVALUE_MARKED((VALUE)p));
gc_mark_children(objspace, (VALUE)p);
}
p += BASE_SLOT_SIZE;
bits >>= 1;
} while (bits);
}
}
static void
gc_marks_wb_unprotected_objects(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *page = 0;
ccan_list_for_each(&heap->pages, page, page_node) {
bits_t *mark_bits = page->mark_bits;
bits_t *wbun_bits = page->wb_unprotected_bits;
uintptr_t p = page->start;
size_t j;
bits_t bits = mark_bits[0] & wbun_bits[0];
bits >>= NUM_IN_PAGE(p);
gc_marks_wb_unprotected_objects_plane(objspace, p, bits);
p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
for (j=1; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
bits_t bits = mark_bits[j] & wbun_bits[j];
gc_marks_wb_unprotected_objects_plane(objspace, p, bits);
p += BITS_BITLENGTH * BASE_SLOT_SIZE;
}
}
gc_mark_stacked_objects_all(objspace);
}
#endif
static void
gc_marks_finish(rb_objspace_t *objspace)
{
#if GC_ENABLE_INCREMENTAL_MARK
/* finish incremental GC */
if (is_incremental_marking(objspace)) {
if (RGENGC_CHECK_MODE && is_mark_stack_empty(&objspace->mark_stack) == 0) {
rb_bug("gc_marks_finish: mark stack is not empty (%"PRIdSIZE").",
mark_stack_size(&objspace->mark_stack));
}
gc_mark_roots(objspace, 0);
while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == false);
#if RGENGC_CHECK_MODE >= 2
if (gc_verify_heap_pages(objspace) != 0) {
rb_bug("gc_marks_finish (incremental): there are remembered old objects.");
}
#endif
objspace->flags.during_incremental_marking = FALSE;
/* check children of all marked wb-unprotected objects */
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
gc_marks_wb_unprotected_objects(objspace, SIZE_POOL_EDEN_HEAP(&size_pools[i]));
}
}
#endif /* GC_ENABLE_INCREMENTAL_MARK */
#if RGENGC_CHECK_MODE >= 2
gc_verify_internal_consistency(objspace);
#endif
if (is_full_marking(objspace)) {
/* See the comment about RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR */
const double r = gc_params.oldobject_limit_factor;
objspace->rgengc.uncollectible_wb_unprotected_objects_limit = (size_t)(objspace->rgengc.uncollectible_wb_unprotected_objects * r);
objspace->rgengc.old_objects_limit = (size_t)(objspace->rgengc.old_objects * r);
}
#if RGENGC_CHECK_MODE >= 4
during_gc = FALSE;
gc_marks_check(objspace, gc_check_after_marks_i, "after_marks");
during_gc = TRUE;
#endif
{
/* decide full GC is needed or not */
size_t total_slots = heap_allocatable_slots(objspace) + heap_eden_total_slots(objspace);
size_t sweep_slots = total_slots - objspace->marked_slots; /* will be swept slots */
size_t max_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_max_ratio);
size_t min_free_slots = (size_t)(total_slots * gc_params.heap_free_slots_min_ratio);
int full_marking = is_full_marking(objspace);
const int r_cnt = GET_VM()->ractor.cnt;
const int r_mul = r_cnt > 8 ? 8 : r_cnt; // upto 8
GC_ASSERT(heap_eden_total_slots(objspace) >= objspace->marked_slots);
/* setup free-able page counts */
if (max_free_slots < gc_params.heap_init_slots * r_mul) {
max_free_slots = gc_params.heap_init_slots * r_mul;
}
if (sweep_slots > max_free_slots) {
heap_pages_freeable_pages = (sweep_slots - max_free_slots) / HEAP_PAGE_OBJ_LIMIT;
}
else {
heap_pages_freeable_pages = 0;
}
/* check free_min */
if (min_free_slots < gc_params.heap_free_slots * r_mul) {
min_free_slots = gc_params.heap_free_slots * r_mul;
}
if (sweep_slots < min_free_slots) {
if (!full_marking) {
if (objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) {
full_marking = TRUE;
/* do not update last_major_gc, because full marking is not done. */
/* goto increment; */
}
else {
gc_report(1, objspace, "gc_marks_finish: next is full GC!!)\n");
objspace->rgengc.need_major_gc |= GPR_FLAG_MAJOR_BY_NOFREE;
}
}
#if !USE_RVARGC
if (full_marking) {
/* increment: */
gc_report(1, objspace, "gc_marks_finish: heap_set_increment!!\n");
rb_size_pool_t *size_pool = &size_pools[0];
size_pool_allocatable_pages_set(objspace, size_pool, heap_extend_pages(objspace, size_pool, sweep_slots, total_slots, heap_allocated_pages + heap_allocatable_pages(objspace)));
heap_increment(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool));
}
#endif
}
if (full_marking) {
/* See the comment about RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR */
const double r = gc_params.oldobject_limit_factor;
objspace->rgengc.uncollectible_wb_unprotected_objects_limit = (size_t)(objspace->rgengc.uncollectible_wb_unprotected_objects * r);
objspace->rgengc.old_objects_limit = (size_t)(objspace->rgengc.old_objects * r);
}
if (objspace->rgengc.uncollectible_wb_unprotected_objects > objspace->rgengc.uncollectible_wb_unprotected_objects_limit) {
objspace->rgengc.need_major_gc |= GPR_FLAG_MAJOR_BY_SHADY;
}
if (objspace->rgengc.old_objects > objspace->rgengc.old_objects_limit) {
objspace->rgengc.need_major_gc |= GPR_FLAG_MAJOR_BY_OLDGEN;
}
if (RGENGC_FORCE_MAJOR_GC) {
objspace->rgengc.need_major_gc = GPR_FLAG_MAJOR_BY_FORCE;
}
gc_report(1, objspace, "gc_marks_finish (marks %"PRIdSIZE" objects, "
"old %"PRIdSIZE" objects, total %"PRIdSIZE" slots, "
"sweep %"PRIdSIZE" slots, increment: %"PRIdSIZE", next GC: %s)\n",
objspace->marked_slots, objspace->rgengc.old_objects, heap_eden_total_slots(objspace), sweep_slots, heap_allocatable_pages(objspace),
objspace->rgengc.need_major_gc ? "major" : "minor");
}
rb_transient_heap_finish_marking();
rb_ractor_finish_marking();
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_MARK, 0);
}
#if GC_ENABLE_INCREMENTAL_MARK
static void
gc_marks_step(rb_objspace_t *objspace, size_t slots)
{
GC_ASSERT(is_marking(objspace));
if (gc_mark_stacked_objects_incremental(objspace, slots)) {
gc_marks_finish(objspace);
gc_sweep(objspace);
}
if (0) fprintf(stderr, "objspace->marked_slots: %"PRIdSIZE"\n", objspace->marked_slots);
}
#endif
static bool
gc_compact_heap_cursors_met_p(rb_heap_t *heap)
{
return heap->sweeping_page == heap->compact_cursor;
}
static rb_size_pool_t *
gc_compact_destination_pool(rb_objspace_t *objspace, rb_size_pool_t *src_pool, VALUE src)
{
size_t obj_size;
switch (BUILTIN_TYPE(src)) {
case T_ARRAY:
obj_size = rb_ary_size_as_embedded(src);
break;
case T_OBJECT:
obj_size = rb_obj_embedded_size(ROBJECT_NUMIV(src));
break;
case T_STRING:
obj_size = rb_str_size_as_embedded(src);
break;
default:
return src_pool;
}
if (rb_gc_size_allocatable_p(obj_size)){
return &size_pools[size_pool_idx_for_size(obj_size)];
}
else {
return &size_pools[0];
}
}
static bool
gc_compact_move(rb_objspace_t *objspace, rb_heap_t *heap, rb_size_pool_t *size_pool, VALUE src)
{
GC_ASSERT(BUILTIN_TYPE(src) != T_MOVED);
rb_heap_t *dheap = SIZE_POOL_EDEN_HEAP(gc_compact_destination_pool(objspace, size_pool, src));
if (gc_compact_heap_cursors_met_p(dheap)) {
return dheap != heap;
}
while (!try_move(objspace, dheap, dheap->free_pages, src)) {
struct gc_sweep_context ctx = {
.page = dheap->sweeping_page,
.final_slots = 0,
.freed_slots = 0,
.empty_slots = 0,
};
/* The page of src could be partially compacted, so it may contain
* T_MOVED. Sweeping a page may read objects on this page, so we
* need to lock the page. */
lock_page_body(objspace, GET_PAGE_BODY(src));
gc_sweep_page(objspace, dheap, &ctx);
unlock_page_body(objspace, GET_PAGE_BODY(src));
if (dheap->sweeping_page->free_slots > 0) {
heap_add_freepage(dheap, dheap->sweeping_page);
};
dheap->sweeping_page = ccan_list_next(&dheap->pages, dheap->sweeping_page, page_node);
if (gc_compact_heap_cursors_met_p(dheap)) {
return false;
}
}
return true;
}
static bool
gc_compact_plane(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap, uintptr_t p, bits_t bitset, struct heap_page *page)
{
short slot_size = page->slot_size;
short slot_bits = slot_size / BASE_SLOT_SIZE;
GC_ASSERT(slot_bits > 0);
do {
VALUE vp = (VALUE)p;
GC_ASSERT(vp % sizeof(RVALUE) == 0);
if (bitset & 1) {
objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++;
if (!gc_compact_move(objspace, heap, size_pool, vp)) {
//the cursors met. bubble up
return false;
}
}
p += slot_size;
bitset >>= slot_bits;
} while (bitset);
return true;
}
// Iterate up all the objects in page, moving them to where they want to go
static bool
gc_compact_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap, struct heap_page *page)
{
GC_ASSERT(page == heap->compact_cursor);
bits_t *mark_bits, *pin_bits;
bits_t bitset;
uintptr_t p = page->start;
mark_bits = page->mark_bits;
pin_bits = page->pinned_bits;
// objects that can be moved are marked and not pinned
bitset = (mark_bits[0] & ~pin_bits[0]);
bitset >>= NUM_IN_PAGE(p);
if (bitset) {
if (!gc_compact_plane(objspace, size_pool, heap, (uintptr_t)p, bitset, page))
return false;
}
p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
for (int j = 1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
bitset = (mark_bits[j] & ~pin_bits[j]);
if (bitset) {
if (!gc_compact_plane(objspace, size_pool, heap, (uintptr_t)p, bitset, page))
return false;
}
p += BITS_BITLENGTH * BASE_SLOT_SIZE;
}
return true;
}
static bool
gc_compact_all_compacted_p(rb_objspace_t *objspace)
{
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
if (heap->total_pages > 0 &&
!gc_compact_heap_cursors_met_p(heap)) {
return false;
}
}
return true;
}
static void
gc_sweep_compact(rb_objspace_t *objspace)
{
gc_compact_start(objspace);
#if RGENGC_CHECK_MODE >= 2
gc_verify_internal_consistency(objspace);
#endif
while (!gc_compact_all_compacted_p(objspace)) {
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
if (gc_compact_heap_cursors_met_p(heap)) {
continue;
}
struct heap_page *start_page = heap->compact_cursor;
if (!gc_compact_page(objspace, size_pool, heap, start_page)) {
lock_page_body(objspace, GET_PAGE_BODY(start_page->start));
continue;
}
// If we get here, we've finished moving all objects on the compact_cursor page
// So we can lock it and move the cursor on to the next one.
lock_page_body(objspace, GET_PAGE_BODY(start_page->start));
heap->compact_cursor = ccan_list_prev(&heap->pages, heap->compact_cursor, page_node);
}
}
gc_compact_finish(objspace);
#if RGENGC_CHECK_MODE >= 2
gc_verify_internal_consistency(objspace);
#endif
}
static void
gc_marks_rest(rb_objspace_t *objspace)
{
gc_report(1, objspace, "gc_marks_rest\n");
#if GC_ENABLE_INCREMENTAL_MARK
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
SIZE_POOL_EDEN_HEAP(&size_pools[i])->pooled_pages = NULL;
}
#endif
if (is_incremental_marking(objspace)) {
while (gc_mark_stacked_objects_incremental(objspace, INT_MAX) == FALSE);
}
else {
gc_mark_stacked_objects_all(objspace);
}
gc_marks_finish(objspace);
/* move to sweep */
gc_sweep(objspace);
}
static void
gc_marks_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
{
GC_ASSERT(dont_gc_val() == FALSE);
#if GC_ENABLE_INCREMENTAL_MARK
unsigned int lock_lev;
gc_enter(objspace, gc_enter_event_mark_continue, &lock_lev);
if (heap->free_pages) {
gc_report(2, objspace, "gc_marks_continue: has pooled pages");
gc_marks_step(objspace, objspace->rincgc.step_slots);
}
else {
gc_report(2, objspace, "gc_marks_continue: no more pooled pages (stack depth: %"PRIdSIZE").\n",
mark_stack_size(&objspace->mark_stack));
gc_marks_rest(objspace);
}
gc_exit(objspace, gc_enter_event_mark_continue, &lock_lev);
#endif
}
static void
gc_marks(rb_objspace_t *objspace, int full_mark)
{
gc_prof_mark_timer_start(objspace);
/* setup marking */
gc_marks_start(objspace, full_mark);
if (!is_incremental_marking(objspace)) {
gc_marks_rest(objspace);
}
#if RGENGC_PROFILE > 0
if (gc_prof_record(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->old_objects = objspace->rgengc.old_objects;
}
#endif
gc_prof_mark_timer_stop(objspace);
}
/* RGENGC */
static void
gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...)
{
if (level <= RGENGC_DEBUG) {
char buf[1024];
FILE *out = stderr;
va_list args;
const char *status = " ";
if (during_gc) {
status = is_full_marking(objspace) ? "+" : "-";
}
else {
if (is_lazy_sweeping(objspace)) {
status = "S";
}
if (is_incremental_marking(objspace)) {
status = "M";
}
}
va_start(args, fmt);
vsnprintf(buf, 1024, fmt, args);
va_end(args);
fprintf(out, "%s|", status);
fputs(buf, out);
}
}
/* bit operations */
static int
rgengc_remembersetbits_get(rb_objspace_t *objspace, VALUE obj)
{
return RVALUE_REMEMBERED(obj);
}
static int
rgengc_remembersetbits_set(rb_objspace_t *objspace, VALUE obj)
{
struct heap_page *page = GET_HEAP_PAGE(obj);
bits_t *bits = &page->marking_bits[0];
GC_ASSERT(!is_incremental_marking(objspace));
if (MARKED_IN_BITMAP(bits, obj)) {
return FALSE;
}
else {
page->flags.has_remembered_objects = TRUE;
MARK_IN_BITMAP(bits, obj);
return TRUE;
}
}
/* wb, etc */
/* return FALSE if already remembered */
static int
rgengc_remember(rb_objspace_t *objspace, VALUE obj)
{
gc_report(6, objspace, "rgengc_remember: %s %s\n", obj_info(obj),
rgengc_remembersetbits_get(objspace, obj) ? "was already remembered" : "is remembered now");
check_rvalue_consistency(obj);
if (RGENGC_CHECK_MODE) {
if (RVALUE_WB_UNPROTECTED(obj)) rb_bug("rgengc_remember: %s is not wb protected.", obj_info(obj));
}
#if RGENGC_PROFILE > 0
if (!rgengc_remembered(objspace, obj)) {
if (RVALUE_WB_UNPROTECTED(obj) == 0) {
objspace->profile.total_remembered_normal_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.remembered_normal_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
}
}
#endif /* RGENGC_PROFILE > 0 */
return rgengc_remembersetbits_set(objspace, obj);
}
static int
rgengc_remembered_sweep(rb_objspace_t *objspace, VALUE obj)
{
int result = rgengc_remembersetbits_get(objspace, obj);
check_rvalue_consistency(obj);
return result;
}
static int
rgengc_remembered(rb_objspace_t *objspace, VALUE obj)
{
gc_report(6, objspace, "rgengc_remembered: %s\n", obj_info(obj));
return rgengc_remembered_sweep(objspace, obj);
}
#ifndef PROFILE_REMEMBERSET_MARK
#define PROFILE_REMEMBERSET_MARK 0
#endif
static inline void
rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitset)
{
if (bitset) {
do {
if (bitset & 1) {
VALUE obj = (VALUE)p;
gc_report(2, objspace, "rgengc_rememberset_mark: mark %s\n", obj_info(obj));
GC_ASSERT(RVALUE_UNCOLLECTIBLE(obj));
GC_ASSERT(RVALUE_OLD_P(obj) || RVALUE_WB_UNPROTECTED(obj));
gc_mark_children(objspace, obj);
}
p += BASE_SLOT_SIZE;
bitset >>= 1;
} while (bitset);
}
}
static void
rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap)
{
size_t j;
struct heap_page *page = 0;
#if PROFILE_REMEMBERSET_MARK
int has_old = 0, has_shady = 0, has_both = 0, skip = 0;
#endif
gc_report(1, objspace, "rgengc_rememberset_mark: start\n");
ccan_list_for_each(&heap->pages, page, page_node) {
if (page->flags.has_remembered_objects | page->flags.has_uncollectible_shady_objects) {
uintptr_t p = page->start;
bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t *marking_bits = page->marking_bits;
bits_t *uncollectible_bits = page->uncollectible_bits;
bits_t *wb_unprotected_bits = page->wb_unprotected_bits;
#if PROFILE_REMEMBERSET_MARK
if (page->flags.has_remembered_objects && page->flags.has_uncollectible_shady_objects) has_both++;
else if (page->flags.has_remembered_objects) has_old++;
else if (page->flags.has_uncollectible_shady_objects) has_shady++;
#endif
for (j=0; j<HEAP_PAGE_BITMAP_LIMIT; j++) {
bits[j] = marking_bits[j] | (uncollectible_bits[j] & wb_unprotected_bits[j]);
marking_bits[j] = 0;
}
page->flags.has_remembered_objects = FALSE;
bitset = bits[0];
bitset >>= NUM_IN_PAGE(p);
rgengc_rememberset_mark_plane(objspace, p, bitset);
p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE;
for (j=1; j < HEAP_PAGE_BITMAP_LIMIT; j++) {
bitset = bits[j];
rgengc_rememberset_mark_plane(objspace, p, bitset);
p += BITS_BITLENGTH * BASE_SLOT_SIZE;
}
}
#if PROFILE_REMEMBERSET_MARK
else {
skip++;
}
#endif
}
#if PROFILE_REMEMBERSET_MARK
fprintf(stderr, "%d\t%d\t%d\t%d\n", has_both, has_old, has_shady, skip);
#endif
gc_report(1, objspace, "rgengc_rememberset_mark: finished\n");
}
static void
rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *page = 0;
ccan_list_for_each(&heap->pages, page, page_node) {
memset(&page->mark_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
memset(&page->uncollectible_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
memset(&page->marking_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
memset(&page->pinned_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
page->flags.has_uncollectible_shady_objects = FALSE;
page->flags.has_remembered_objects = FALSE;
}
}
/* RGENGC: APIs */
NOINLINE(static void gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace));
static void
gc_writebarrier_generational(VALUE a, VALUE b, rb_objspace_t *objspace)
{
if (RGENGC_CHECK_MODE) {
if (!RVALUE_OLD_P(a)) rb_bug("gc_writebarrier_generational: %s is not an old object.", obj_info(a));
if ( RVALUE_OLD_P(b)) rb_bug("gc_writebarrier_generational: %s is an old object.", obj_info(b));
if (is_incremental_marking(objspace)) rb_bug("gc_writebarrier_generational: called while incremental marking: %s -> %s", obj_info(a), obj_info(b));
}
#if 1
/* mark `a' and remember (default behavior) */
if (!rgengc_remembered(objspace, a)) {
RB_VM_LOCK_ENTER_NO_BARRIER();
{
rgengc_remember(objspace, a);
}
RB_VM_LOCK_LEAVE_NO_BARRIER();
gc_report(1, objspace, "gc_writebarrier_generational: %s (remembered) -> %s\n", obj_info(a), obj_info(b));
}
#else
/* mark `b' and remember */
MARK_IN_BITMAP(GET_HEAP_MARK_BITS(b), b);
if (RVALUE_WB_UNPROTECTED(b)) {
gc_remember_unprotected(objspace, b);
}
else {
RVALUE_AGE_SET_OLD(objspace, b);
rgengc_remember(objspace, b);
}
gc_report(1, objspace, "gc_writebarrier_generational: %s -> %s (remembered)\n", obj_info(a), obj_info(b));
#endif
check_rvalue_consistency(a);
check_rvalue_consistency(b);
}
#if GC_ENABLE_INCREMENTAL_MARK
static void
gc_mark_from(rb_objspace_t *objspace, VALUE obj, VALUE parent)
{
gc_mark_set_parent(objspace, parent);
rgengc_check_relation(objspace, obj);
if (gc_mark_set(objspace, obj) == FALSE) return;
gc_aging(objspace, obj);
gc_grey(objspace, obj);
}
NOINLINE(static void gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace));
static void
gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace)
{
gc_report(2, objspace, "gc_writebarrier_incremental: [LG] %p -> %s\n", (void *)a, obj_info(b));
if (RVALUE_BLACK_P(a)) {
if (RVALUE_WHITE_P(b)) {
if (!RVALUE_WB_UNPROTECTED(a)) {
gc_report(2, objspace, "gc_writebarrier_incremental: [IN] %p -> %s\n", (void *)a, obj_info(b));
gc_mark_from(objspace, b, a);
}
}
else if (RVALUE_OLD_P(a) && !RVALUE_OLD_P(b)) {
if (!RVALUE_WB_UNPROTECTED(b)) {
gc_report(1, objspace, "gc_writebarrier_incremental: [GN] %p -> %s\n", (void *)a, obj_info(b));
RVALUE_AGE_SET_OLD(objspace, b);
if (RVALUE_BLACK_P(b)) {
gc_grey(objspace, b);
}
}
else {
gc_report(1, objspace, "gc_writebarrier_incremental: [LL] %p -> %s\n", (void *)a, obj_info(b));
gc_remember_unprotected(objspace, b);
}
}
if (UNLIKELY(objspace->flags.during_compacting)) {
MARK_IN_BITMAP(GET_HEAP_PINNED_BITS(b), b);
}
}
}
#else
#define gc_writebarrier_incremental(a, b, objspace)
#endif
void
rb_gc_writebarrier(VALUE a, VALUE b)
{
rb_objspace_t *objspace = &rb_objspace;
if (RGENGC_CHECK_MODE && SPECIAL_CONST_P(a)) rb_bug("rb_gc_writebarrier: a is special const");
if (RGENGC_CHECK_MODE && SPECIAL_CONST_P(b)) rb_bug("rb_gc_writebarrier: b is special const");
retry:
if (!is_incremental_marking(objspace)) {
if (!RVALUE_OLD_P(a) || RVALUE_OLD_P(b)) {
// do nothing
}
else {
gc_writebarrier_generational(a, b, objspace);
}
}
else {
bool retry = false;
/* slow path */
RB_VM_LOCK_ENTER_NO_BARRIER();
{
if (is_incremental_marking(objspace)) {
gc_writebarrier_incremental(a, b, objspace);
}
else {
retry = true;
}
}
RB_VM_LOCK_LEAVE_NO_BARRIER();
if (retry) goto retry;
}
return;
}
void
rb_gc_writebarrier_unprotect(VALUE obj)
{
if (RVALUE_WB_UNPROTECTED(obj)) {
return;
}
else {
rb_objspace_t *objspace = &rb_objspace;
gc_report(2, objspace, "rb_gc_writebarrier_unprotect: %s %s\n", obj_info(obj),
rgengc_remembered(objspace, obj) ? " (already remembered)" : "");
RB_VM_LOCK_ENTER_NO_BARRIER();
{
if (RVALUE_OLD_P(obj)) {
gc_report(1, objspace, "rb_gc_writebarrier_unprotect: %s\n", obj_info(obj));
RVALUE_DEMOTE(objspace, obj);
gc_mark_set(objspace, obj);
gc_remember_unprotected(objspace, obj);
#if RGENGC_PROFILE
objspace->profile.total_shade_operation_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.shade_operation_count_types[BUILTIN_TYPE(obj)]++;
#endif /* RGENGC_PROFILE >= 2 */
#endif /* RGENGC_PROFILE */
}
else {
RVALUE_AGE_RESET(obj);
}
RB_DEBUG_COUNTER_INC(obj_wb_unprotect);
MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
}
RB_VM_LOCK_LEAVE_NO_BARRIER();
}
}
/*
* remember `obj' if needed.
*/
MJIT_FUNC_EXPORTED void
rb_gc_writebarrier_remember(VALUE obj)
{
rb_objspace_t *objspace = &rb_objspace;
gc_report(1, objspace, "rb_gc_writebarrier_remember: %s\n", obj_info(obj));
if (is_incremental_marking(objspace)) {
if (RVALUE_BLACK_P(obj)) {
gc_grey(objspace, obj);
}
}
else {
if (RVALUE_OLD_P(obj)) {
rgengc_remember(objspace, obj);
}
}
}
static st_table *rgengc_unprotect_logging_table;
static int
rgengc_unprotect_logging_exit_func_i(st_data_t key, st_data_t val, st_data_t arg)
{
fprintf(stderr, "%s\t%"PRIuVALUE"\n", (char *)key, (VALUE)val);
return ST_CONTINUE;
}
static void
rgengc_unprotect_logging_exit_func(void)
{
st_foreach(rgengc_unprotect_logging_table, rgengc_unprotect_logging_exit_func_i, 0);
}
void
rb_gc_unprotect_logging(void *objptr, const char *filename, int line)
{
VALUE obj = (VALUE)objptr;
if (rgengc_unprotect_logging_table == 0) {
rgengc_unprotect_logging_table = st_init_strtable();
atexit(rgengc_unprotect_logging_exit_func);
}
if (RVALUE_WB_UNPROTECTED(obj) == 0) {
char buff[0x100];
st_data_t cnt = 1;
char *ptr = buff;
snprintf(ptr, 0x100 - 1, "%s|%s:%d", obj_info(obj), filename, line);
if (st_lookup(rgengc_unprotect_logging_table, (st_data_t)ptr, &cnt)) {
cnt++;
}
else {
ptr = (strdup)(buff);
if (!ptr) rb_memerror();
}
st_insert(rgengc_unprotect_logging_table, (st_data_t)ptr, cnt);
}
}
void
rb_copy_wb_protected_attribute(VALUE dest, VALUE obj)
{
rb_objspace_t *objspace = &rb_objspace;
if (RVALUE_WB_UNPROTECTED(obj) && !RVALUE_WB_UNPROTECTED(dest)) {
if (!RVALUE_OLD_P(dest)) {
MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest);
RVALUE_AGE_RESET_RAW(dest);
}
else {
RVALUE_DEMOTE(objspace, dest);
}
}
check_rvalue_consistency(dest);
}
/* RGENGC analysis information */
VALUE
rb_obj_rgengc_writebarrier_protected_p(VALUE obj)
{
return RBOOL(!RVALUE_WB_UNPROTECTED(obj));
}
VALUE
rb_obj_rgengc_promoted_p(VALUE obj)
{
return RBOOL(OBJ_PROMOTED(obj));
}
size_t
rb_obj_gc_flags(VALUE obj, ID* flags, size_t max)
{
size_t n = 0;
static ID ID_marked;
static ID ID_wb_protected, ID_old, ID_marking, ID_uncollectible, ID_pinned;
if (!ID_marked) {
#define I(s) ID_##s = rb_intern(#s);
I(marked);
I(wb_protected);
I(old);
I(marking);
I(uncollectible);
I(pinned);
#undef I
}
if (RVALUE_WB_UNPROTECTED(obj) == 0 && n<max) flags[n++] = ID_wb_protected;
if (RVALUE_OLD_P(obj) && n<max) flags[n++] = ID_old;
if (RVALUE_UNCOLLECTIBLE(obj) && n<max) flags[n++] = ID_uncollectible;
if (MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj) && n<max) flags[n++] = ID_marking;
if (MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj) && n<max) flags[n++] = ID_marked;
if (MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj) && n<max) flags[n++] = ID_pinned;
return n;
}
/* GC */
void
rb_gc_ractor_newobj_cache_clear(rb_ractor_newobj_cache_t *newobj_cache)
{
#if GC_ENABLE_INCREMENTAL_MARK
newobj_cache->incremental_mark_step_allocated_slots = 0;
#endif
for (size_t size_pool_idx = 0; size_pool_idx < SIZE_POOL_COUNT; size_pool_idx++) {
rb_ractor_newobj_size_pool_cache_t *cache = &newobj_cache->size_pool_caches[size_pool_idx];
struct heap_page *page = cache->using_page;
RVALUE *freelist = cache->freelist;
RUBY_DEBUG_LOG("ractor using_page:%p freelist:%p", (void *)page, (void *)freelist);
heap_page_freelist_append(page, freelist);
cache->using_page = NULL;
cache->freelist = NULL;
}
}
void
rb_gc_force_recycle(VALUE obj)
{
/* no-op */
}
#ifndef MARK_OBJECT_ARY_BUCKET_SIZE
#define MARK_OBJECT_ARY_BUCKET_SIZE 1024
#endif
void
rb_gc_register_mark_object(VALUE obj)
{
if (!is_pointer_to_heap(&rb_objspace, (void *)obj))
return;
RB_VM_LOCK_ENTER();
{
VALUE ary_ary = GET_VM()->mark_object_ary;
VALUE ary = rb_ary_last(0, 0, ary_ary);
if (NIL_P(ary) || RARRAY_LEN(ary) >= MARK_OBJECT_ARY_BUCKET_SIZE) {
ary = rb_ary_hidden_new(MARK_OBJECT_ARY_BUCKET_SIZE);
rb_ary_push(ary_ary, ary);
}
rb_ary_push(ary, obj);
}
RB_VM_LOCK_LEAVE();
}
void
rb_gc_register_address(VALUE *addr)
{
rb_objspace_t *objspace = &rb_objspace;
struct gc_list *tmp;
tmp = ALLOC(struct gc_list);
tmp->next = global_list;
tmp->varptr = addr;
global_list = tmp;
}
void
rb_gc_unregister_address(VALUE *addr)
{
rb_objspace_t *objspace = &rb_objspace;
struct gc_list *tmp = global_list;
if (tmp->varptr == addr) {
global_list = tmp->next;
xfree(tmp);
return;
}
while (tmp->next) {
if (tmp->next->varptr == addr) {
struct gc_list *t = tmp->next;
tmp->next = tmp->next->next;
xfree(t);
break;
}
tmp = tmp->next;
}
}
void
rb_global_variable(VALUE *var)
{
rb_gc_register_address(var);
}
#define GC_NOTIFY 0
enum {
gc_stress_no_major,
gc_stress_no_immediate_sweep,
gc_stress_full_mark_after_malloc,
gc_stress_max
};
#define gc_stress_full_mark_after_malloc_p() \
(FIXNUM_P(ruby_gc_stress_mode) && (FIX2LONG(ruby_gc_stress_mode) & (1<<gc_stress_full_mark_after_malloc)))
static void
heap_ready_to_gc(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap)
{
if (!heap->free_pages) {
if (!heap_increment(objspace, size_pool, heap)) {
size_pool_allocatable_pages_set(objspace, size_pool, 1);
heap_increment(objspace, size_pool, heap);
}
}
}
static int
ready_to_gc(rb_objspace_t *objspace)
{
if (dont_gc_val() || during_gc || ruby_disable_gc) {
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
heap_ready_to_gc(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool));
}
return FALSE;
}
else {
return TRUE;
}
}
static void
gc_reset_malloc_info(rb_objspace_t *objspace, bool full_mark)
{
gc_prof_set_malloc_info(objspace);
{
size_t inc = ATOMIC_SIZE_EXCHANGE(malloc_increase, 0);
size_t old_limit = malloc_limit;
if (inc > malloc_limit) {
malloc_limit = (size_t)(inc * gc_params.malloc_limit_growth_factor);
if (malloc_limit > gc_params.malloc_limit_max) {
malloc_limit = gc_params.malloc_limit_max;
}
}
else {
malloc_limit = (size_t)(malloc_limit * 0.98); /* magic number */
if (malloc_limit < gc_params.malloc_limit_min) {
malloc_limit = gc_params.malloc_limit_min;
}
}
if (0) {
if (old_limit != malloc_limit) {
fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: %"PRIuSIZE" -> %"PRIuSIZE"\n",
rb_gc_count(), old_limit, malloc_limit);
}
else {
fprintf(stderr, "[%"PRIuSIZE"] malloc_limit: not changed (%"PRIuSIZE")\n",
rb_gc_count(), malloc_limit);
}
}
}
/* reset oldmalloc info */
#if RGENGC_ESTIMATE_OLDMALLOC
if (!full_mark) {
if (objspace->rgengc.oldmalloc_increase > objspace->rgengc.oldmalloc_increase_limit) {
objspace->rgengc.need_major_gc |= GPR_FLAG_MAJOR_BY_OLDMALLOC;
objspace->rgengc.oldmalloc_increase_limit =
(size_t)(objspace->rgengc.oldmalloc_increase_limit * gc_params.oldmalloc_limit_growth_factor);
if (objspace->rgengc.oldmalloc_increase_limit > gc_params.oldmalloc_limit_max) {
objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_max;
}
}
if (0) fprintf(stderr, "%"PRIdSIZE"\t%d\t%"PRIuSIZE"\t%"PRIuSIZE"\t%"PRIdSIZE"\n",
rb_gc_count(),
objspace->rgengc.need_major_gc,
objspace->rgengc.oldmalloc_increase,
objspace->rgengc.oldmalloc_increase_limit,
gc_params.oldmalloc_limit_max);
}
else {
/* major GC */
objspace->rgengc.oldmalloc_increase = 0;
if ((objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_BY_OLDMALLOC) == 0) {
objspace->rgengc.oldmalloc_increase_limit =
(size_t)(objspace->rgengc.oldmalloc_increase_limit / ((gc_params.oldmalloc_limit_growth_factor - 1)/10 + 1));
if (objspace->rgengc.oldmalloc_increase_limit < gc_params.oldmalloc_limit_min) {
objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
}
}
}
#endif
}
static int
garbage_collect(rb_objspace_t *objspace, unsigned int reason)
{
int ret;
RB_VM_LOCK_ENTER();
{
#if GC_PROFILE_MORE_DETAIL
objspace->profile.prepare_time = getrusage_time();
#endif
gc_rest(objspace);
#if GC_PROFILE_MORE_DETAIL
objspace->profile.prepare_time = getrusage_time() - objspace->profile.prepare_time;
#endif
ret = gc_start(objspace, reason);
}
RB_VM_LOCK_LEAVE();
return ret;
}
static int
gc_start(rb_objspace_t *objspace, unsigned int reason)
{
unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK);
#if GC_ENABLE_INCREMENTAL_MARK
unsigned int immediate_mark = reason & GPR_FLAG_IMMEDIATE_MARK;
#endif
/* reason may be clobbered, later, so keep set immediate_sweep here */
objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP);
/* Explicitly enable compaction (GC.compact) */
if (do_full_mark && ruby_enable_autocompact) {
objspace->flags.during_compacting = TRUE;
}
else {
objspace->flags.during_compacting = !!(reason & GPR_FLAG_COMPACT);
}
if (!heap_allocated_pages) return FALSE; /* heap is not ready */
if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */
GC_ASSERT(gc_mode(objspace) == gc_mode_none);
GC_ASSERT(!is_lazy_sweeping(objspace));
GC_ASSERT(!is_incremental_marking(objspace));
unsigned int lock_lev;
gc_enter(objspace, gc_enter_event_start, &lock_lev);
#if RGENGC_CHECK_MODE >= 2
gc_verify_internal_consistency(objspace);
#endif
if (ruby_gc_stressful) {
int flag = FIXNUM_P(ruby_gc_stress_mode) ? FIX2INT(ruby_gc_stress_mode) : 0;
if ((flag & (1<<gc_stress_no_major)) == 0) {
do_full_mark = TRUE;
}
objspace->flags.immediate_sweep = !(flag & (1<<gc_stress_no_immediate_sweep));
}
else {
if (objspace->rgengc.need_major_gc) {
reason |= objspace->rgengc.need_major_gc;
do_full_mark = TRUE;
}
else if (RGENGC_FORCE_MAJOR_GC) {
reason = GPR_FLAG_MAJOR_BY_FORCE;
do_full_mark = TRUE;
}
objspace->rgengc.need_major_gc = GPR_FLAG_NONE;
}
if (do_full_mark && (reason & GPR_FLAG_MAJOR_MASK) == 0) {
reason |= GPR_FLAG_MAJOR_BY_FORCE; /* GC by CAPI, METHOD, and so on. */
}
#if GC_ENABLE_INCREMENTAL_MARK
if (!GC_ENABLE_INCREMENTAL_MARK || objspace->flags.dont_incremental || immediate_mark) {
objspace->flags.during_incremental_marking = FALSE;
}
else {
objspace->flags.during_incremental_marking = do_full_mark;
}
#endif
if (!GC_ENABLE_LAZY_SWEEP || objspace->flags.dont_incremental) {
objspace->flags.immediate_sweep = TRUE;
}
if (objspace->flags.immediate_sweep) reason |= GPR_FLAG_IMMEDIATE_SWEEP;
gc_report(1, objspace, "gc_start(reason: %x) => %u, %d, %d\n",
reason,
do_full_mark, !is_incremental_marking(objspace), objspace->flags.immediate_sweep);
#if USE_DEBUG_COUNTER
RB_DEBUG_COUNTER_INC(gc_count);
if (reason & GPR_FLAG_MAJOR_MASK) {
(void)RB_DEBUG_COUNTER_INC_IF(gc_major_nofree, reason & GPR_FLAG_MAJOR_BY_NOFREE);
(void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldgen, reason & GPR_FLAG_MAJOR_BY_OLDGEN);
(void)RB_DEBUG_COUNTER_INC_IF(gc_major_shady, reason & GPR_FLAG_MAJOR_BY_SHADY);
(void)RB_DEBUG_COUNTER_INC_IF(gc_major_force, reason & GPR_FLAG_MAJOR_BY_FORCE);
#if RGENGC_ESTIMATE_OLDMALLOC
(void)RB_DEBUG_COUNTER_INC_IF(gc_major_oldmalloc, reason & GPR_FLAG_MAJOR_BY_OLDMALLOC);
#endif
}
else {
(void)RB_DEBUG_COUNTER_INC_IF(gc_minor_newobj, reason & GPR_FLAG_NEWOBJ);
(void)RB_DEBUG_COUNTER_INC_IF(gc_minor_malloc, reason & GPR_FLAG_MALLOC);
(void)RB_DEBUG_COUNTER_INC_IF(gc_minor_method, reason & GPR_FLAG_METHOD);
(void)RB_DEBUG_COUNTER_INC_IF(gc_minor_capi, reason & GPR_FLAG_CAPI);
(void)RB_DEBUG_COUNTER_INC_IF(gc_minor_stress, reason & GPR_FLAG_STRESS);
}
#endif
objspace->profile.count++;
objspace->profile.latest_gc_info = reason;
objspace->profile.total_allocated_objects_at_gc_start = objspace->total_allocated_objects;
objspace->profile.heap_used_at_gc_start = heap_allocated_pages;
gc_prof_setup_new_record(objspace, reason);
gc_reset_malloc_info(objspace, do_full_mark);
rb_transient_heap_start_marking(do_full_mark);
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_START, 0 /* TODO: pass minor/immediate flag? */);
GC_ASSERT(during_gc);
gc_prof_timer_start(objspace);
{
gc_marks(objspace, do_full_mark);
}
gc_prof_timer_stop(objspace);
gc_exit(objspace, gc_enter_event_start, &lock_lev);
return TRUE;
}
static void
gc_rest(rb_objspace_t *objspace)
{
int marking = is_incremental_marking(objspace);
int sweeping = is_lazy_sweeping(objspace);
if (marking || sweeping) {
unsigned int lock_lev;
gc_enter(objspace, gc_enter_event_rest, &lock_lev);
if (RGENGC_CHECK_MODE >= 2) gc_verify_internal_consistency(objspace);
if (is_incremental_marking(objspace)) {
gc_marks_rest(objspace);
}
if (is_lazy_sweeping(objspace)) {
gc_sweep_rest(objspace);
}
gc_exit(objspace, gc_enter_event_rest, &lock_lev);
}
}
struct objspace_and_reason {
rb_objspace_t *objspace;
unsigned int reason;
};
static void
gc_current_status_fill(rb_objspace_t *objspace, char *buff)
{
int i = 0;
if (is_marking(objspace)) {
buff[i++] = 'M';
if (is_full_marking(objspace)) buff[i++] = 'F';
#if GC_ENABLE_INCREMENTAL_MARK
if (is_incremental_marking(objspace)) buff[i++] = 'I';
#endif
}
else if (is_sweeping(objspace)) {
buff[i++] = 'S';
if (is_lazy_sweeping(objspace)) buff[i++] = 'L';
}
else {
buff[i++] = 'N';
}
buff[i] = '\0';
}
static const char *
gc_current_status(rb_objspace_t *objspace)
{
static char buff[0x10];
gc_current_status_fill(objspace, buff);
return buff;
}
#if PRINT_ENTER_EXIT_TICK
static tick_t last_exit_tick;
static tick_t enter_tick;
static int enter_count = 0;
static char last_gc_status[0x10];
static inline void
gc_record(rb_objspace_t *objspace, int direction, const char *event)
{
if (direction == 0) { /* enter */
enter_count++;
enter_tick = tick();
gc_current_status_fill(objspace, last_gc_status);
}
else { /* exit */
tick_t exit_tick = tick();
char current_gc_status[0x10];
gc_current_status_fill(objspace, current_gc_status);
#if 1
/* [last mutator time] [gc time] [event] */
fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
enter_tick - last_exit_tick,
exit_tick - enter_tick,
event,
last_gc_status, current_gc_status,
(objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
last_exit_tick = exit_tick;
#else
/* [enter_tick] [gc time] [event] */
fprintf(stderr, "%"PRItick"\t%"PRItick"\t%s\t[%s->%s|%c]\n",
enter_tick,
exit_tick - enter_tick,
event,
last_gc_status, current_gc_status,
(objspace->profile.latest_gc_info & GPR_FLAG_MAJOR_MASK) ? '+' : '-');
#endif
}
}
#else /* PRINT_ENTER_EXIT_TICK */
static inline void
gc_record(rb_objspace_t *objspace, int direction, const char *event)
{
/* null */
}
#endif /* PRINT_ENTER_EXIT_TICK */
static const char *
gc_enter_event_cstr(enum gc_enter_event event)
{
switch (event) {
case gc_enter_event_start: return "start";
case gc_enter_event_mark_continue: return "mark_continue";
case gc_enter_event_sweep_continue: return "sweep_continue";
case gc_enter_event_rest: return "rest";
case gc_enter_event_finalizer: return "finalizer";
case gc_enter_event_rb_memerror: return "rb_memerror";
}
return NULL;
}
static void
gc_enter_count(enum gc_enter_event event)
{
switch (event) {
case gc_enter_event_start: RB_DEBUG_COUNTER_INC(gc_enter_start); break;
case gc_enter_event_mark_continue: RB_DEBUG_COUNTER_INC(gc_enter_mark_continue); break;
case gc_enter_event_sweep_continue: RB_DEBUG_COUNTER_INC(gc_enter_sweep_continue); break;
case gc_enter_event_rest: RB_DEBUG_COUNTER_INC(gc_enter_rest); break;
case gc_enter_event_finalizer: RB_DEBUG_COUNTER_INC(gc_enter_finalizer); break;
case gc_enter_event_rb_memerror: /* nothing */ break;
}
}
#ifndef MEASURE_GC
#define MEASURE_GC (objspace->flags.measure_gc)
#endif
static bool
gc_enter_event_measure_p(rb_objspace_t *objspace, enum gc_enter_event event)
{
if (!MEASURE_GC) return false;
switch (event) {
case gc_enter_event_start:
case gc_enter_event_mark_continue:
case gc_enter_event_sweep_continue:
case gc_enter_event_rest:
return true;
default:
// case gc_enter_event_finalizer:
// case gc_enter_event_rb_memerror:
return false;
}
}
static bool current_process_time(struct timespec *ts);
static void
gc_enter_clock(rb_objspace_t *objspace, enum gc_enter_event event)
{
if (gc_enter_event_measure_p(objspace, event)) {
if (!current_process_time(&objspace->profile.start_time)) {
objspace->profile.start_time.tv_sec = 0;
objspace->profile.start_time.tv_nsec = 0;
}
}
}
static void
gc_exit_clock(rb_objspace_t *objspace, enum gc_enter_event event)
{
if (gc_enter_event_measure_p(objspace, event)) {
struct timespec end_time;
if ((objspace->profile.start_time.tv_sec > 0 ||
objspace->profile.start_time.tv_nsec > 0) &&
current_process_time(&end_time)) {
if (end_time.tv_sec < objspace->profile.start_time.tv_sec) {
return; // ignore
}
else {
uint64_t ns =
(uint64_t)(end_time.tv_sec - objspace->profile.start_time.tv_sec) * (1000 * 1000 * 1000) +
(end_time.tv_nsec - objspace->profile.start_time.tv_nsec);
objspace->profile.total_time_ns += ns;
}
}
}
}
static inline void
gc_enter(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
{
RB_VM_LOCK_ENTER_LEV(lock_lev);
gc_enter_clock(objspace, event);
switch (event) {
case gc_enter_event_rest:
if (!is_marking(objspace)) break;
// fall through
case gc_enter_event_start:
case gc_enter_event_mark_continue:
// stop other ractors
rb_vm_barrier();
break;
default:
break;
}
gc_enter_count(event);
if (UNLIKELY(during_gc != 0)) rb_bug("during_gc != 0");
if (RGENGC_CHECK_MODE >= 3) gc_verify_internal_consistency(objspace);
during_gc = TRUE;
RUBY_DEBUG_LOG("%s (%s)",gc_enter_event_cstr(event), gc_current_status(objspace));
gc_report(1, objspace, "gc_enter: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
gc_record(objspace, 0, gc_enter_event_cstr(event));
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_ENTER, 0); /* TODO: which parameter should be passed? */
}
static inline void
gc_exit(rb_objspace_t *objspace, enum gc_enter_event event, unsigned int *lock_lev)
{
GC_ASSERT(during_gc != 0);
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_EXIT, 0); /* TODO: which parameter should be passsed? */
gc_record(objspace, 1, gc_enter_event_cstr(event));
RUBY_DEBUG_LOG("%s (%s)", gc_enter_event_cstr(event), gc_current_status(objspace));
gc_report(1, objspace, "gc_exit: %s [%s]\n", gc_enter_event_cstr(event), gc_current_status(objspace));
during_gc = FALSE;
gc_exit_clock(objspace, event);
RB_VM_LOCK_LEAVE_LEV(lock_lev);
#if RGENGC_CHECK_MODE >= 2
if (event == gc_enter_event_sweep_continue && gc_mode(objspace) == gc_mode_none) {
GC_ASSERT(!during_gc);
// sweep finished
gc_verify_internal_consistency(objspace);
}
#endif
}
static void *
gc_with_gvl(void *ptr)
{
struct objspace_and_reason *oar = (struct objspace_and_reason *)ptr;
return (void *)(VALUE)garbage_collect(oar->objspace, oar->reason);
}
static int
garbage_collect_with_gvl(rb_objspace_t *objspace, unsigned int reason)
{
if (dont_gc_val()) return TRUE;
if (ruby_thread_has_gvl_p()) {
return garbage_collect(objspace, reason);
}
else {
if (ruby_native_thread_p()) {
struct objspace_and_reason oar;
oar.objspace = objspace;
oar.reason = reason;
return (int)(VALUE)rb_thread_call_with_gvl(gc_with_gvl, (void *)&oar);
}
else {
/* no ruby thread */
fprintf(stderr, "[FATAL] failed to allocate memory\n");
exit(EXIT_FAILURE);
}
}
}
static VALUE
gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE immediate_mark, VALUE immediate_sweep, VALUE compact)
{
rb_objspace_t *objspace = &rb_objspace;
unsigned int reason = (GPR_FLAG_FULL_MARK |
GPR_FLAG_IMMEDIATE_MARK |
GPR_FLAG_IMMEDIATE_SWEEP |
GPR_FLAG_METHOD);
/* For now, compact implies full mark / sweep, so ignore other flags */
if (RTEST(compact)) {
GC_ASSERT(GC_COMPACTION_SUPPORTED);
reason |= GPR_FLAG_COMPACT;
}
else {
if (!RTEST(full_mark)) reason &= ~GPR_FLAG_FULL_MARK;
if (!RTEST(immediate_mark)) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
if (!RTEST(immediate_sweep)) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
}
garbage_collect(objspace, reason);
gc_finalize_deferred(objspace);
return Qnil;
}
static int
gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
{
GC_ASSERT(!SPECIAL_CONST_P(obj));
switch (BUILTIN_TYPE(obj)) {
case T_NONE:
case T_NIL:
case T_MOVED:
case T_ZOMBIE:
return FALSE;
case T_SYMBOL:
if (DYNAMIC_SYM_P(obj) && (RSYMBOL(obj)->id & ~ID_SCOPE_MASK)) {
return FALSE;
}
/* fall through */
case T_STRING:
case T_OBJECT:
case T_FLOAT:
case T_IMEMO:
case T_ARRAY:
case T_BIGNUM:
case T_ICLASS:
case T_MODULE:
case T_REGEXP:
case T_DATA:
case T_MATCH:
case T_STRUCT:
case T_HASH:
case T_FILE:
case T_COMPLEX:
case T_RATIONAL:
case T_NODE:
case T_CLASS:
if (FL_TEST(obj, FL_FINALIZE)) {
/* The finalizer table is a numtable. It looks up objects by address.
* We can't mark the keys in the finalizer table because that would
* prevent the objects from being collected. This check prevents
* objects that are keys in the finalizer table from being moved
* without directly pinning them. */
if (st_is_member(finalizer_table, obj)) {
return FALSE;
}
}
GC_ASSERT(RVALUE_MARKED(obj));
GC_ASSERT(!RVALUE_PINNED(obj));
return TRUE;
default:
rb_bug("gc_is_moveable_obj: unreachable (%d)", (int)BUILTIN_TYPE(obj));
break;
}
return FALSE;
}
static VALUE
gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, size_t slot_size)
{
int marked;
int wb_unprotected;
int uncollectible;
int marking;
RVALUE *dest = (RVALUE *)free;
RVALUE *src = (RVALUE *)scan;
gc_report(4, objspace, "Moving object: %p -> %p\n", (void*)scan, (void *)free);
GC_ASSERT(BUILTIN_TYPE(scan) != T_NONE);
GC_ASSERT(!MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(free), free));
/* Save off bits for current object. */
marked = rb_objspace_marked_object_p((VALUE)src);
wb_unprotected = RVALUE_WB_UNPROTECTED((VALUE)src);
uncollectible = RVALUE_UNCOLLECTIBLE((VALUE)src);
marking = RVALUE_MARKING((VALUE)src);
/* Clear bits for eventual T_MOVED */
CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS((VALUE)src), (VALUE)src);
CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS((VALUE)src), (VALUE)src);
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS((VALUE)src), (VALUE)src);
CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS((VALUE)src), (VALUE)src);
if (FL_TEST((VALUE)src, FL_EXIVAR)) {
/* Same deal as below. Generic ivars are held in st tables.
* Resizing the table could cause a GC to happen and we can't allow it */
VALUE already_disabled = rb_gc_disable_no_rest();
rb_mv_generic_ivar((VALUE)src, (VALUE)dest);
if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
}
st_data_t srcid = (st_data_t)src, id;
/* If the source object's object_id has been seen, we need to update
* the object to object id mapping. */
if (st_lookup(objspace->obj_to_id_tbl, srcid, &id)) {
gc_report(4, objspace, "Moving object with seen id: %p -> %p\n", (void *)src, (void *)dest);
/* inserting in the st table can cause the GC to run. We need to
* prevent re-entry in to the GC since `gc_move` is running in the GC,
* so temporarily disable the GC around the st table mutation */
VALUE already_disabled = rb_gc_disable_no_rest();
st_delete(objspace->obj_to_id_tbl, &srcid, 0);
st_insert(objspace->obj_to_id_tbl, (st_data_t)dest, id);
if (already_disabled == Qfalse) rb_objspace_gc_enable(objspace);
}
/* Move the object */
memcpy(dest, src, MIN(src_slot_size, slot_size));
memset(src, 0, src_slot_size);
/* Set bits for object in new location */
if (marking) {
MARK_IN_BITMAP(GET_HEAP_MARKING_BITS((VALUE)dest), (VALUE)dest);
}
else {
CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS((VALUE)dest), (VALUE)dest);
}
if (marked) {
MARK_IN_BITMAP(GET_HEAP_MARK_BITS((VALUE)dest), (VALUE)dest);
}
else {
CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS((VALUE)dest), (VALUE)dest);
}
if (wb_unprotected) {
MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS((VALUE)dest), (VALUE)dest);
}
else {
CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS((VALUE)dest), (VALUE)dest);
}
if (uncollectible) {
MARK_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS((VALUE)dest), (VALUE)dest);
}
else {
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS((VALUE)dest), (VALUE)dest);
}
/* Assign forwarding address */
src->as.moved.flags = T_MOVED;
src->as.moved.dummy = Qundef;
src->as.moved.destination = (VALUE)dest;
GC_ASSERT(BUILTIN_TYPE((VALUE)dest) != T_NONE);
return (VALUE)src;
}
#if GC_CAN_COMPILE_COMPACTION
static int
compare_free_slots(const void *left, const void *right, void *dummy)
{
struct heap_page *left_page;
struct heap_page *right_page;
left_page = *(struct heap_page * const *)left;
right_page = *(struct heap_page * const *)right;
return left_page->free_slots - right_page->free_slots;
}
static void
gc_sort_heap_by_empty_slots(rb_objspace_t *objspace)
{
for (int j = 0; j < SIZE_POOL_COUNT; j++) {
rb_size_pool_t *size_pool = &size_pools[j];
size_t total_pages = SIZE_POOL_EDEN_HEAP(size_pool)->total_pages;
size_t size = size_mul_or_raise(total_pages, sizeof(struct heap_page *), rb_eRuntimeError);
struct heap_page *page = 0, **page_list = malloc(size);
size_t i = 0;
SIZE_POOL_EDEN_HEAP(size_pool)->free_pages = NULL;
ccan_list_for_each(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, page, page_node) {
page_list[i++] = page;
GC_ASSERT(page);
}
GC_ASSERT((size_t)i == total_pages);
/* Sort the heap so "filled pages" are first. `heap_add_page` adds to the
* head of the list, so empty pages will end up at the start of the heap */
ruby_qsort(page_list, total_pages, sizeof(struct heap_page *), compare_free_slots, NULL);
/* Reset the eden heap */
ccan_list_head_init(&SIZE_POOL_EDEN_HEAP(size_pool)->pages);
for (i = 0; i < total_pages; i++) {
ccan_list_add(&SIZE_POOL_EDEN_HEAP(size_pool)->pages, &page_list[i]->page_node);
if (page_list[i]->free_slots != 0) {
heap_add_freepage(SIZE_POOL_EDEN_HEAP(size_pool), page_list[i]);
}
}
free(page_list);
}
}
#endif
static void
gc_ref_update_array(rb_objspace_t * objspace, VALUE v)
{
if (ARY_SHARED_P(v)) {
#if USE_RVARGC
VALUE old_root = RARRAY(v)->as.heap.aux.shared_root;
#endif
UPDATE_IF_MOVED(objspace, RARRAY(v)->as.heap.aux.shared_root);
#if USE_RVARGC
VALUE new_root = RARRAY(v)->as.heap.aux.shared_root;
// If the root is embedded and its location has changed
if (ARY_EMBED_P(new_root) && new_root != old_root) {
size_t offset = (size_t)(RARRAY(v)->as.heap.ptr - RARRAY(old_root)->as.ary);
GC_ASSERT(RARRAY(v)->as.heap.ptr >= RARRAY(old_root)->as.ary);
RARRAY(v)->as.heap.ptr = RARRAY(new_root)->as.ary + offset;
}
#endif
}
else {
long len = RARRAY_LEN(v);
if (len > 0) {
VALUE *ptr = (VALUE *)RARRAY_CONST_PTR_TRANSIENT(v);
for (long i = 0; i < len; i++) {
UPDATE_IF_MOVED(objspace, ptr[i]);
}
}
#if USE_RVARGC
if ((size_t)GET_HEAP_PAGE(v)->slot_size >= rb_ary_size_as_embedded(v)) {
if (rb_ary_embeddable_p(v)) {
rb_ary_make_embedded(v);
}
}
#endif
}
}
static void
gc_ref_update_object(rb_objspace_t *objspace, VALUE v)
{
VALUE *ptr = ROBJECT_IVPTR(v);
uint32_t numiv = ROBJECT_NUMIV(v);
#if USE_RVARGC
size_t slot_size = rb_gc_obj_slot_size(v);
size_t embed_size = rb_obj_embedded_size(numiv);
if (slot_size >= embed_size && !RB_FL_TEST_RAW(v, ROBJECT_EMBED)) {
// Object can be re-embedded
memcpy(ROBJECT(v)->as.ary, ptr, sizeof(VALUE) * numiv);
RB_FL_SET_RAW(v, ROBJECT_EMBED);
if (ROBJ_TRANSIENT_P(v)) {
ROBJ_TRANSIENT_UNSET(v);
}
else {
xfree(ptr);
}
ptr = ROBJECT(v)->as.ary;
uint32_t capa = (uint32_t)((slot_size - offsetof(struct RObject, as.ary)) / sizeof(VALUE));
ROBJECT(v)->numiv = capa;
// Fill end with Qundef
for (uint32_t i = numiv; i < capa; i++) {
ptr[i] = Qundef;
}
}
#endif
for (uint32_t i = 0; i < ROBJECT_IV_COUNT(v); i++) {
UPDATE_IF_MOVED(objspace, ptr[i]);
}
}
static int
hash_replace_ref(st_data_t *key, st_data_t *value, st_data_t argp, int existing)
{
rb_objspace_t *objspace = (rb_objspace_t *)argp;
if (gc_object_moved_p(objspace, (VALUE)*key)) {
*key = rb_gc_location((VALUE)*key);
}
if (gc_object_moved_p(objspace, (VALUE)*value)) {
*value = rb_gc_location((VALUE)*value);
}
return ST_CONTINUE;
}
static int
hash_foreach_replace(st_data_t key, st_data_t value, st_data_t argp, int error)
{
rb_objspace_t *objspace;
objspace = (rb_objspace_t *)argp;
if (gc_object_moved_p(objspace, (VALUE)key)) {
return ST_REPLACE;
}
if (gc_object_moved_p(objspace, (VALUE)value)) {
return ST_REPLACE;
}
return ST_CONTINUE;
}
static int
hash_replace_ref_value(st_data_t *key, st_data_t *value, st_data_t argp, int existing)
{
rb_objspace_t *objspace = (rb_objspace_t *)argp;
if (gc_object_moved_p(objspace, (VALUE)*value)) {
*value = rb_gc_location((VALUE)*value);
}
return ST_CONTINUE;
}
static int
hash_foreach_replace_value(st_data_t key, st_data_t value, st_data_t argp, int error)
{
rb_objspace_t *objspace;
objspace = (rb_objspace_t *)argp;
if (gc_object_moved_p(objspace, (VALUE)value)) {
return ST_REPLACE;
}
return ST_CONTINUE;
}
static void
gc_update_tbl_refs(rb_objspace_t * objspace, st_table *tbl)
{
if (!tbl || tbl->num_entries == 0) return;
if (st_foreach_with_replace(tbl, hash_foreach_replace_value, hash_replace_ref_value, (st_data_t)objspace)) {
rb_raise(rb_eRuntimeError, "hash modified during iteration");
}
}
static void
gc_update_table_refs(rb_objspace_t * objspace, st_table *tbl)
{
if (!tbl || tbl->num_entries == 0) return;
if (st_foreach_with_replace(tbl, hash_foreach_replace, hash_replace_ref, (st_data_t)objspace)) {
rb_raise(rb_eRuntimeError, "hash modified during iteration");
}
}
/* Update MOVED references in an st_table */
void
rb_gc_update_tbl_refs(st_table *ptr)
{
rb_objspace_t *objspace = &rb_objspace;
gc_update_table_refs(objspace, ptr);
}
static void
gc_ref_update_hash(rb_objspace_t * objspace, VALUE v)
{
rb_hash_stlike_foreach_with_replace(v, hash_foreach_replace, hash_replace_ref, (st_data_t)objspace);
}
static void
gc_ref_update_method_entry(rb_objspace_t *objspace, rb_method_entry_t *me)
{
rb_method_definition_t *def = me->def;
UPDATE_IF_MOVED(objspace, me->owner);
UPDATE_IF_MOVED(objspace, me->defined_class);
if (def) {
switch (def->type) {
case VM_METHOD_TYPE_ISEQ:
if (def->body.iseq.iseqptr) {
TYPED_UPDATE_IF_MOVED(objspace, rb_iseq_t *, def->body.iseq.iseqptr);
}
TYPED_UPDATE_IF_MOVED(objspace, rb_cref_t *, def->body.iseq.cref);
break;
case VM_METHOD_TYPE_ATTRSET:
case VM_METHOD_TYPE_IVAR:
UPDATE_IF_MOVED(objspace, def->body.attr.location);
break;
case VM_METHOD_TYPE_BMETHOD:
UPDATE_IF_MOVED(objspace, def->body.bmethod.proc);
break;
case VM_METHOD_TYPE_ALIAS:
TYPED_UPDATE_IF_MOVED(objspace, struct rb_method_entry_struct *, def->body.alias.original_me);
return;
case VM_METHOD_TYPE_REFINED:
TYPED_UPDATE_IF_MOVED(objspace, struct rb_method_entry_struct *, def->body.refined.orig_me);
UPDATE_IF_MOVED(objspace, def->body.refined.owner);
break;
case VM_METHOD_TYPE_CFUNC:
case VM_METHOD_TYPE_ZSUPER:
case VM_METHOD_TYPE_MISSING:
case VM_METHOD_TYPE_OPTIMIZED:
case VM_METHOD_TYPE_UNDEF:
case VM_METHOD_TYPE_NOTIMPLEMENTED:
break;
}
}
}
static void
gc_update_values(rb_objspace_t *objspace, long n, VALUE *values)
{
long i;
for (i=0; i<n; i++) {
UPDATE_IF_MOVED(objspace, values[i]);
}
}
static void
gc_ref_update_imemo(rb_objspace_t *objspace, VALUE obj)
{
switch (imemo_type(obj)) {
case imemo_env:
{
rb_env_t *env = (rb_env_t *)obj;
if (LIKELY(env->ep)) {
// just after newobj() can be NULL here.
TYPED_UPDATE_IF_MOVED(objspace, rb_iseq_t *, env->iseq);
UPDATE_IF_MOVED(objspace, env->ep[VM_ENV_DATA_INDEX_ENV]);
gc_update_values(objspace, (long)env->env_size, (VALUE *)env->env);
}
}
break;
case imemo_cref:
UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.cref.klass_or_self);
TYPED_UPDATE_IF_MOVED(objspace, struct rb_cref_struct *, RANY(obj)->as.imemo.cref.next);
UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.cref.refinements);
break;
case imemo_svar:
UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.svar.cref_or_me);
UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.svar.lastline);
UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.svar.backref);
UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.svar.others);
break;
case imemo_throw_data:
UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.throw_data.throw_obj);
break;
case imemo_ifunc:
break;
case imemo_memo:
UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.memo.v1);
UPDATE_IF_MOVED(objspace, RANY(obj)->as.imemo.memo.v2);
break;
case imemo_ment:
gc_ref_update_method_entry(objspace, &RANY(obj)->as.imemo.ment);
break;
case imemo_iseq:
rb_iseq_update_references((rb_iseq_t *)obj);
break;
case imemo_ast:
rb_ast_update_references((rb_ast_t *)obj);
break;
case imemo_callcache:
{
const struct rb_callcache *cc = (const struct rb_callcache *)obj;
if (cc->klass) {
UPDATE_IF_MOVED(objspace, cc->klass);
if (!is_live_object(objspace, cc->klass)) {
*((VALUE *)(&cc->klass)) = (VALUE)0;
}
}
if (cc->cme_) {
TYPED_UPDATE_IF_MOVED(objspace, struct rb_callable_method_entry_struct *, cc->cme_);
if (!is_live_object(objspace, (VALUE)cc->cme_)) {
*((struct rb_callable_method_entry_struct **)(&cc->cme_)) = (struct rb_callable_method_entry_struct *)0;
}
}
}
break;
case imemo_constcache:
{
const struct iseq_inline_constant_cache_entry *ice = (struct iseq_inline_constant_cache_entry *)obj;
UPDATE_IF_MOVED(objspace, ice->value);
}
break;
case imemo_parser_strterm:
case imemo_tmpbuf:
case imemo_callinfo:
break;
default:
rb_bug("not reachable %d", imemo_type(obj));
break;
}
}
static enum rb_id_table_iterator_result
check_id_table_move(VALUE value, void *data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
if (gc_object_moved_p(objspace, (VALUE)value)) {
return ID_TABLE_REPLACE;
}
return ID_TABLE_CONTINUE;
}
/* Returns the new location of an object, if it moved. Otherwise returns
* the existing location. */
VALUE
rb_gc_location(VALUE value)
{
VALUE destination;
if (!SPECIAL_CONST_P(value)) {
void *poisoned = asan_unpoison_object_temporary(value);
if (BUILTIN_TYPE(value) == T_MOVED) {
destination = (VALUE)RMOVED(value)->destination;
GC_ASSERT(BUILTIN_TYPE(destination) != T_NONE);
}
else {
destination = value;
}
/* Re-poison slot if it's not the one we want */
if (poisoned) {
GC_ASSERT(BUILTIN_TYPE(value) == T_NONE);
asan_poison_object(value);
}
}
else {
destination = value;
}
return destination;
}
static enum rb_id_table_iterator_result
update_id_table(VALUE *value, void *data, int existing)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
if (gc_object_moved_p(objspace, (VALUE)*value)) {
*value = rb_gc_location((VALUE)*value);
}
return ID_TABLE_CONTINUE;
}
static void
update_m_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
{
if (tbl) {
rb_id_table_foreach_values_with_replace(tbl, check_id_table_move, update_id_table, objspace);
}
}
static enum rb_id_table_iterator_result
update_cc_tbl_i(VALUE ccs_ptr, void *data)
{
rb_objspace_t *objspace = (rb_objspace_t *)data;
struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_ptr;
VM_ASSERT(vm_ccs_p(ccs));
if (gc_object_moved_p(objspace, (VALUE)ccs->cme)) {
ccs->cme = (const rb_callable_method_entry_t *)rb_gc_location((VALUE)ccs->cme);
}
for (int i=0; i<ccs->len; i++) {
if (gc_object_moved_p(objspace, (VALUE)ccs->entries[i].ci)) {
ccs->entries[i].ci = (struct rb_callinfo *)rb_gc_location((VALUE)ccs->entries[i].ci);
}
if (gc_object_moved_p(objspace, (VALUE)ccs->entries[i].cc)) {
ccs->entries[i].cc = (struct rb_callcache *)rb_gc_location((VALUE)ccs->entries[i].cc);
}
}
// do not replace
return ID_TABLE_CONTINUE;
}
static void
update_cc_tbl(rb_objspace_t *objspace, VALUE klass)
{
struct rb_id_table *tbl = RCLASS_CC_TBL(klass);
if (tbl) {
rb_id_table_foreach_values(tbl, update_cc_tbl_i, objspace);
}
}
static enum rb_id_table_iterator_result
update_cvc_tbl_i(VALUE cvc_entry, void *data)
{
struct rb_cvar_class_tbl_entry *entry;
entry = (struct rb_cvar_class_tbl_entry *)cvc_entry;
entry->class_value = rb_gc_location(entry->class_value);
return ID_TABLE_CONTINUE;
}
static void
update_cvc_tbl(rb_objspace_t *objspace, VALUE klass)
{
struct rb_id_table *tbl = RCLASS_CVC_TBL(klass);
if (tbl) {
rb_id_table_foreach_values(tbl, update_cvc_tbl_i, objspace);
}
}
static enum rb_id_table_iterator_result
update_const_table(VALUE value, void *data)
{
rb_const_entry_t *ce = (rb_const_entry_t *)value;
rb_objspace_t * objspace = (rb_objspace_t *)data;
if (gc_object_moved_p(objspace, ce->value)) {
ce->value = rb_gc_location(ce->value);
}
if (gc_object_moved_p(objspace, ce->file)) {
ce->file = rb_gc_location(ce->file);
}
return ID_TABLE_CONTINUE;
}
static void
update_const_tbl(rb_objspace_t *objspace, struct rb_id_table *tbl)
{
if (!tbl) return;
rb_id_table_foreach_values(tbl, update_const_table, objspace);
}
static void
update_subclass_entries(rb_objspace_t *objspace, rb_subclass_entry_t *entry)
{
while (entry) {
UPDATE_IF_MOVED(objspace, entry->klass);
entry = entry->next;
}
}
static void
update_class_ext(rb_objspace_t *objspace, rb_classext_t *ext)
{
UPDATE_IF_MOVED(objspace, ext->origin_);
UPDATE_IF_MOVED(objspace, ext->includer);
UPDATE_IF_MOVED(objspace, ext->refined_class);
update_subclass_entries(objspace, ext->subclasses);
}
static void
update_superclasses(rb_objspace_t *objspace, VALUE obj)
{
if (FL_TEST_RAW(obj, RCLASS_SUPERCLASSES_INCLUDE_SELF)) {
for (size_t i = 0; i < RCLASS_SUPERCLASS_DEPTH(obj) + 1; i++) {
UPDATE_IF_MOVED(objspace, RCLASS_SUPERCLASSES(obj)[i]);
}
}
}
static void
gc_update_object_references(rb_objspace_t *objspace, VALUE obj)
{
RVALUE *any = RANY(obj);
gc_report(4, objspace, "update-refs: %p ->\n", (void *)obj);
switch (BUILTIN_TYPE(obj)) {
case T_CLASS:
case T_MODULE:
if (RCLASS_SUPER((VALUE)obj)) {
UPDATE_IF_MOVED(objspace, RCLASS(obj)->super);
}
if (!RCLASS_EXT(obj)) break;
update_m_tbl(objspace, RCLASS_M_TBL(obj));
update_cc_tbl(objspace, obj);
update_cvc_tbl(objspace, obj);
update_superclasses(objspace, obj);
gc_update_tbl_refs(objspace, RCLASS_IV_TBL(obj));
update_class_ext(objspace, RCLASS_EXT(obj));
update_const_tbl(objspace, RCLASS_CONST_TBL(obj));
break;
case T_ICLASS:
if (FL_TEST(obj, RICLASS_IS_ORIGIN) &&
!FL_TEST(obj, RICLASS_ORIGIN_SHARED_MTBL)) {
update_m_tbl(objspace, RCLASS_M_TBL(obj));
}
if (RCLASS_SUPER((VALUE)obj)) {
UPDATE_IF_MOVED(objspace, RCLASS(obj)->super);
}
if (!RCLASS_EXT(obj)) break;
if (RCLASS_IV_TBL(obj)) {
gc_update_tbl_refs(objspace, RCLASS_IV_TBL(obj));
}
update_class_ext(objspace, RCLASS_EXT(obj));
update_m_tbl(objspace, RCLASS_CALLABLE_M_TBL(obj));
update_cc_tbl(objspace, obj);
break;
case T_IMEMO:
gc_ref_update_imemo(objspace, obj);
return;
case T_NIL:
case T_FIXNUM:
case T_NODE:
case T_MOVED:
case T_NONE:
/* These can't move */
return;
case T_ARRAY:
gc_ref_update_array(objspace, obj);
break;
case T_HASH:
gc_ref_update_hash(objspace, obj);
UPDATE_IF_MOVED(objspace, any->as.hash.ifnone);
break;
case T_STRING:
{
#if USE_RVARGC
#endif
if (STR_SHARED_P(obj)) {
#if USE_RVARGC
VALUE old_root = any->as.string.as.heap.aux.shared;
#endif
UPDATE_IF_MOVED(objspace, any->as.string.as.heap.aux.shared);
#if USE_RVARGC
VALUE new_root = any->as.string.as.heap.aux.shared;
rb_str_update_shared_ary(obj, old_root, new_root);
// if, after move the string is not embedded, and can fit in the
// slot it's been placed in, then re-embed it
if ((size_t)GET_HEAP_PAGE(obj)->slot_size >= rb_str_size_as_embedded(obj)) {
if (!STR_EMBED_P(obj) && rb_str_reembeddable_p(obj)) {
rb_str_make_embedded(obj);
}
}
#endif
}
break;
}
case T_DATA:
/* Call the compaction callback, if it exists */
{
void *const ptr = DATA_PTR(obj);
if (ptr) {
if (RTYPEDDATA_P(obj)) {
RUBY_DATA_FUNC compact_func = any->as.typeddata.type->function.dcompact;
if (compact_func) (*compact_func)(ptr);
}
}
}
break;
case T_OBJECT:
gc_ref_update_object(objspace, obj);
break;
case T_FILE:
if (any->as.file.fptr) {
UPDATE_IF_MOVED(objspace, any->as.file.fptr->self);
UPDATE_IF_MOVED(objspace, any->as.file.fptr->pathv);
UPDATE_IF_MOVED(objspace, any->as.file.fptr->tied_io_for_writing);
UPDATE_IF_MOVED(objspace, any->as.file.fptr->writeconv_asciicompat);
UPDATE_IF_MOVED(objspace, any->as.file.fptr->writeconv_pre_ecopts);
UPDATE_IF_MOVED(objspace, any->as.file.fptr->encs.ecopts);
UPDATE_IF_MOVED(objspace, any->as.file.fptr->write_lock);
}
break;
case T_REGEXP:
UPDATE_IF_MOVED(objspace, any->as.regexp.src);
break;
case T_SYMBOL:
if (DYNAMIC_SYM_P((VALUE)any)) {
UPDATE_IF_MOVED(objspace, RSYMBOL(any)->fstr);
}
break;
case T_FLOAT:
case T_BIGNUM:
break;
case T_MATCH:
UPDATE_IF_MOVED(objspace, any->as.match.regexp);
if (any->as.match.str) {
UPDATE_IF_MOVED(objspace, any->as.match.str);
}
break;
case T_RATIONAL:
UPDATE_IF_MOVED(objspace, any->as.rational.num);
UPDATE_IF_MOVED(objspace, any->as.rational.den);
break;
case T_COMPLEX:
UPDATE_IF_MOVED(objspace, any->as.complex.real);
UPDATE_IF_MOVED(objspace, any->as.complex.imag);
break;
case T_STRUCT:
{
long i, len = RSTRUCT_LEN(obj);
VALUE *ptr = (VALUE *)RSTRUCT_CONST_PTR(obj);
for (i = 0; i < len; i++) {
UPDATE_IF_MOVED(objspace, ptr[i]);
}
}
break;
default:
#if GC_DEBUG
rb_gcdebug_print_obj_condition((VALUE)obj);
rb_obj_info_dump(obj);
rb_bug("unreachable");
#endif
break;
}
UPDATE_IF_MOVED(objspace, RBASIC(obj)->klass);
gc_report(4, objspace, "update-refs: %p <-\n", (void *)obj);
}
static int
gc_ref_update(void *vstart, void *vend, size_t stride, rb_objspace_t * objspace, struct heap_page *page)
{
VALUE v = (VALUE)vstart;
asan_unlock_freelist(page);
asan_lock_freelist(page);
page->flags.has_uncollectible_shady_objects = FALSE;
page->flags.has_remembered_objects = FALSE;
/* For each object on the page */
for (; v != (VALUE)vend; v += stride) {
void *poisoned = asan_unpoison_object_temporary(v);
switch (BUILTIN_TYPE(v)) {
case T_NONE:
case T_MOVED:
case T_ZOMBIE:
break;
default:
if (RVALUE_WB_UNPROTECTED(v)) {
page->flags.has_uncollectible_shady_objects = TRUE;
}
if (RVALUE_PAGE_MARKING(page, v)) {
page->flags.has_remembered_objects = TRUE;
}
if (page->flags.before_sweep) {
if (RVALUE_MARKED(v)) {
gc_update_object_references(objspace, v);
}
}
else {
gc_update_object_references(objspace, v);
}
}
if (poisoned) {
asan_poison_object(v);
}
}
return 0;
}
extern rb_symbols_t ruby_global_symbols;
#define global_symbols ruby_global_symbols
static void
gc_update_references(rb_objspace_t *objspace)
{
rb_execution_context_t *ec = GET_EC();
rb_vm_t *vm = rb_ec_vm_ptr(ec);
struct heap_page *page = NULL;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
bool should_set_mark_bits = TRUE;
rb_size_pool_t *size_pool = &size_pools[i];
rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
ccan_list_for_each(&heap->pages, page, page_node) {
uintptr_t start = (uintptr_t)page->start;
uintptr_t end = start + (page->total_slots * size_pool->slot_size);
gc_ref_update((void *)start, (void *)end, size_pool->slot_size, objspace, page);
if (page == heap->sweeping_page) {
should_set_mark_bits = FALSE;
}
if (should_set_mark_bits) {
gc_setup_mark_bits(page);
}
}
}
rb_vm_update_references(vm);
rb_transient_heap_update_references();
rb_gc_update_global_tbl();
global_symbols.ids = rb_gc_location(global_symbols.ids);
global_symbols.dsymbol_fstr_hash = rb_gc_location(global_symbols.dsymbol_fstr_hash);
gc_update_tbl_refs(objspace, objspace->obj_to_id_tbl);
gc_update_table_refs(objspace, objspace->id_to_obj_tbl);
gc_update_table_refs(objspace, global_symbols.str_sym);
gc_update_table_refs(objspace, finalizer_table);
}
#if GC_CAN_COMPILE_COMPACTION
/*
* call-seq:
* GC.latest_compact_info -> {:considered=>{:T_CLASS=>11}, :moved=>{:T_CLASS=>11}}
*
* Returns information about object moved in the most recent GC compaction.
*
* The returned hash has two keys :considered and :moved. The hash for
* :considered lists the number of objects that were considered for movement
* by the compactor, and the :moved hash lists the number of objects that
* were actually moved. Some objects can't be moved (maybe they were pinned)
* so these numbers can be used to calculate compaction efficiency.
*/
static VALUE
gc_compact_stats(VALUE self)
{
size_t i;
rb_objspace_t *objspace = &rb_objspace;
VALUE h = rb_hash_new();
VALUE considered = rb_hash_new();
VALUE moved = rb_hash_new();
VALUE moved_up = rb_hash_new();
VALUE moved_down = rb_hash_new();
for (i=0; i<T_MASK; i++) {
if (objspace->rcompactor.considered_count_table[i]) {
rb_hash_aset(considered, type_sym(i), SIZET2NUM(objspace->rcompactor.considered_count_table[i]));
}
if (objspace->rcompactor.moved_count_table[i]) {
rb_hash_aset(moved, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_count_table[i]));
}
if (objspace->rcompactor.moved_up_count_table[i]) {
rb_hash_aset(moved_up, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_up_count_table[i]));
}
if (objspace->rcompactor.moved_down_count_table[i]) {
rb_hash_aset(moved_down, type_sym(i), SIZET2NUM(objspace->rcompactor.moved_down_count_table[i]));
}
}
rb_hash_aset(h, ID2SYM(rb_intern("considered")), considered);
rb_hash_aset(h, ID2SYM(rb_intern("moved")), moved);
rb_hash_aset(h, ID2SYM(rb_intern("moved_up")), moved_up);
rb_hash_aset(h, ID2SYM(rb_intern("moved_down")), moved_down);
return h;
}
#else
# define gc_compact_stats rb_f_notimplement
#endif
#if GC_CAN_COMPILE_COMPACTION
static void
root_obj_check_moved_i(const char *category, VALUE obj, void *data)
{
if (gc_object_moved_p(&rb_objspace, obj)) {
rb_bug("ROOT %s points to MOVED: %p -> %s\n", category, (void *)obj, obj_info(rb_gc_location(obj)));
}
}
static void
reachable_object_check_moved_i(VALUE ref, void *data)
{
VALUE parent = (VALUE)data;
if (gc_object_moved_p(&rb_objspace, ref)) {
rb_bug("Object %s points to MOVED: %p -> %s\n", obj_info(parent), (void *)ref, obj_info(rb_gc_location(ref)));
}
}
static int
heap_check_moved_i(void *vstart, void *vend, size_t stride, void *data)
{
VALUE v = (VALUE)vstart;
for (; v != (VALUE)vend; v += stride) {
if (gc_object_moved_p(&rb_objspace, v)) {
/* Moved object still on the heap, something may have a reference. */
}
else {
void *poisoned = asan_unpoison_object_temporary(v);
switch (BUILTIN_TYPE(v)) {
case T_NONE:
case T_ZOMBIE:
break;
default:
if (!rb_objspace_garbage_object_p(v)) {
rb_objspace_reachable_objects_from(v, reachable_object_check_moved_i, (void *)v);
}
}
if (poisoned) {
GC_ASSERT(BUILTIN_TYPE(v) == T_NONE);
asan_poison_object(v);
}
}
}
return 0;
}
/*
* call-seq:
* GC.compact
*
* This function compacts objects together in Ruby's heap. It eliminates
* unused space (or fragmentation) in the heap by moving objects in to that
* unused space. This function returns a hash which contains statistics about
* which objects were moved. See `GC.latest_gc_info` for details about
* compaction statistics.
*
* This method is implementation specific and not expected to be implemented
* in any implementation besides MRI.
*
* To test whether GC compaction is supported, use the idiom:
*
* GC.respond_to?(:compact)
*/
static VALUE
gc_compact(VALUE self)
{
/* Run GC with compaction enabled */
gc_start_internal(NULL, self, Qtrue, Qtrue, Qtrue, Qtrue);
return gc_compact_stats(self);
}
#else
# define gc_compact rb_f_notimplement
#endif
#if GC_CAN_COMPILE_COMPACTION
static VALUE
gc_verify_compaction_references(rb_execution_context_t *ec, VALUE self, VALUE double_heap, VALUE expand_heap, VALUE toward_empty)
{
rb_objspace_t *objspace = &rb_objspace;
/* Clear the heap. */
gc_start_internal(NULL, self, Qtrue, Qtrue, Qtrue, Qfalse);
size_t growth_slots = gc_params.heap_init_slots;
if (RTEST(double_heap)) {
rb_warn("double_heap is deprecated, please use expand_heap instead");
}
RB_VM_LOCK_ENTER();
{
gc_rest(objspace);
/* if both double_heap and expand_heap are set, expand_heap takes precedence */
if (RTEST(double_heap) || RTEST(expand_heap)) {
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool);
if (RTEST(expand_heap)) {
size_t required_pages = growth_slots / size_pool->slot_size;
heap_add_pages(objspace, size_pool, heap, MAX(required_pages, heap->total_pages));
}
else {
heap_add_pages(objspace, size_pool, heap, heap->total_pages);
}
}
}
if (RTEST(toward_empty)) {
gc_sort_heap_by_empty_slots(objspace);
}
}
RB_VM_LOCK_LEAVE();
gc_start_internal(NULL, self, Qtrue, Qtrue, Qtrue, Qtrue);
objspace_reachable_objects_from_root(objspace, root_obj_check_moved_i, NULL);
objspace_each_objects(objspace, heap_check_moved_i, NULL, TRUE);
return gc_compact_stats(self);
}
#else
# define gc_verify_compaction_references (rb_builtin_arity3_function_type)rb_f_notimplement
#endif
VALUE
rb_gc_start(void)
{
rb_gc();
return Qnil;
}
void
rb_gc(void)
{
rb_objspace_t *objspace = &rb_objspace;
unsigned int reason = GPR_DEFAULT_REASON;
garbage_collect(objspace, reason);
}
int
rb_during_gc(void)
{
rb_objspace_t *objspace = &rb_objspace;
return during_gc;
}
#if RGENGC_PROFILE >= 2
static const char *type_name(int type, VALUE obj);
static void
gc_count_add_each_types(VALUE hash, const char *name, const size_t *types)
{
VALUE result = rb_hash_new_with_size(T_MASK);
int i;
for (i=0; i<T_MASK; i++) {
const char *type = type_name(i, 0);
rb_hash_aset(result, ID2SYM(rb_intern(type)), SIZET2NUM(types[i]));
}
rb_hash_aset(hash, ID2SYM(rb_intern(name)), result);
}
#endif
size_t
rb_gc_count(void)
{
return rb_objspace.profile.count;
}
static VALUE
gc_count(rb_execution_context_t *ec, VALUE self)
{
return SIZET2NUM(rb_gc_count());
}
static VALUE
gc_info_decode(rb_objspace_t *objspace, const VALUE hash_or_key, const unsigned int orig_flags)
{
static VALUE sym_major_by = Qnil, sym_gc_by, sym_immediate_sweep, sym_have_finalizer, sym_state;
static VALUE sym_nofree, sym_oldgen, sym_shady, sym_force, sym_stress;
#if RGENGC_ESTIMATE_OLDMALLOC
static VALUE sym_oldmalloc;
#endif
static VALUE sym_newobj, sym_malloc, sym_method, sym_capi;
static VALUE sym_none, sym_marking, sym_sweeping;
VALUE hash = Qnil, key = Qnil;
VALUE major_by;
unsigned int flags = orig_flags ? orig_flags : objspace->profile.latest_gc_info;
if (SYMBOL_P(hash_or_key)) {
key = hash_or_key;
}
else if (RB_TYPE_P(hash_or_key, T_HASH)) {
hash = hash_or_key;
}
else {
rb_raise(rb_eTypeError, "non-hash or symbol given");
}
if (NIL_P(sym_major_by)) {
#define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
S(major_by);
S(gc_by);
S(immediate_sweep);
S(have_finalizer);
S(state);
S(stress);
S(nofree);
S(oldgen);
S(shady);
S(force);
#if RGENGC_ESTIMATE_OLDMALLOC
S(oldmalloc);
#endif
S(newobj);
S(malloc);
S(method);
S(capi);
S(none);
S(marking);
S(sweeping);
#undef S
}
#define SET(name, attr) \
if (key == sym_##name) \
return (attr); \
else if (hash != Qnil) \
rb_hash_aset(hash, sym_##name, (attr));
major_by =
(flags & GPR_FLAG_MAJOR_BY_NOFREE) ? sym_nofree :
(flags & GPR_FLAG_MAJOR_BY_OLDGEN) ? sym_oldgen :
(flags & GPR_FLAG_MAJOR_BY_SHADY) ? sym_shady :
(flags & GPR_FLAG_MAJOR_BY_FORCE) ? sym_force :
#if RGENGC_ESTIMATE_OLDMALLOC
(flags & GPR_FLAG_MAJOR_BY_OLDMALLOC) ? sym_oldmalloc :
#endif
Qnil;
SET(major_by, major_by);
SET(gc_by,
(flags & GPR_FLAG_NEWOBJ) ? sym_newobj :
(flags & GPR_FLAG_MALLOC) ? sym_malloc :
(flags & GPR_FLAG_METHOD) ? sym_method :
(flags & GPR_FLAG_CAPI) ? sym_capi :
(flags & GPR_FLAG_STRESS) ? sym_stress :
Qnil
);
SET(have_finalizer, RBOOL(flags & GPR_FLAG_HAVE_FINALIZE));
SET(immediate_sweep, RBOOL(flags & GPR_FLAG_IMMEDIATE_SWEEP));
if (orig_flags == 0) {
SET(state, gc_mode(objspace) == gc_mode_none ? sym_none :
gc_mode(objspace) == gc_mode_marking ? sym_marking : sym_sweeping);
}
#undef SET
if (!NIL_P(key)) {/* matched key should return above */
rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
}
return hash;
}
VALUE
rb_gc_latest_gc_info(VALUE key)
{
rb_objspace_t *objspace = &rb_objspace;
return gc_info_decode(objspace, key, 0);
}
static VALUE
gc_latest_gc_info(rb_execution_context_t *ec, VALUE self, VALUE arg)
{
rb_objspace_t *objspace = &rb_objspace;
if (NIL_P(arg)) {
arg = rb_hash_new();
}
else if (!SYMBOL_P(arg) && !RB_TYPE_P(arg, T_HASH)) {
rb_raise(rb_eTypeError, "non-hash or symbol given");
}
return gc_info_decode(objspace, arg, 0);
}
enum gc_stat_sym {
gc_stat_sym_count,
gc_stat_sym_time,
gc_stat_sym_heap_allocated_pages,
gc_stat_sym_heap_sorted_length,
gc_stat_sym_heap_allocatable_pages,
gc_stat_sym_heap_available_slots,
gc_stat_sym_heap_live_slots,
gc_stat_sym_heap_free_slots,
gc_stat_sym_heap_final_slots,
gc_stat_sym_heap_marked_slots,
gc_stat_sym_heap_eden_pages,
gc_stat_sym_heap_tomb_pages,
gc_stat_sym_total_allocated_pages,
gc_stat_sym_total_freed_pages,
gc_stat_sym_total_allocated_objects,
gc_stat_sym_total_freed_objects,
gc_stat_sym_malloc_increase_bytes,
gc_stat_sym_malloc_increase_bytes_limit,
gc_stat_sym_minor_gc_count,
gc_stat_sym_major_gc_count,
gc_stat_sym_compact_count,
gc_stat_sym_read_barrier_faults,
gc_stat_sym_total_moved_objects,
gc_stat_sym_remembered_wb_unprotected_objects,
gc_stat_sym_remembered_wb_unprotected_objects_limit,
gc_stat_sym_old_objects,
gc_stat_sym_old_objects_limit,
#if RGENGC_ESTIMATE_OLDMALLOC
gc_stat_sym_oldmalloc_increase_bytes,
gc_stat_sym_oldmalloc_increase_bytes_limit,
#endif
#if RGENGC_PROFILE
gc_stat_sym_total_generated_normal_object_count,
gc_stat_sym_total_generated_shady_object_count,
gc_stat_sym_total_shade_operation_count,
gc_stat_sym_total_promoted_count,
gc_stat_sym_total_remembered_normal_object_count,
gc_stat_sym_total_remembered_shady_object_count,
#endif
gc_stat_sym_last
};
static VALUE gc_stat_symbols[gc_stat_sym_last];
static void
setup_gc_stat_symbols(void)
{
if (gc_stat_symbols[0] == 0) {
#define S(s) gc_stat_symbols[gc_stat_sym_##s] = ID2SYM(rb_intern_const(#s))
S(count);
S(time);
S(heap_allocated_pages);
S(heap_sorted_length);
S(heap_allocatable_pages);
S(heap_available_slots);
S(heap_live_slots);
S(heap_free_slots);
S(heap_final_slots);
S(heap_marked_slots);
S(heap_eden_pages);
S(heap_tomb_pages);
S(total_allocated_pages);
S(total_freed_pages);
S(total_allocated_objects);
S(total_freed_objects);
S(malloc_increase_bytes);
S(malloc_increase_bytes_limit);
S(minor_gc_count);
S(major_gc_count);
S(compact_count);
S(read_barrier_faults);
S(total_moved_objects);
S(remembered_wb_unprotected_objects);
S(remembered_wb_unprotected_objects_limit);
S(old_objects);
S(old_objects_limit);
#if RGENGC_ESTIMATE_OLDMALLOC
S(oldmalloc_increase_bytes);
S(oldmalloc_increase_bytes_limit);
#endif
#if RGENGC_PROFILE
S(total_generated_normal_object_count);
S(total_generated_shady_object_count);
S(total_shade_operation_count);
S(total_promoted_count);
S(total_remembered_normal_object_count);
S(total_remembered_shady_object_count);
#endif /* RGENGC_PROFILE */
#undef S
}
}
static size_t
gc_stat_internal(VALUE hash_or_sym)
{
rb_objspace_t *objspace = &rb_objspace;
VALUE hash = Qnil, key = Qnil;
setup_gc_stat_symbols();
if (RB_TYPE_P(hash_or_sym, T_HASH)) {
hash = hash_or_sym;
}
else if (SYMBOL_P(hash_or_sym)) {
key = hash_or_sym;
}
else {
rb_raise(rb_eTypeError, "non-hash or symbol argument");
}
#define SET(name, attr) \
if (key == gc_stat_symbols[gc_stat_sym_##name]) \
return attr; \
else if (hash != Qnil) \
rb_hash_aset(hash, gc_stat_symbols[gc_stat_sym_##name], SIZET2NUM(attr));
SET(count, objspace->profile.count);
SET(time, (size_t) (objspace->profile.total_time_ns / (1000 * 1000) /* ns -> ms */)); // TODO: UINT64T2NUM
/* implementation dependent counters */
SET(heap_allocated_pages, heap_allocated_pages);
SET(heap_sorted_length, heap_pages_sorted_length);
SET(heap_allocatable_pages, heap_allocatable_pages(objspace));
SET(heap_available_slots, objspace_available_slots(objspace));
SET(heap_live_slots, objspace_live_slots(objspace));
SET(heap_free_slots, objspace_free_slots(objspace));
SET(heap_final_slots, heap_pages_final_slots);
SET(heap_marked_slots, objspace->marked_slots);
SET(heap_eden_pages, heap_eden_total_pages(objspace));
SET(heap_tomb_pages, heap_tomb_total_pages(objspace));
SET(total_allocated_pages, total_allocated_pages(objspace));
SET(total_freed_pages, total_freed_pages(objspace));
SET(total_allocated_objects, objspace->total_allocated_objects);
SET(total_freed_objects, objspace->profile.total_freed_objects);
SET(malloc_increase_bytes, malloc_increase);
SET(malloc_increase_bytes_limit, malloc_limit);
SET(minor_gc_count, objspace->profile.minor_gc_count);
SET(major_gc_count, objspace->profile.major_gc_count);
SET(compact_count, objspace->profile.compact_count);
SET(read_barrier_faults, objspace->profile.read_barrier_faults);
SET(total_moved_objects, objspace->rcompactor.total_moved);
SET(remembered_wb_unprotected_objects, objspace->rgengc.uncollectible_wb_unprotected_objects);
SET(remembered_wb_unprotected_objects_limit, objspace->rgengc.uncollectible_wb_unprotected_objects_limit);
SET(old_objects, objspace->rgengc.old_objects);
SET(old_objects_limit, objspace->rgengc.old_objects_limit);
#if RGENGC_ESTIMATE_OLDMALLOC
SET(oldmalloc_increase_bytes, objspace->rgengc.oldmalloc_increase);
SET(oldmalloc_increase_bytes_limit, objspace->rgengc.oldmalloc_increase_limit);
#endif
#if RGENGC_PROFILE
SET(total_generated_normal_object_count, objspace->profile.total_generated_normal_object_count);
SET(total_generated_shady_object_count, objspace->profile.total_generated_shady_object_count);
SET(total_shade_operation_count, objspace->profile.total_shade_operation_count);
SET(total_promoted_count, objspace->profile.total_promoted_count);
SET(total_remembered_normal_object_count, objspace->profile.total_remembered_normal_object_count);
SET(total_remembered_shady_object_count, objspace->profile.total_remembered_shady_object_count);
#endif /* RGENGC_PROFILE */
#undef SET
if (!NIL_P(key)) { /* matched key should return above */
rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
}
#if defined(RGENGC_PROFILE) && RGENGC_PROFILE >= 2
if (hash != Qnil) {
gc_count_add_each_types(hash, "generated_normal_object_count_types", objspace->profile.generated_normal_object_count_types);
gc_count_add_each_types(hash, "generated_shady_object_count_types", objspace->profile.generated_shady_object_count_types);
gc_count_add_each_types(hash, "shade_operation_count_types", objspace->profile.shade_operation_count_types);
gc_count_add_each_types(hash, "promoted_types", objspace->profile.promoted_types);
gc_count_add_each_types(hash, "remembered_normal_object_count_types", objspace->profile.remembered_normal_object_count_types);
gc_count_add_each_types(hash, "remembered_shady_object_count_types", objspace->profile.remembered_shady_object_count_types);
}
#endif
return 0;
}
static VALUE
gc_stat(rb_execution_context_t *ec, VALUE self, VALUE arg) // arg is (nil || hash || symbol)
{
if (NIL_P(arg)) {
arg = rb_hash_new();
}
else if (SYMBOL_P(arg)) {
size_t value = gc_stat_internal(arg);
return SIZET2NUM(value);
}
else if (RB_TYPE_P(arg, T_HASH)) {
// ok
}
else {
rb_raise(rb_eTypeError, "non-hash or symbol given");
}
gc_stat_internal(arg);
return arg;
}
size_t
rb_gc_stat(VALUE key)
{
if (SYMBOL_P(key)) {
size_t value = gc_stat_internal(key);
return value;
}
else {
gc_stat_internal(key);
return 0;
}
}
enum gc_stat_heap_sym {
gc_stat_heap_sym_slot_size,
gc_stat_heap_sym_heap_allocatable_pages,
gc_stat_heap_sym_heap_eden_pages,
gc_stat_heap_sym_heap_eden_slots,
gc_stat_heap_sym_heap_tomb_pages,
gc_stat_heap_sym_heap_tomb_slots,
gc_stat_heap_sym_total_allocated_pages,
gc_stat_heap_sym_total_freed_pages,
gc_stat_heap_sym_force_major_gc_count,
gc_stat_heap_sym_last
};
static VALUE gc_stat_heap_symbols[gc_stat_heap_sym_last];
static void
setup_gc_stat_heap_symbols(void)
{
if (gc_stat_heap_symbols[0] == 0) {
#define S(s) gc_stat_heap_symbols[gc_stat_heap_sym_##s] = ID2SYM(rb_intern_const(#s))
S(slot_size);
S(heap_allocatable_pages);
S(heap_eden_pages);
S(heap_eden_slots);
S(heap_tomb_pages);
S(heap_tomb_slots);
S(total_allocated_pages);
S(total_freed_pages);
S(force_major_gc_count);
#undef S
}
}
static size_t
gc_stat_heap_internal(int size_pool_idx, VALUE hash_or_sym)
{
rb_objspace_t *objspace = &rb_objspace;
VALUE hash = Qnil, key = Qnil;
setup_gc_stat_heap_symbols();
if (RB_TYPE_P(hash_or_sym, T_HASH)) {
hash = hash_or_sym;
}
else if (SYMBOL_P(hash_or_sym)) {
key = hash_or_sym;
}
else {
rb_raise(rb_eTypeError, "non-hash or symbol argument");
}
if (size_pool_idx < 0 || size_pool_idx >= SIZE_POOL_COUNT) {
rb_raise(rb_eArgError, "size pool index out of range");
}
rb_size_pool_t *size_pool = &size_pools[size_pool_idx];
#define SET(name, attr) \
if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \
return attr; \
else if (hash != Qnil) \
rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr));
SET(slot_size, size_pool->slot_size);
SET(heap_allocatable_pages, size_pool->allocatable_pages);
SET(heap_eden_pages, SIZE_POOL_EDEN_HEAP(size_pool)->total_pages);
SET(heap_eden_slots, SIZE_POOL_EDEN_HEAP(size_pool)->total_slots);
SET(heap_tomb_pages, SIZE_POOL_TOMB_HEAP(size_pool)->total_pages);
SET(heap_tomb_slots, SIZE_POOL_TOMB_HEAP(size_pool)->total_slots);
SET(total_allocated_pages, size_pool->total_allocated_pages);
SET(total_freed_pages, size_pool->total_freed_pages);
SET(force_major_gc_count, size_pool->force_major_gc_count);
#undef SET
if (!NIL_P(key)) { /* matched key should return above */
rb_raise(rb_eArgError, "unknown key: %"PRIsVALUE, rb_sym2str(key));
}
return 0;
}
static VALUE
gc_stat_heap(rb_execution_context_t *ec, VALUE self, VALUE heap_name, VALUE arg)
{
if (NIL_P(heap_name)) {
if (NIL_P(arg)) {
arg = rb_hash_new();
}
else if (RB_TYPE_P(arg, T_HASH)) {
// ok
}
else {
rb_raise(rb_eTypeError, "non-hash given");
}
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
VALUE hash = rb_hash_aref(arg, INT2FIX(i));
if (NIL_P(hash)) {
hash = rb_hash_new();
rb_hash_aset(arg, INT2FIX(i), hash);
}
gc_stat_heap_internal(i, hash);
}
}
else if (FIXNUM_P(heap_name)) {
int size_pool_idx = FIX2INT(heap_name);
if (NIL_P(arg)) {
arg = rb_hash_new();
}
else if (SYMBOL_P(arg)) {
size_t value = gc_stat_heap_internal(size_pool_idx, arg);
return SIZET2NUM(value);
}
else if (RB_TYPE_P(arg, T_HASH)) {
// ok
}
else {
rb_raise(rb_eTypeError, "non-hash or symbol given");
}
gc_stat_heap_internal(size_pool_idx, arg);
}
else {
rb_raise(rb_eTypeError, "heap_name must be nil or an Integer");
}
return arg;
}
static VALUE
gc_stress_get(rb_execution_context_t *ec, VALUE self)
{
rb_objspace_t *objspace = &rb_objspace;
return ruby_gc_stress_mode;
}
static void
gc_stress_set(rb_objspace_t *objspace, VALUE flag)
{
objspace->flags.gc_stressful = RTEST(flag);
objspace->gc_stress_mode = flag;
}
static VALUE
gc_stress_set_m(rb_execution_context_t *ec, VALUE self, VALUE flag)
{
rb_objspace_t *objspace = &rb_objspace;
gc_stress_set(objspace, flag);
return flag;
}
VALUE
rb_gc_enable(void)
{
rb_objspace_t *objspace = &rb_objspace;
return rb_objspace_gc_enable(objspace);
}
VALUE
rb_objspace_gc_enable(rb_objspace_t *objspace)
{
int old = dont_gc_val();
dont_gc_off();
return RBOOL(old);
}
static VALUE
gc_enable(rb_execution_context_t *ec, VALUE _)
{
return rb_gc_enable();
}
VALUE
rb_gc_disable_no_rest(void)
{
rb_objspace_t *objspace = &rb_objspace;
return gc_disable_no_rest(objspace);
}
static VALUE
gc_disable_no_rest(rb_objspace_t *objspace)
{
int old = dont_gc_val();
dont_gc_on();
return RBOOL(old);
}
VALUE
rb_gc_disable(void)
{
rb_objspace_t *objspace = &rb_objspace;
return rb_objspace_gc_disable(objspace);
}
VALUE
rb_objspace_gc_disable(rb_objspace_t *objspace)
{
gc_rest(objspace);
return gc_disable_no_rest(objspace);
}
static VALUE
gc_disable(rb_execution_context_t *ec, VALUE _)
{
return rb_gc_disable();
}
#if GC_CAN_COMPILE_COMPACTION
/*
* call-seq:
* GC.auto_compact = flag
*
* Updates automatic compaction mode.
*
* When enabled, the compactor will execute on every major collection.
*
* Enabling compaction will degrade performance on major collections.
*/
static VALUE
gc_set_auto_compact(VALUE _, VALUE v)
{
GC_ASSERT(GC_COMPACTION_SUPPORTED);
ruby_enable_autocompact = RTEST(v);
return v;
}
#else
# define gc_set_auto_compact rb_f_notimplement
#endif
#if GC_CAN_COMPILE_COMPACTION
/*
* call-seq:
* GC.auto_compact -> true or false
*
* Returns whether or not automatic compaction has been enabled.
*/
static VALUE
gc_get_auto_compact(VALUE _)
{
return RBOOL(ruby_enable_autocompact);
}
#else
# define gc_get_auto_compact rb_f_notimplement
#endif
static int
get_envparam_size(const char *name, size_t *default_value, size_t lower_bound)
{
const char *ptr = getenv(name);
ssize_t val;
if (ptr != NULL && *ptr) {
size_t unit = 0;
char *end;
#if SIZEOF_SIZE_T == SIZEOF_LONG_LONG
val = strtoll(ptr, &end, 0);
#else
val = strtol(ptr, &end, 0);
#endif
switch (*end) {
case 'k': case 'K':
unit = 1024;
++end;
break;
case 'm': case 'M':
unit = 1024*1024;
++end;
break;
case 'g': case 'G':
unit = 1024*1024*1024;
++end;
break;
}
while (*end && isspace((unsigned char)*end)) end++;
if (*end) {
if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
return 0;
}
if (unit > 0) {
if (val < -(ssize_t)(SIZE_MAX / 2 / unit) || (ssize_t)(SIZE_MAX / 2 / unit) < val) {
if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%s is ignored because it overflows\n", name, ptr);
return 0;
}
val *= unit;
}
if (val > 0 && (size_t)val > lower_bound) {
if (RTEST(ruby_verbose)) {
fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE")\n", name, val, *default_value);
}
*default_value = (size_t)val;
return 1;
}
else {
if (RTEST(ruby_verbose)) {
fprintf(stderr, "%s=%"PRIdSIZE" (default value: %"PRIuSIZE") is ignored because it must be greater than %"PRIuSIZE".\n",
name, val, *default_value, lower_bound);
}
return 0;
}
}
return 0;
}
static int
get_envparam_double(const char *name, double *default_value, double lower_bound, double upper_bound, int accept_zero)
{
const char *ptr = getenv(name);
double val;
if (ptr != NULL && *ptr) {
char *end;
val = strtod(ptr, &end);
if (!*ptr || *end) {
if (RTEST(ruby_verbose)) fprintf(stderr, "invalid string for %s: %s\n", name, ptr);
return 0;
}
if (accept_zero && val == 0.0) {
goto accept;
}
else if (val <= lower_bound) {
if (RTEST(ruby_verbose)) {
fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be greater than %f.\n",
name, val, *default_value, lower_bound);
}
}
else if (upper_bound != 0.0 && /* ignore upper_bound if it is 0.0 */
val > upper_bound) {
if (RTEST(ruby_verbose)) {
fprintf(stderr, "%s=%f (default value: %f) is ignored because it must be lower than %f.\n",
name, val, *default_value, upper_bound);
}
}
else {
goto accept;
}
}
return 0;
accept:
if (RTEST(ruby_verbose)) fprintf(stderr, "%s=%f (default value: %f)\n", name, val, *default_value);
*default_value = val;
return 1;
}
static void
gc_set_initial_pages(void)
{
size_t min_pages;
rb_objspace_t *objspace = &rb_objspace;
gc_rest(objspace);
min_pages = gc_params.heap_init_slots / HEAP_PAGE_OBJ_LIMIT;
size_t pages_per_class = (min_pages - heap_eden_total_pages(objspace)) / SIZE_POOL_COUNT;
for (int i = 0; i < SIZE_POOL_COUNT; i++) {
rb_size_pool_t *size_pool = &size_pools[i];
heap_add_pages(objspace, size_pool, SIZE_POOL_EDEN_HEAP(size_pool), pages_per_class);
}
heap_add_pages(objspace, &size_pools[0], SIZE_POOL_EDEN_HEAP(&size_pools[0]), min_pages - heap_eden_total_pages(objspace));
}
/*
* GC tuning environment variables
*
* * RUBY_GC_HEAP_INIT_SLOTS
* - Initial allocation slots.
* * RUBY_GC_HEAP_FREE_SLOTS
* - Prepare at least this amount of slots after GC.
* - Allocate slots if there are not enough slots.
* * RUBY_GC_HEAP_GROWTH_FACTOR (new from 2.1)
* - Allocate slots by this factor.
* - (next slots number) = (current slots number) * (this factor)
* * RUBY_GC_HEAP_GROWTH_MAX_SLOTS (new from 2.1)
* - Allocation rate is limited to this number of slots.
* * RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO (new from 2.4)
* - Allocate additional pages when the number of free slots is
* lower than the value (total_slots * (this ratio)).
* * RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO (new from 2.4)
* - Allocate slots to satisfy this formula:
* free_slots = total_slots * goal_ratio
* - In other words, prepare (total_slots * goal_ratio) free slots.
* - if this value is 0.0, then use RUBY_GC_HEAP_GROWTH_FACTOR directly.
* * RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO (new from 2.4)
* - Allow to free pages when the number of free slots is
* greater than the value (total_slots * (this ratio)).
* * RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR (new from 2.1.1)
* - Do full GC when the number of old objects is more than R * N
* where R is this factor and
* N is the number of old objects just after last full GC.
*
* * obsolete
* * RUBY_FREE_MIN -> RUBY_GC_HEAP_FREE_SLOTS (from 2.1)
* * RUBY_HEAP_MIN_SLOTS -> RUBY_GC_HEAP_INIT_SLOTS (from 2.1)
*
* * RUBY_GC_MALLOC_LIMIT
* * RUBY_GC_MALLOC_LIMIT_MAX (new from 2.1)
* * RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
*
* * RUBY_GC_OLDMALLOC_LIMIT (new from 2.1)
* * RUBY_GC_OLDMALLOC_LIMIT_MAX (new from 2.1)
* * RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR (new from 2.1)
*/
void
ruby_gc_set_params(void)
{
rb_objspace_t *objspace = &rb_objspace;
/* RUBY_GC_HEAP_FREE_SLOTS */
if (get_envparam_size("RUBY_GC_HEAP_FREE_SLOTS", &gc_params.heap_free_slots, 0)) {
/* ok */
}
/* RUBY_GC_HEAP_INIT_SLOTS */
if (get_envparam_size("RUBY_GC_HEAP_INIT_SLOTS", &gc_params.heap_init_slots, 0)) {
gc_set_initial_pages();
}
get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE);
get_envparam_size ("RUBY_GC_HEAP_GROWTH_MAX_SLOTS", &gc_params.growth_max_slots, 0);
get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO", &gc_params.heap_free_slots_min_ratio,
0.0, 1.0, FALSE);
get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO", &gc_params.heap_free_slots_max_ratio,
gc_params.heap_free_slots_min_ratio, 1.0, FALSE);
get_envparam_double("RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO", &gc_params.heap_free_slots_goal_ratio,
gc_params.heap_free_slots_min_ratio, gc_params.heap_free_slots_max_ratio, TRUE);
get_envparam_double("RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR", &gc_params.oldobject_limit_factor, 0.0, 0.0, TRUE);
if (get_envparam_size("RUBY_GC_MALLOC_LIMIT", &gc_params.malloc_limit_min, 0)) {
malloc_limit = gc_params.malloc_limit_min;
}
get_envparam_size ("RUBY_GC_MALLOC_LIMIT_MAX", &gc_params.malloc_limit_max, 0);
if (!gc_params.malloc_limit_max) { /* ignore max-check if 0 */
gc_params.malloc_limit_max = SIZE_MAX;
}
get_envparam_double("RUBY_GC_MALLOC_LIMIT_GROWTH_FACTOR", &gc_params.malloc_limit_growth_factor, 1.0, 0.0, FALSE);
#if RGENGC_ESTIMATE_OLDMALLOC
if (get_envparam_size("RUBY_GC_OLDMALLOC_LIMIT", &gc_params.oldmalloc_limit_min, 0)) {
objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
}
get_envparam_size ("RUBY_GC_OLDMALLOC_LIMIT_MAX", &gc_params.oldmalloc_limit_max, 0);
get_envparam_double("RUBY_GC_OLDMALLOC_LIMIT_GROWTH_FACTOR", &gc_params.oldmalloc_limit_growth_factor, 1.0, 0.0, FALSE);
#endif
}
static void
reachable_objects_from_callback(VALUE obj)
{
rb_ractor_t *cr = GET_RACTOR();
cr->mfd->mark_func(obj, cr->mfd->data);
}
void
rb_objspace_reachable_objects_from(VALUE obj, void (func)(VALUE, void *), void *data)
{
rb_objspace_t *objspace = &rb_objspace;
RB_VM_LOCK_ENTER();
{
if (during_gc) rb_bug("rb_objspace_reachable_objects_from() is not supported while during_gc == true");
if (is_markable_object(objspace, obj)) {
rb_ractor_t *cr = GET_RACTOR();
struct gc_mark_func_data_struct mfd = {
.mark_func = func,
.data = data,
}, *prev_mfd = cr->mfd;
cr->mfd = &mfd;
gc_mark_children(objspace, obj);
cr->mfd = prev_mfd;
}
}
RB_VM_LOCK_LEAVE();
}
struct root_objects_data {
const char *category;
void (*func)(const char *category, VALUE, void *);
void *data;
};
static void
root_objects_from(VALUE obj, void *ptr)
{
const struct root_objects_data *data = (struct root_objects_data *)ptr;
(*data->func)(data->category, obj, data->data);
}
void
rb_objspace_reachable_objects_from_root(void (func)(const char *category, VALUE, void *), void *passing_data)
{
rb_objspace_t *objspace = &rb_objspace;
objspace_reachable_objects_from_root(objspace, func, passing_data);
}
static void
objspace_reachable_objects_from_root(rb_objspace_t *objspace, void (func)(const char *category, VALUE, void *), void *passing_data)
{
if (during_gc) rb_bug("objspace_reachable_objects_from_root() is not supported while during_gc == true");
rb_ractor_t *cr = GET_RACTOR();
struct root_objects_data data = {
.func = func,
.data = passing_data,
};
struct gc_mark_func_data_struct mfd = {
.mark_func = root_objects_from,
.data = &data,
}, *prev_mfd = cr->mfd;
cr->mfd = &mfd;
gc_mark_roots(objspace, &data.category);
cr->mfd = prev_mfd;
}
/*
------------------------ Extended allocator ------------------------
*/
struct gc_raise_tag {
VALUE exc;
const char *fmt;
va_list *ap;
};
static void *
gc_vraise(void *ptr)
{
struct gc_raise_tag *argv = ptr;
rb_vraise(argv->exc, argv->fmt, *argv->ap);
UNREACHABLE_RETURN(NULL);
}
static void
gc_raise(VALUE exc, const char *fmt, ...)
{
va_list ap;
va_start(ap, fmt);
struct gc_raise_tag argv = {
exc, fmt, &ap,
};
if (ruby_thread_has_gvl_p()) {
gc_vraise(&argv);
UNREACHABLE;
}
else if (ruby_native_thread_p()) {
rb_thread_call_with_gvl(gc_vraise, &argv);
UNREACHABLE;
}
else {
/* Not in a ruby thread */
fprintf(stderr, "%s", "[FATAL] ");
vfprintf(stderr, fmt, ap);
}
va_end(ap);
abort();
}
static void objspace_xfree(rb_objspace_t *objspace, void *ptr, size_t size);
static void
negative_size_allocation_error(const char *msg)
{
gc_raise(rb_eNoMemError, "%s", msg);
}
static void *
ruby_memerror_body(void *dummy)
{
rb_memerror();
return 0;
}
NORETURN(static void ruby_memerror(void));
RBIMPL_ATTR_MAYBE_UNUSED()
static void
ruby_memerror(void)
{
if (ruby_thread_has_gvl_p()) {
rb_memerror();
}
else {
if (ruby_native_thread_p()) {
rb_thread_call_with_gvl(ruby_memerror_body, 0);
}
else {
/* no ruby thread */
fprintf(stderr, "[FATAL] failed to allocate memory\n");
}
}
exit(EXIT_FAILURE);
}
void
rb_memerror(void)
{
rb_execution_context_t *ec = GET_EC();
rb_objspace_t *objspace = rb_objspace_of(rb_ec_vm_ptr(ec));
VALUE exc;
if (0) {
// Print out pid, sleep, so you can attach debugger to see what went wrong:
fprintf(stderr, "rb_memerror pid=%"PRI_PIDT_PREFIX"d\n", getpid());
sleep(60);
}
if (during_gc) {
// TODO: OMG!! How to implement it?
gc_exit(objspace, gc_enter_event_rb_memerror, NULL);
}
exc = nomem_error;
if (!exc ||
rb_ec_raised_p(ec, RAISED_NOMEMORY)) {
fprintf(stderr, "[FATAL] failed to allocate memory\n");
exit(EXIT_FAILURE);
}
if (rb_ec_raised_p(ec, RAISED_NOMEMORY)) {
rb_ec_raised_clear(ec);
}
else {
rb_ec_raised_set(ec, RAISED_NOMEMORY);
exc = ruby_vm_special_exception_copy(exc);
}
ec->errinfo = exc;
EC_JUMP_TAG(ec, TAG_RAISE);
}
void *
rb_aligned_malloc(size_t alignment, size_t size)
{
/* alignment must be a power of 2 */
GC_ASSERT(((alignment - 1) & alignment) == 0);
GC_ASSERT(alignment % sizeof(void*) == 0);
void *res;
#if defined __MINGW32__
res = __mingw_aligned_malloc(size, alignment);
#elif defined _WIN32
void *_aligned_malloc(size_t, size_t);
res = _aligned_malloc(size, alignment);
#elif defined(HAVE_POSIX_MEMALIGN)
if (posix_memalign(&res, alignment, size) != 0) {
return NULL;
}
#elif defined(HAVE_MEMALIGN)
res = memalign(alignment, size);
#else
char* aligned;
res = malloc(alignment + size + sizeof(void*));
aligned = (char*)res + alignment + sizeof(void*);
aligned -= ((VALUE)aligned & (alignment - 1));
((void**)aligned)[-1] = res;
res = (void*)aligned;
#endif
GC_ASSERT((uintptr_t)res % alignment == 0);
return res;
}
static void
rb_aligned_free(void *ptr, size_t size)
{
#if defined __MINGW32__
__mingw_aligned_free(ptr);
#elif defined _WIN32
_aligned_free(ptr);
#elif defined(HAVE_POSIX_MEMALIGN) || defined(HAVE_MEMALIGN)
free(ptr);
#else
free(((void**)ptr)[-1]);
#endif
}
static inline size_t
objspace_malloc_size(rb_objspace_t *objspace, void *ptr, size_t hint)
{
#ifdef HAVE_MALLOC_USABLE_SIZE
return malloc_usable_size(ptr);
#else
return hint;
#endif
}
enum memop_type {
MEMOP_TYPE_MALLOC = 0,
MEMOP_TYPE_FREE,
MEMOP_TYPE_REALLOC
};
static inline void
atomic_sub_nounderflow(size_t *var, size_t sub)
{
if (sub == 0) return;
while (1) {
size_t val = *var;
if (val < sub) sub = val;
if (ATOMIC_SIZE_CAS(*var, val, val-sub) == val) break;
}
}
static void
objspace_malloc_gc_stress(rb_objspace_t *objspace)
{
if (ruby_gc_stressful && ruby_native_thread_p()) {
unsigned int reason = (GPR_FLAG_IMMEDIATE_MARK | GPR_FLAG_IMMEDIATE_SWEEP |
GPR_FLAG_STRESS | GPR_FLAG_MALLOC);
if (gc_stress_full_mark_after_malloc_p()) {
reason |= GPR_FLAG_FULL_MARK;
}
garbage_collect_with_gvl(objspace, reason);
}
}
static inline bool
objspace_malloc_increase_report(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type)
{
if (0) fprintf(stderr, "increase - ptr: %p, type: %s, new_size: %"PRIdSIZE", old_size: %"PRIdSIZE"\n",
mem,
type == MEMOP_TYPE_MALLOC ? "malloc" :
type == MEMOP_TYPE_FREE ? "free " :
type == MEMOP_TYPE_REALLOC ? "realloc": "error",
new_size, old_size);
return false;
}
static bool
objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_size, size_t old_size, enum memop_type type)
{
if (new_size > old_size) {
ATOMIC_SIZE_ADD(malloc_increase, new_size - old_size);
#if RGENGC_ESTIMATE_OLDMALLOC
ATOMIC_SIZE_ADD(objspace->rgengc.oldmalloc_increase, new_size - old_size);
#endif
}
else {
atomic_sub_nounderflow(&malloc_increase, old_size - new_size);
#if RGENGC_ESTIMATE_OLDMALLOC
atomic_sub_nounderflow(&objspace->rgengc.oldmalloc_increase, old_size - new_size);
#endif
}
if (type == MEMOP_TYPE_MALLOC) {
retry:
if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) {
if (ruby_thread_has_gvl_p() && is_lazy_sweeping(objspace)) {
gc_rest(objspace); /* gc_rest can reduce malloc_increase */
goto retry;
}
garbage_collect_with_gvl(objspace, GPR_FLAG_MALLOC);
}
}
#if MALLOC_ALLOCATED_SIZE
if (new_size >= old_size) {
ATOMIC_SIZE_ADD(objspace->malloc_params.allocated_size, new_size - old_size);
}
else {
size_t dec_size = old_size - new_size;
size_t allocated_size = objspace->malloc_params.allocated_size;
#if MALLOC_ALLOCATED_SIZE_CHECK
if (allocated_size < dec_size) {
rb_bug("objspace_malloc_increase: underflow malloc_params.allocated_size.");
}
#endif
atomic_sub_nounderflow(&objspace->malloc_params.allocated_size, dec_size);
}
switch (type) {
case MEMOP_TYPE_MALLOC:
ATOMIC_SIZE_INC(objspace->malloc_params.allocations);
break;
case MEMOP_TYPE_FREE:
{
size_t allocations = objspace->malloc_params.allocations;
if (allocations > 0) {
atomic_sub_nounderflow(&objspace->malloc_params.allocations, 1);
}
#if MALLOC_ALLOCATED_SIZE_CHECK
else {
GC_ASSERT(objspace->malloc_params.allocations > 0);
}
#endif
}
break;
case MEMOP_TYPE_REALLOC: /* ignore */ break;
}
#endif
return true;
}
#define objspace_malloc_increase(...) \
for (bool malloc_increase_done = objspace_malloc_increase_report(__VA_ARGS__); \
!malloc_increase_done; \
malloc_increase_done = objspace_malloc_increase_body(__VA_ARGS__))
struct malloc_obj_info { /* 4 words */
size_t size;
#if USE_GC_MALLOC_OBJ_INFO_DETAILS
size_t gen;
const char *file;
size_t line;
#endif
};
#if USE_GC_MALLOC_OBJ_INFO_DETAILS
const char *ruby_malloc_info_file;
int ruby_malloc_info_line;
#endif
static inline size_t
objspace_malloc_prepare(rb_objspace_t *objspace, size_t size)
{
if (size == 0) size = 1;
#if CALC_EXACT_MALLOC_SIZE
size += sizeof(struct malloc_obj_info);
#endif
return size;
}
static inline void *
objspace_malloc_fixup(rb_objspace_t *objspace, void *mem, size_t size)
{
size = objspace_malloc_size(objspace, mem, size);
objspace_malloc_increase(objspace, mem, size, 0, MEMOP_TYPE_MALLOC);
#if CALC_EXACT_MALLOC_SIZE
{
struct malloc_obj_info *info = mem;
info->size = size;
#if USE_GC_MALLOC_OBJ_INFO_DETAILS
info->gen = objspace->profile.count;
info->file = ruby_malloc_info_file;
info->line = info->file ? ruby_malloc_info_line : 0;
#endif
mem = info + 1;
}
#endif
return mem;
}
#if defined(__GNUC__) && RUBY_DEBUG
#define RB_BUG_INSTEAD_OF_RB_MEMERROR 1
#endif
#ifndef RB_BUG_INSTEAD_OF_RB_MEMERROR
# define RB_BUG_INSTEAD_OF_RB_MEMERROR 0
#endif
#define GC_MEMERROR(...) \
((RB_BUG_INSTEAD_OF_RB_MEMERROR+0) ? rb_bug("" __VA_ARGS__) : rb_memerror())
#define TRY_WITH_GC(siz, expr) do { \
const gc_profile_record_flag gpr = \
GPR_FLAG_FULL_MARK | \
GPR_FLAG_IMMEDIATE_MARK | \
GPR_FLAG_IMMEDIATE_SWEEP | \
GPR_FLAG_MALLOC; \
objspace_malloc_gc_stress(objspace); \
\
if (LIKELY((expr))) { \
/* Success on 1st try */ \
} \
else if (!garbage_collect_with_gvl(objspace, gpr)) { \
/* @shyouhei thinks this doesn't happen */ \
GC_MEMERROR("TRY_WITH_GC: could not GC"); \
} \
else if ((expr)) { \
/* Success on 2nd try */ \
} \
else { \
GC_MEMERROR("TRY_WITH_GC: could not allocate:" \
"%"PRIdSIZE" bytes for %s", \
siz, # expr); \
} \
} while (0)
/* these shouldn't be called directly.
* objspace_* functions do not check allocation size.
*/
static void *
objspace_xmalloc0(rb_objspace_t *objspace, size_t size)
{
void *mem;
size = objspace_malloc_prepare(objspace, size);
TRY_WITH_GC(size, mem = malloc(size));
RB_DEBUG_COUNTER_INC(heap_xmalloc);
return objspace_malloc_fixup(objspace, mem, size);
}
static inline size_t
xmalloc2_size(const size_t count, const size_t elsize)
{
return size_mul_or_raise(count, elsize, rb_eArgError);
}
static void *
objspace_xrealloc(rb_objspace_t *objspace, void *ptr, size_t new_size, size_t old_size)
{
void *mem;
if (!ptr) return objspace_xmalloc0(objspace, new_size);
/*
* The behavior of realloc(ptr, 0) is implementation defined.
* Therefore we don't use realloc(ptr, 0) for portability reason.
* see http://www.open-std.org/jtc1/sc22/wg14/www/docs/dr_400.htm
*/
if (new_size == 0) {
if ((mem = objspace_xmalloc0(objspace, 0)) != NULL) {
/*
* - OpenBSD's malloc(3) man page says that when 0 is passed, it
* returns a non-NULL pointer to an access-protected memory page.
* The returned pointer cannot be read / written at all, but
* still be a valid argument of free().
*
* https://man.openbsd.org/malloc.3
*
* - Linux's malloc(3) man page says that it _might_ perhaps return
* a non-NULL pointer when its argument is 0. That return value
* is safe (and is expected) to be passed to free().
*
* https://man7.org/linux/man-pages/man3/malloc.3.html
*
* - As I read the implementation jemalloc's malloc() returns fully
* normal 16 bytes memory region when its argument is 0.
*
* - As I read the implementation musl libc's malloc() returns
* fully normal 32 bytes memory region when its argument is 0.
*
* - Other malloc implementations can also return non-NULL.
*/
objspace_xfree(objspace, ptr, old_size);
return mem;
}
else {
/*
* It is dangerous to return NULL here, because that could lead to
* RCE. Fallback to 1 byte instead of zero.
*
* https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-11932
*/
new_size = 1;
}
}
#if CALC_EXACT_MALLOC_SIZE
{
struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
new_size += sizeof(struct malloc_obj_info);
ptr = info;
old_size = info->size;
}
#endif
old_size = objspace_malloc_size(objspace, ptr, old_size);
TRY_WITH_GC(new_size, mem = RB_GNUC_EXTENSION_BLOCK(realloc(ptr, new_size)));
new_size = objspace_malloc_size(objspace, mem, new_size);
#if CALC_EXACT_MALLOC_SIZE
{
struct malloc_obj_info *info = mem;
info->size = new_size;
mem = info + 1;
}
#endif
objspace_malloc_increase(objspace, mem, new_size, old_size, MEMOP_TYPE_REALLOC);
RB_DEBUG_COUNTER_INC(heap_xrealloc);
return mem;
}
#if CALC_EXACT_MALLOC_SIZE && USE_GC_MALLOC_OBJ_INFO_DETAILS
#define MALLOC_INFO_GEN_SIZE 100
#define MALLOC_INFO_SIZE_SIZE 10
static size_t malloc_info_gen_cnt[MALLOC_INFO_GEN_SIZE];
static size_t malloc_info_gen_size[MALLOC_INFO_GEN_SIZE];
static size_t malloc_info_size[MALLOC_INFO_SIZE_SIZE+1];
static st_table *malloc_info_file_table;
static int
mmalloc_info_file_i(st_data_t key, st_data_t val, st_data_t dmy)
{
const char *file = (void *)key;
const size_t *data = (void *)val;
fprintf(stderr, "%s\t%"PRIdSIZE"\t%"PRIdSIZE"\n", file, data[0], data[1]);
return ST_CONTINUE;
}
__attribute__((destructor))
void
rb_malloc_info_show_results(void)
{
int i;
fprintf(stderr, "* malloc_info gen statistics\n");
for (i=0; i<MALLOC_INFO_GEN_SIZE; i++) {
if (i == MALLOC_INFO_GEN_SIZE-1) {
fprintf(stderr, "more\t%"PRIdSIZE"\t%"PRIdSIZE"\n", malloc_info_gen_cnt[i], malloc_info_gen_size[i]);
}
else {
fprintf(stderr, "%d\t%"PRIdSIZE"\t%"PRIdSIZE"\n", i, malloc_info_gen_cnt[i], malloc_info_gen_size[i]);
}
}
fprintf(stderr, "* malloc_info size statistics\n");
for (i=0; i<MALLOC_INFO_SIZE_SIZE; i++) {
int s = 16 << i;
fprintf(stderr, "%d\t%"PRIdSIZE"\n", s, malloc_info_size[i]);
}
fprintf(stderr, "more\t%"PRIdSIZE"\n", malloc_info_size[i]);
if (malloc_info_file_table) {
fprintf(stderr, "* malloc_info file statistics\n");
st_foreach(malloc_info_file_table, mmalloc_info_file_i, 0);
}
}
#else
void
rb_malloc_info_show_results(void)
{
}
#endif
static void
objspace_xfree(rb_objspace_t *objspace, void *ptr, size_t old_size)
{
if (!ptr) {
/*
* ISO/IEC 9899 says "If ptr is a null pointer, no action occurs" since
* its first version. We would better follow.
*/
return;
}
#if CALC_EXACT_MALLOC_SIZE
struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
ptr = info;
old_size = info->size;
#if USE_GC_MALLOC_OBJ_INFO_DETAILS
{
int gen = (int)(objspace->profile.count - info->gen);
int gen_index = gen >= MALLOC_INFO_GEN_SIZE ? MALLOC_INFO_GEN_SIZE-1 : gen;
int i;
malloc_info_gen_cnt[gen_index]++;
malloc_info_gen_size[gen_index] += info->size;
for (i=0; i<MALLOC_INFO_SIZE_SIZE; i++) {
size_t s = 16 << i;
if (info->size <= s) {
malloc_info_size[i]++;
goto found;
}
}
malloc_info_size[i]++;
found:;
{
st_data_t key = (st_data_t)info->file, d;
size_t *data;
if (malloc_info_file_table == NULL) {
malloc_info_file_table = st_init_numtable_with_size(1024);
}
if (st_lookup(malloc_info_file_table, key, &d)) {
/* hit */
data = (size_t *)d;
}
else {
data = malloc(xmalloc2_size(2, sizeof(size_t)));
if (data == NULL) rb_bug("objspace_xfree: can not allocate memory");
data[0] = data[1] = 0;
st_insert(malloc_info_file_table, key, (st_data_t)data);
}
data[0] ++;
data[1] += info->size;
};
if (0 && gen >= 2) { /* verbose output */
if (info->file) {
fprintf(stderr, "free - size:%"PRIdSIZE", gen:%d, pos: %s:%"PRIdSIZE"\n",
info->size, gen, info->file, info->line);
}
else {
fprintf(stderr, "free - size:%"PRIdSIZE", gen:%d\n",
info->size, gen);
}
}
}
#endif
#endif
old_size = objspace_malloc_size(objspace, ptr, old_size);
objspace_malloc_increase(objspace, ptr, 0, old_size, MEMOP_TYPE_FREE) {
free(ptr);
ptr = NULL;
RB_DEBUG_COUNTER_INC(heap_xfree);
}
}
static void *
ruby_xmalloc0(size_t size)
{
return objspace_xmalloc0(&rb_objspace, size);
}
void *
ruby_xmalloc_body(size_t size)
{
if ((ssize_t)size < 0) {
negative_size_allocation_error("too large allocation size");
}
return ruby_xmalloc0(size);
}
void
ruby_malloc_size_overflow(size_t count, size_t elsize)
{
rb_raise(rb_eArgError,
"malloc: possible integer overflow (%"PRIuSIZE"*%"PRIuSIZE")",
count, elsize);
}
void *
ruby_xmalloc2_body(size_t n, size_t size)
{
return objspace_xmalloc0(&rb_objspace, xmalloc2_size(n, size));
}
static void *
objspace_xcalloc(rb_objspace_t *objspace, size_t size)
{
void *mem;
size = objspace_malloc_prepare(objspace, size);
TRY_WITH_GC(size, mem = calloc1(size));
return objspace_malloc_fixup(objspace, mem, size);
}
void *
ruby_xcalloc_body(size_t n, size_t size)
{
return objspace_xcalloc(&rb_objspace, xmalloc2_size(n, size));
}
#ifdef ruby_sized_xrealloc
#undef ruby_sized_xrealloc
#endif
void *
ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size)
{
if ((ssize_t)new_size < 0) {
negative_size_allocation_error("too large allocation size");
}
return objspace_xrealloc(&rb_objspace, ptr, new_size, old_size);
}
void *
ruby_xrealloc_body(void *ptr, size_t new_size)
{
return ruby_sized_xrealloc(ptr, new_size, 0);
}
#ifdef ruby_sized_xrealloc2
#undef ruby_sized_xrealloc2
#endif
void *
ruby_sized_xrealloc2(void *ptr, size_t n, size_t size, size_t old_n)
{
size_t len = xmalloc2_size(n, size);
return objspace_xrealloc(&rb_objspace, ptr, len, old_n * size);
}
void *
ruby_xrealloc2_body(void *ptr, size_t n, size_t size)
{
return ruby_sized_xrealloc2(ptr, n, size, 0);
}
#ifdef ruby_sized_xfree
#undef ruby_sized_xfree
#endif
void
ruby_sized_xfree(void *x, size_t size)
{
if (x) {
objspace_xfree(&rb_objspace, x, size);
}
}
void
ruby_xfree(void *x)
{
ruby_sized_xfree(x, 0);
}
void *
rb_xmalloc_mul_add(size_t x, size_t y, size_t z) /* x * y + z */
{
size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
return ruby_xmalloc(w);
}
void *
rb_xcalloc_mul_add(size_t x, size_t y, size_t z) /* x * y + z */
{
size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
return ruby_xcalloc(w, 1);
}
void *
rb_xrealloc_mul_add(const void *p, size_t x, size_t y, size_t z) /* x * y + z */
{
size_t w = size_mul_add_or_raise(x, y, z, rb_eArgError);
return ruby_xrealloc((void *)p, w);
}
void *
rb_xmalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
{
size_t u = size_mul_add_mul_or_raise(x, y, z, w, rb_eArgError);
return ruby_xmalloc(u);
}
void *
rb_xcalloc_mul_add_mul(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
{
size_t u = size_mul_add_mul_or_raise(x, y, z, w, rb_eArgError);
return ruby_xcalloc(u, 1);
}
/* Mimic ruby_xmalloc, but need not rb_objspace.
* should return pointer suitable for ruby_xfree
*/
void *
ruby_mimmalloc(size_t size)
{
void *mem;
#if CALC_EXACT_MALLOC_SIZE
size += sizeof(struct malloc_obj_info);
#endif
mem = malloc(size);
#if CALC_EXACT_MALLOC_SIZE
if (!mem) {
return NULL;
}
else
/* set 0 for consistency of allocated_size/allocations */
{
struct malloc_obj_info *info = mem;
info->size = 0;
#if USE_GC_MALLOC_OBJ_INFO_DETAILS
info->gen = 0;
info->file = NULL;
info->line = 0;
#endif
mem = info + 1;
}
#endif
return mem;
}
void
ruby_mimfree(void *ptr)
{
#if CALC_EXACT_MALLOC_SIZE
struct malloc_obj_info *info = (struct malloc_obj_info *)ptr - 1;
ptr = info;
#endif
free(ptr);
}
void *
rb_alloc_tmp_buffer_with_count(volatile VALUE *store, size_t size, size_t cnt)
{
void *ptr;
VALUE imemo;
rb_imemo_tmpbuf_t *tmpbuf;
/* Keep the order; allocate an empty imemo first then xmalloc, to
* get rid of potential memory leak */
imemo = rb_imemo_tmpbuf_auto_free_maybe_mark_buffer(NULL, 0);
*store = imemo;
ptr = ruby_xmalloc0(size);
tmpbuf = (rb_imemo_tmpbuf_t *)imemo;
tmpbuf->ptr = ptr;
tmpbuf->cnt = cnt;
return ptr;
}
void *
rb_alloc_tmp_buffer(volatile VALUE *store, long len)
{
long cnt;
if (len < 0 || (cnt = (long)roomof(len, sizeof(VALUE))) < 0) {
rb_raise(rb_eArgError, "negative buffer size (or size too big)");
}
return rb_alloc_tmp_buffer_with_count(store, len, cnt);
}
void
rb_free_tmp_buffer(volatile VALUE *store)
{
rb_imemo_tmpbuf_t *s = (rb_imemo_tmpbuf_t*)ATOMIC_VALUE_EXCHANGE(*store, 0);
if (s) {
void *ptr = ATOMIC_PTR_EXCHANGE(s->ptr, 0);
s->cnt = 0;
ruby_xfree(ptr);
}
}
#if MALLOC_ALLOCATED_SIZE
/*
* call-seq:
* GC.malloc_allocated_size -> Integer
*
* Returns the size of memory allocated by malloc().
*
* Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
*/
static VALUE
gc_malloc_allocated_size(VALUE self)
{
return UINT2NUM(rb_objspace.malloc_params.allocated_size);
}
/*
* call-seq:
* GC.malloc_allocations -> Integer
*
* Returns the number of malloc() allocations.
*
* Only available if ruby was built with +CALC_EXACT_MALLOC_SIZE+.
*/
static VALUE
gc_malloc_allocations(VALUE self)
{
return UINT2NUM(rb_objspace.malloc_params.allocations);
}
#endif
void
rb_gc_adjust_memory_usage(ssize_t diff)
{
rb_objspace_t *objspace = &rb_objspace;
if (diff > 0) {
objspace_malloc_increase(objspace, 0, diff, 0, MEMOP_TYPE_REALLOC);
}
else if (diff < 0) {
objspace_malloc_increase(objspace, 0, 0, -diff, MEMOP_TYPE_REALLOC);
}
}
/*
------------------------------ WeakMap ------------------------------
*/
struct weakmap {
st_table *obj2wmap; /* obj -> [ref,...] */
st_table *wmap2obj; /* ref -> obj */
VALUE final;
};
#define WMAP_DELETE_DEAD_OBJECT_IN_MARK 0
#if WMAP_DELETE_DEAD_OBJECT_IN_MARK
static int
wmap_mark_map(st_data_t key, st_data_t val, st_data_t arg)
{
rb_objspace_t *objspace = (rb_objspace_t *)arg;
VALUE obj = (VALUE)val;
if (!is_live_object(objspace, obj)) return ST_DELETE;
return ST_CONTINUE;
}
#endif
static void
wmap_compact(void *ptr)
{
struct weakmap *w = ptr;
if (w->wmap2obj) rb_gc_update_tbl_refs(w->wmap2obj);
if (w->obj2wmap) rb_gc_update_tbl_refs(w->obj2wmap);
w->final = rb_gc_location(w->final);
}
static void
wmap_mark(void *ptr)
{
struct weakmap *w = ptr;
#if WMAP_DELETE_DEAD_OBJECT_IN_MARK
if (w->obj2wmap) st_foreach(w->obj2wmap, wmap_mark_map, (st_data_t)&rb_objspace);
#endif
rb_gc_mark_movable(w->final);
}
static int
wmap_free_map(st_data_t key, st_data_t val, st_data_t arg)
{
VALUE *ptr = (VALUE *)val;
ruby_sized_xfree(ptr, (ptr[0] + 1) * sizeof(VALUE));
return ST_CONTINUE;
}
static void
wmap_free(void *ptr)
{
struct weakmap *w = ptr;
st_foreach(w->obj2wmap, wmap_free_map, 0);
st_free_table(w->obj2wmap);
st_free_table(w->wmap2obj);
}
static int
wmap_memsize_map(st_data_t key, st_data_t val, st_data_t arg)
{
VALUE *ptr = (VALUE *)val;
*(size_t *)arg += (ptr[0] + 1) * sizeof(VALUE);
return ST_CONTINUE;
}
static size_t
wmap_memsize(const void *ptr)
{
size_t size;
const struct weakmap *w = ptr;
size = sizeof(*w);
size += st_memsize(w->obj2wmap);
size += st_memsize(w->wmap2obj);
st_foreach(w->obj2wmap, wmap_memsize_map, (st_data_t)&size);
return size;
}
static const rb_data_type_t weakmap_type = {
"weakmap",
{
wmap_mark,
wmap_free,
wmap_memsize,
wmap_compact,
},
0, 0, RUBY_TYPED_FREE_IMMEDIATELY
};
static VALUE wmap_finalize(RB_BLOCK_CALL_FUNC_ARGLIST(objid, self));
static VALUE
wmap_allocate(VALUE klass)
{
struct weakmap *w;
VALUE obj = TypedData_Make_Struct(klass, struct weakmap, &weakmap_type, w);
w->obj2wmap = rb_init_identtable();
w->wmap2obj = rb_init_identtable();
w->final = rb_func_lambda_new(wmap_finalize, obj, 1, 1);
return obj;
}
static int
wmap_live_p(rb_objspace_t *objspace, VALUE obj)
{
if (SPECIAL_CONST_P(obj)) return TRUE;
/* If is_pointer_to_heap returns false, the page could be in the tomb heap
* or have already been freed. */
if (!is_pointer_to_heap(objspace, (void *)obj)) return FALSE;
void *poisoned = asan_unpoison_object_temporary(obj);
enum ruby_value_type t = BUILTIN_TYPE(obj);
int ret = (!(t == T_NONE || t >= T_FIXNUM || t == T_ICLASS) &&
is_live_object(objspace, obj));
if (poisoned) {
asan_poison_object(obj);
}
return ret;
}
static int
wmap_final_func(st_data_t *key, st_data_t *value, st_data_t arg, int existing)
{
VALUE wmap, *ptr, size, i, j;
if (!existing) return ST_STOP;
wmap = (VALUE)arg, ptr = (VALUE *)*value;
for (i = j = 1, size = ptr[0]; i <= size; ++i) {
if (ptr[i] != wmap) {
ptr[j++] = ptr[i];
}
}
if (j == 1) {
ruby_sized_xfree(ptr, i * sizeof(VALUE));
return ST_DELETE;
}
if (j < i) {
SIZED_REALLOC_N(ptr, VALUE, j + 1, i);
ptr[0] = j;
*value = (st_data_t)ptr;
}
return ST_CONTINUE;
}
/* :nodoc: */
static VALUE
wmap_finalize(RB_BLOCK_CALL_FUNC_ARGLIST(objid, self))
{
st_data_t orig, wmap, data;
VALUE obj, *rids, i, size;
struct weakmap *w;
TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
/* Get reference from object id. */
if ((obj = id2ref_obj_tbl(&rb_objspace, objid)) == Qundef) {
rb_bug("wmap_finalize: objid is not found.");
}
/* obj is original referenced object and/or weak reference. */
orig = (st_data_t)obj;
if (st_delete(w->obj2wmap, &orig, &data)) {
rids = (VALUE *)data;
size = *rids++;
for (i = 0; i < size; ++i) {
wmap = (st_data_t)rids[i];
st_delete(w->wmap2obj, &wmap, NULL);
}
ruby_sized_xfree((VALUE *)data, (size + 1) * sizeof(VALUE));
}
wmap = (st_data_t)obj;
if (st_delete(w->wmap2obj, &wmap, &orig)) {
wmap = (st_data_t)obj;
st_update(w->obj2wmap, orig, wmap_final_func, wmap);
}
return self;
}
struct wmap_iter_arg {
rb_objspace_t *objspace;
VALUE value;
};
static VALUE
wmap_inspect_append(rb_objspace_t *objspace, VALUE str, VALUE obj)
{
if (SPECIAL_CONST_P(obj)) {
return rb_str_append(str, rb_inspect(obj));
}
else if (wmap_live_p(objspace, obj)) {
return rb_str_append(str, rb_any_to_s(obj));
}
else {
return rb_str_catf(str, "#<collected:%p>", (void*)obj);
}
}
static int
wmap_inspect_i(st_data_t key, st_data_t val, st_data_t arg)
{
struct wmap_iter_arg *argp = (struct wmap_iter_arg *)arg;
rb_objspace_t *objspace = argp->objspace;
VALUE str = argp->value;
VALUE k = (VALUE)key, v = (VALUE)val;
if (RSTRING_PTR(str)[0] == '#') {
rb_str_cat2(str, ", ");
}
else {
rb_str_cat2(str, ": ");
RSTRING_PTR(str)[0] = '#';
}
wmap_inspect_append(objspace, str, k);
rb_str_cat2(str, " => ");
wmap_inspect_append(objspace, str, v);
return ST_CONTINUE;
}
static VALUE
wmap_inspect(VALUE self)
{
VALUE str;
VALUE c = rb_class_name(CLASS_OF(self));
struct weakmap *w;
struct wmap_iter_arg args;
TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
str = rb_sprintf("-<%"PRIsVALUE":%p", c, (void *)self);
if (w->wmap2obj) {
args.objspace = &rb_objspace;
args.value = str;
st_foreach(w->wmap2obj, wmap_inspect_i, (st_data_t)&args);
}
RSTRING_PTR(str)[0] = '#';
rb_str_cat2(str, ">");
return str;
}
static inline bool
wmap_live_entry_p(rb_objspace_t *objspace, st_data_t key, st_data_t val)
{
return wmap_live_p(objspace, (VALUE)key) && wmap_live_p(objspace, (VALUE)val);
}
static int
wmap_each_i(st_data_t key, st_data_t val, st_data_t arg)
{
rb_objspace_t *objspace = (rb_objspace_t *)arg;
if (wmap_live_entry_p(objspace, key, val)) {
rb_yield_values(2, (VALUE)key, (VALUE)val);
return ST_CONTINUE;
}
else {
return ST_DELETE;
}
}
/* Iterates over keys and objects in a weakly referenced object */
static VALUE
wmap_each(VALUE self)
{
struct weakmap *w;
rb_objspace_t *objspace = &rb_objspace;
TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
st_foreach(w->wmap2obj, wmap_each_i, (st_data_t)objspace);
return self;
}
static int
wmap_each_key_i(st_data_t key, st_data_t val, st_data_t arg)
{
rb_objspace_t *objspace = (rb_objspace_t *)arg;
if (wmap_live_entry_p(objspace, key, val)) {
rb_yield((VALUE)key);
return ST_CONTINUE;
}
else {
return ST_DELETE;
}
}
/* Iterates over keys and objects in a weakly referenced object */
static VALUE
wmap_each_key(VALUE self)
{
struct weakmap *w;
rb_objspace_t *objspace = &rb_objspace;
TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
st_foreach(w->wmap2obj, wmap_each_key_i, (st_data_t)objspace);
return self;
}
static int
wmap_each_value_i(st_data_t key, st_data_t val, st_data_t arg)
{
rb_objspace_t *objspace = (rb_objspace_t *)arg;
if (wmap_live_entry_p(objspace, key, val)) {
rb_yield((VALUE)val);
return ST_CONTINUE;
}
else {
return ST_DELETE;
}
}
/* Iterates over keys and objects in a weakly referenced object */
static VALUE
wmap_each_value(VALUE self)
{
struct weakmap *w;
rb_objspace_t *objspace = &rb_objspace;
TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
st_foreach(w->wmap2obj, wmap_each_value_i, (st_data_t)objspace);
return self;
}
static int
wmap_keys_i(st_data_t key, st_data_t val, st_data_t arg)
{
struct wmap_iter_arg *argp = (struct wmap_iter_arg *)arg;
rb_objspace_t *objspace = argp->objspace;
VALUE ary = argp->value;
if (wmap_live_entry_p(objspace, key, val)) {
rb_ary_push(ary, (VALUE)key);
return ST_CONTINUE;
}
else {
return ST_DELETE;
}
}
/* Iterates over keys and objects in a weakly referenced object */
static VALUE
wmap_keys(VALUE self)
{
struct weakmap *w;
struct wmap_iter_arg args;
TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
args.objspace = &rb_objspace;
args.value = rb_ary_new();
st_foreach(w->wmap2obj, wmap_keys_i, (st_data_t)&args);
return args.value;
}
static int
wmap_values_i(st_data_t key, st_data_t val, st_data_t arg)
{
struct wmap_iter_arg *argp = (struct wmap_iter_arg *)arg;
rb_objspace_t *objspace = argp->objspace;
VALUE ary = argp->value;
if (wmap_live_entry_p(objspace, key, val)) {
rb_ary_push(ary, (VALUE)val);
return ST_CONTINUE;
}
else {
return ST_DELETE;
}
}
/* Iterates over values and objects in a weakly referenced object */
static VALUE
wmap_values(VALUE self)
{
struct weakmap *w;
struct wmap_iter_arg args;
TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
args.objspace = &rb_objspace;
args.value = rb_ary_new();
st_foreach(w->wmap2obj, wmap_values_i, (st_data_t)&args);
return args.value;
}
static int
wmap_aset_update(st_data_t *key, st_data_t *val, st_data_t arg, int existing)
{
VALUE size, *ptr, *optr;
if (existing) {
size = (ptr = optr = (VALUE *)*val)[0];
++size;
SIZED_REALLOC_N(ptr, VALUE, size + 1, size);
}
else {
optr = 0;
size = 1;
ptr = ruby_xmalloc0(2 * sizeof(VALUE));
}
ptr[0] = size;
ptr[size] = (VALUE)arg;
if (ptr == optr) return ST_STOP;
*val = (st_data_t)ptr;
return ST_CONTINUE;
}
/* Creates a weak reference from the given key to the given value */
static VALUE
wmap_aset(VALUE self, VALUE key, VALUE value)
{
struct weakmap *w;
TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
if (FL_ABLE(value)) {
define_final0(value, w->final);
}
if (FL_ABLE(key)) {
define_final0(key, w->final);
}
st_update(w->obj2wmap, (st_data_t)value, wmap_aset_update, key);
st_insert(w->wmap2obj, (st_data_t)key, (st_data_t)value);
return nonspecial_obj_id(value);
}
/* Retrieves a weakly referenced object with the given key */
static VALUE
wmap_lookup(VALUE self, VALUE key)
{
st_data_t data;
VALUE obj;
struct weakmap *w;
rb_objspace_t *objspace = &rb_objspace;
GC_ASSERT(wmap_live_p(objspace, key));
TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
if (!st_lookup(w->wmap2obj, (st_data_t)key, &data)) return Qundef;
obj = (VALUE)data;
if (!wmap_live_p(objspace, obj)) return Qundef;
return obj;
}
/* Retrieves a weakly referenced object with the given key */
static VALUE
wmap_aref(VALUE self, VALUE key)
{
VALUE obj = wmap_lookup(self, key);
return obj != Qundef ? obj : Qnil;
}
/* Returns +true+ if +key+ is registered */
static VALUE
wmap_has_key(VALUE self, VALUE key)
{
return RBOOL(wmap_lookup(self, key) != Qundef);
}
/* Returns the number of referenced objects */
static VALUE
wmap_size(VALUE self)
{
struct weakmap *w;
st_index_t n;
TypedData_Get_Struct(self, struct weakmap, &weakmap_type, w);
n = w->wmap2obj->num_entries;
#if SIZEOF_ST_INDEX_T <= SIZEOF_LONG
return ULONG2NUM(n);
#else
return ULL2NUM(n);
#endif
}
/*
------------------------------ GC profiler ------------------------------
*/
#define GC_PROFILE_RECORD_DEFAULT_SIZE 100
static bool
current_process_time(struct timespec *ts)
{
#if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
{
static int try_clock_gettime = 1;
if (try_clock_gettime && clock_gettime(CLOCK_PROCESS_CPUTIME_ID, ts) == 0) {
return true;
}
else {
try_clock_gettime = 0;
}
}
#endif
#ifdef RUSAGE_SELF
{
struct rusage usage;
struct timeval time;
if (getrusage(RUSAGE_SELF, &usage) == 0) {
time = usage.ru_utime;
ts->tv_sec = time.tv_sec;
ts->tv_nsec = (int32_t)time.tv_usec * 1000;
return true;
}
}
#endif
#ifdef _WIN32
{
FILETIME creation_time, exit_time, kernel_time, user_time;
ULARGE_INTEGER ui;
if (GetProcessTimes(GetCurrentProcess(),
&creation_time, &exit_time, &kernel_time, &user_time) != 0) {
memcpy(&ui, &user_time, sizeof(FILETIME));
#define PER100NSEC (uint64_t)(1000 * 1000 * 10)
ts->tv_nsec = (long)(ui.QuadPart % PER100NSEC);
ts->tv_sec = (time_t)(ui.QuadPart / PER100NSEC);
return true;
}
}
#endif
return false;
}
static double
getrusage_time(void)
{
struct timespec ts;
if (current_process_time(&ts)) {
return ts.tv_sec + ts.tv_nsec * 1e-9;
}
else {
return 0.0;
}
}
static inline void
gc_prof_setup_new_record(rb_objspace_t *objspace, unsigned int reason)
{
if (objspace->profile.run) {
size_t index = objspace->profile.next_index;
gc_profile_record *record;
/* create new record */
objspace->profile.next_index++;
if (!objspace->profile.records) {
objspace->profile.size = GC_PROFILE_RECORD_DEFAULT_SIZE;
objspace->profile.records = malloc(xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
}
if (index >= objspace->profile.size) {
void *ptr;
objspace->profile.size += 1000;
ptr = realloc(objspace->profile.records, xmalloc2_size(sizeof(gc_profile_record), objspace->profile.size));
if (!ptr) rb_memerror();
objspace->profile.records = ptr;
}
if (!objspace->profile.records) {
rb_bug("gc_profile malloc or realloc miss");
}
record = objspace->profile.current_record = &objspace->profile.records[objspace->profile.next_index - 1];
MEMZERO(record, gc_profile_record, 1);
/* setup before-GC parameter */
record->flags = reason | (ruby_gc_stressful ? GPR_FLAG_STRESS : 0);
#if MALLOC_ALLOCATED_SIZE
record->allocated_size = malloc_allocated_size;
#endif
#if GC_PROFILE_MORE_DETAIL && GC_PROFILE_DETAIL_MEMORY
#ifdef RUSAGE_SELF
{
struct rusage usage;
if (getrusage(RUSAGE_SELF, &usage) == 0) {
record->maxrss = usage.ru_maxrss;
record->minflt = usage.ru_minflt;
record->majflt = usage.ru_majflt;
}
}
#endif
#endif
}
}
static inline void
gc_prof_timer_start(rb_objspace_t *objspace)
{
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
#if GC_PROFILE_MORE_DETAIL
record->prepare_time = objspace->profile.prepare_time;
#endif
record->gc_time = 0;
record->gc_invoke_time = getrusage_time();
}
}
static double
elapsed_time_from(double time)
{
double now = getrusage_time();
if (now > time) {
return now - time;
}
else {
return 0;
}
}
static inline void
gc_prof_timer_stop(rb_objspace_t *objspace)
{
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->gc_time = elapsed_time_from(record->gc_invoke_time);
record->gc_invoke_time -= objspace->profile.invoke_time;
}
}
#define RUBY_DTRACE_GC_HOOK(name) \
do {if (RUBY_DTRACE_GC_##name##_ENABLED()) RUBY_DTRACE_GC_##name();} while (0)
static inline void
gc_prof_mark_timer_start(rb_objspace_t *objspace)
{
RUBY_DTRACE_GC_HOOK(MARK_BEGIN);
#if GC_PROFILE_MORE_DETAIL
if (gc_prof_enabled(objspace)) {
gc_prof_record(objspace)->gc_mark_time = getrusage_time();
}
#endif
}
static inline void
gc_prof_mark_timer_stop(rb_objspace_t *objspace)
{
RUBY_DTRACE_GC_HOOK(MARK_END);
#if GC_PROFILE_MORE_DETAIL
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->gc_mark_time = elapsed_time_from(record->gc_mark_time);
}
#endif
}
static inline void
gc_prof_sweep_timer_start(rb_objspace_t *objspace)
{
RUBY_DTRACE_GC_HOOK(SWEEP_BEGIN);
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
if (record->gc_time > 0 || GC_PROFILE_MORE_DETAIL) {
objspace->profile.gc_sweep_start_time = getrusage_time();
}
}
}
static inline void
gc_prof_sweep_timer_stop(rb_objspace_t *objspace)
{
RUBY_DTRACE_GC_HOOK(SWEEP_END);
if (gc_prof_enabled(objspace)) {
double sweep_time;
gc_profile_record *record = gc_prof_record(objspace);
if (record->gc_time > 0) {
sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
/* need to accumulate GC time for lazy sweep after gc() */
record->gc_time += sweep_time;
}
else if (GC_PROFILE_MORE_DETAIL) {
sweep_time = elapsed_time_from(objspace->profile.gc_sweep_start_time);
}
#if GC_PROFILE_MORE_DETAIL
record->gc_sweep_time += sweep_time;
if (heap_pages_deferred_final) record->flags |= GPR_FLAG_HAVE_FINALIZE;
#endif
if (heap_pages_deferred_final) objspace->profile.latest_gc_info |= GPR_FLAG_HAVE_FINALIZE;
}
}
static inline void
gc_prof_set_malloc_info(rb_objspace_t *objspace)
{
#if GC_PROFILE_MORE_DETAIL
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->allocate_increase = malloc_increase;
record->allocate_limit = malloc_limit;
}
#endif
}
static inline void
gc_prof_set_heap_info(rb_objspace_t *objspace)
{
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
size_t live = objspace->profile.total_allocated_objects_at_gc_start - objspace->profile.total_freed_objects;
size_t total = objspace->profile.heap_used_at_gc_start * HEAP_PAGE_OBJ_LIMIT;
#if GC_PROFILE_MORE_DETAIL
record->heap_use_pages = objspace->profile.heap_used_at_gc_start;
record->heap_live_objects = live;
record->heap_free_objects = total - live;
#endif
record->heap_total_objects = total;
record->heap_use_size = live * sizeof(RVALUE);
record->heap_total_size = total * sizeof(RVALUE);
}
}
/*
* call-seq:
* GC::Profiler.clear -> nil
*
* Clears the GC profiler data.
*
*/
static VALUE
gc_profile_clear(VALUE _)
{
rb_objspace_t *objspace = &rb_objspace;
void *p = objspace->profile.records;
objspace->profile.records = NULL;
objspace->profile.size = 0;
objspace->profile.next_index = 0;
objspace->profile.current_record = 0;
if (p) {
free(p);
}
return Qnil;
}
/*
* call-seq:
* GC::Profiler.raw_data -> [Hash, ...]
*
* Returns an Array of individual raw profile data Hashes ordered
* from earliest to latest by +:GC_INVOKE_TIME+.
*
* For example:
*
* [
* {
* :GC_TIME=>1.3000000000000858e-05,
* :GC_INVOKE_TIME=>0.010634999999999999,
* :HEAP_USE_SIZE=>289640,
* :HEAP_TOTAL_SIZE=>588960,
* :HEAP_TOTAL_OBJECTS=>14724,
* :GC_IS_MARKED=>false
* },
* # ...
* ]
*
* The keys mean:
*
* +:GC_TIME+::
* Time elapsed in seconds for this GC run
* +:GC_INVOKE_TIME+::
* Time elapsed in seconds from startup to when the GC was invoked
* +:HEAP_USE_SIZE+::
* Total bytes of heap used
* +:HEAP_TOTAL_SIZE+::
* Total size of heap in bytes
* +:HEAP_TOTAL_OBJECTS+::
* Total number of objects
* +:GC_IS_MARKED+::
* Returns +true+ if the GC is in mark phase
*
* If ruby was built with +GC_PROFILE_MORE_DETAIL+, you will also have access
* to the following hash keys:
*
* +:GC_MARK_TIME+::
* +:GC_SWEEP_TIME+::
* +:ALLOCATE_INCREASE+::
* +:ALLOCATE_LIMIT+::
* +:HEAP_USE_PAGES+::
* +:HEAP_LIVE_OBJECTS+::
* +:HEAP_FREE_OBJECTS+::
* +:HAVE_FINALIZE+::
*
*/
static VALUE
gc_profile_record_get(VALUE _)
{
VALUE prof;
VALUE gc_profile = rb_ary_new();
size_t i;
rb_objspace_t *objspace = (&rb_objspace);
if (!objspace->profile.run) {
return Qnil;
}
for (i =0; i < objspace->profile.next_index; i++) {
gc_profile_record *record = &objspace->profile.records[i];
prof = rb_hash_new();
rb_hash_aset(prof, ID2SYM(rb_intern("GC_FLAGS")), gc_info_decode(0, rb_hash_new(), record->flags));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_TIME")), DBL2NUM(record->gc_time));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_INVOKE_TIME")), DBL2NUM(record->gc_invoke_time));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_SIZE")), SIZET2NUM(record->heap_use_size));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_SIZE")), SIZET2NUM(record->heap_total_size));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_TOTAL_OBJECTS")), SIZET2NUM(record->heap_total_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("MOVED_OBJECTS")), SIZET2NUM(record->moved_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_IS_MARKED")), Qtrue);
#if GC_PROFILE_MORE_DETAIL
rb_hash_aset(prof, ID2SYM(rb_intern("GC_MARK_TIME")), DBL2NUM(record->gc_mark_time));
rb_hash_aset(prof, ID2SYM(rb_intern("GC_SWEEP_TIME")), DBL2NUM(record->gc_sweep_time));
rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_INCREASE")), SIZET2NUM(record->allocate_increase));
rb_hash_aset(prof, ID2SYM(rb_intern("ALLOCATE_LIMIT")), SIZET2NUM(record->allocate_limit));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_USE_PAGES")), SIZET2NUM(record->heap_use_pages));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_LIVE_OBJECTS")), SIZET2NUM(record->heap_live_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("HEAP_FREE_OBJECTS")), SIZET2NUM(record->heap_free_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("REMOVING_OBJECTS")), SIZET2NUM(record->removing_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("EMPTY_OBJECTS")), SIZET2NUM(record->empty_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("HAVE_FINALIZE")), RBOOL(record->flags & GPR_FLAG_HAVE_FINALIZE));
#endif
#if RGENGC_PROFILE > 0
rb_hash_aset(prof, ID2SYM(rb_intern("OLD_OBJECTS")), SIZET2NUM(record->old_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_NORMAL_OBJECTS")), SIZET2NUM(record->remembered_normal_objects));
rb_hash_aset(prof, ID2SYM(rb_intern("REMEMBERED_SHADY_OBJECTS")), SIZET2NUM(record->remembered_shady_objects));
#endif
rb_ary_push(gc_profile, prof);
}
return gc_profile;
}
#if GC_PROFILE_MORE_DETAIL
#define MAJOR_REASON_MAX 0x10
static char *
gc_profile_dump_major_reason(unsigned int flags, char *buff)
{
unsigned int reason = flags & GPR_FLAG_MAJOR_MASK;
int i = 0;
if (reason == GPR_FLAG_NONE) {
buff[0] = '-';
buff[1] = 0;
}
else {
#define C(x, s) \
if (reason & GPR_FLAG_MAJOR_BY_##x) { \
buff[i++] = #x[0]; \
if (i >= MAJOR_REASON_MAX) rb_bug("gc_profile_dump_major_reason: overflow"); \
buff[i] = 0; \
}
C(NOFREE, N);
C(OLDGEN, O);
C(SHADY, S);
#if RGENGC_ESTIMATE_OLDMALLOC
C(OLDMALLOC, M);
#endif
#undef C
}
return buff;
}
#endif
static void
gc_profile_dump_on(VALUE out, VALUE (*append)(VALUE, VALUE))
{
rb_objspace_t *objspace = &rb_objspace;
size_t count = objspace->profile.next_index;
#ifdef MAJOR_REASON_MAX
char reason_str[MAJOR_REASON_MAX];
#endif
if (objspace->profile.run && count /* > 1 */) {
size_t i;
const gc_profile_record *record;
append(out, rb_sprintf("GC %"PRIuSIZE" invokes.\n", objspace->profile.count));
append(out, rb_str_new_cstr("Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC Time(ms)\n"));
for (i = 0; i < count; i++) {
record = &objspace->profile.records[i];
append(out, rb_sprintf("%5"PRIuSIZE" %19.3f %20"PRIuSIZE" %20"PRIuSIZE" %20"PRIuSIZE" %30.20f\n",
i+1, record->gc_invoke_time, record->heap_use_size,
record->heap_total_size, record->heap_total_objects, record->gc_time*1000));
}
#if GC_PROFILE_MORE_DETAIL
const char *str = "\n\n" \
"More detail.\n" \
"Prepare Time = Previously GC's rest sweep time\n"
"Index Flags Allocate Inc. Allocate Limit"
#if CALC_EXACT_MALLOC_SIZE
" Allocated Size"
#endif
" Use Page Mark Time(ms) Sweep Time(ms) Prepare Time(ms) LivingObj FreeObj RemovedObj EmptyObj"
#if RGENGC_PROFILE
" OldgenObj RemNormObj RemShadObj"
#endif
#if GC_PROFILE_DETAIL_MEMORY
" MaxRSS(KB) MinorFLT MajorFLT"
#endif
"\n";
append(out, rb_str_new_cstr(str));
for (i = 0; i < count; i++) {
record = &objspace->profile.records[i];
append(out, rb_sprintf("%5"PRIuSIZE" %4s/%c/%6s%c %13"PRIuSIZE" %15"PRIuSIZE
#if CALC_EXACT_MALLOC_SIZE
" %15"PRIuSIZE
#endif
" %9"PRIuSIZE" %17.12f %17.12f %17.12f %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
#if RGENGC_PROFILE
"%10"PRIuSIZE" %10"PRIuSIZE" %10"PRIuSIZE
#endif
#if GC_PROFILE_DETAIL_MEMORY
"%11ld %8ld %8ld"
#endif
"\n",
i+1,
gc_profile_dump_major_reason(record->flags, reason_str),
(record->flags & GPR_FLAG_HAVE_FINALIZE) ? 'F' : '.',
(record->flags & GPR_FLAG_NEWOBJ) ? "NEWOBJ" :
(record->flags & GPR_FLAG_MALLOC) ? "MALLOC" :
(record->flags & GPR_FLAG_METHOD) ? "METHOD" :
(record->flags & GPR_FLAG_CAPI) ? "CAPI__" : "??????",
(record->flags & GPR_FLAG_STRESS) ? '!' : ' ',
record->allocate_increase, record->allocate_limit,
#if CALC_EXACT_MALLOC_SIZE
record->allocated_size,
#endif
record->heap_use_pages,
record->gc_mark_time*1000,
record->gc_sweep_time*1000,
record->prepare_time*1000,
record->heap_live_objects,
record->heap_free_objects,
record->removing_objects,
record->empty_objects
#if RGENGC_PROFILE
,
record->old_objects,
record->remembered_normal_objects,
record->remembered_shady_objects
#endif
#if GC_PROFILE_DETAIL_MEMORY
,
record->maxrss / 1024,
record->minflt,
record->majflt
#endif
));
}
#endif
}
}
/*
* call-seq:
* GC::Profiler.result -> String
*
* Returns a profile data report such as:
*
* GC 1 invokes.
* Index Invoke Time(sec) Use Size(byte) Total Size(byte) Total Object GC time(ms)
* 1 0.012 159240 212940 10647 0.00000000000001530000
*/
static VALUE
gc_profile_result(VALUE _)
{
VALUE str = rb_str_buf_new(0);
gc_profile_dump_on(str, rb_str_buf_append);
return str;
}
/*
* call-seq:
* GC::Profiler.report
* GC::Profiler.report(io)
*
* Writes the GC::Profiler.result to <tt>$stdout</tt> or the given IO object.
*
*/
static VALUE
gc_profile_report(int argc, VALUE *argv, VALUE self)
{
VALUE out;
out = (!rb_check_arity(argc, 0, 1) ? rb_stdout : argv[0]);
gc_profile_dump_on(out, rb_io_write);
return Qnil;
}
/*
* call-seq:
* GC::Profiler.total_time -> float
*
* The total time used for garbage collection in seconds
*/
static VALUE
gc_profile_total_time(VALUE self)
{
double time = 0;
rb_objspace_t *objspace = &rb_objspace;
if (objspace->profile.run && objspace->profile.next_index > 0) {
size_t i;
size_t count = objspace->profile.next_index;
for (i = 0; i < count; i++) {
time += objspace->profile.records[i].gc_time;
}
}
return DBL2NUM(time);
}
/*
* call-seq:
* GC::Profiler.enabled? -> true or false
*
* The current status of GC profile mode.
*/
static VALUE
gc_profile_enable_get(VALUE self)
{
rb_objspace_t *objspace = &rb_objspace;
return RBOOL(objspace->profile.run);
}
/*
* call-seq:
* GC::Profiler.enable -> nil
*
* Starts the GC profiler.
*
*/
static VALUE
gc_profile_enable(VALUE _)
{
rb_objspace_t *objspace = &rb_objspace;
objspace->profile.run = TRUE;
objspace->profile.current_record = 0;
return Qnil;
}
/*
* call-seq:
* GC::Profiler.disable -> nil
*
* Stops the GC profiler.
*
*/
static VALUE
gc_profile_disable(VALUE _)
{
rb_objspace_t *objspace = &rb_objspace;
objspace->profile.run = FALSE;
objspace->profile.current_record = 0;
return Qnil;
}
/*
------------------------------ DEBUG ------------------------------
*/
static const char *
type_name(int type, VALUE obj)
{
switch (type) {
#define TYPE_NAME(t) case (t): return #t;
TYPE_NAME(T_NONE);
TYPE_NAME(T_OBJECT);
TYPE_NAME(T_CLASS);
TYPE_NAME(T_MODULE);
TYPE_NAME(T_FLOAT);
TYPE_NAME(T_STRING);
TYPE_NAME(T_REGEXP);
TYPE_NAME(T_ARRAY);
TYPE_NAME(T_HASH);
TYPE_NAME(T_STRUCT);
TYPE_NAME(T_BIGNUM);
TYPE_NAME(T_FILE);
TYPE_NAME(T_MATCH);
TYPE_NAME(T_COMPLEX);
TYPE_NAME(T_RATIONAL);
TYPE_NAME(T_NIL);
TYPE_NAME(T_TRUE);
TYPE_NAME(T_FALSE);
TYPE_NAME(T_SYMBOL);
TYPE_NAME(T_FIXNUM);
TYPE_NAME(T_UNDEF);
TYPE_NAME(T_IMEMO);
TYPE_NAME(T_ICLASS);
TYPE_NAME(T_MOVED);
TYPE_NAME(T_ZOMBIE);
case T_DATA:
if (obj && rb_objspace_data_type_name(obj)) {
return rb_objspace_data_type_name(obj);
}
return "T_DATA";
#undef TYPE_NAME
}
return "unknown";
}
static const char *
obj_type_name(VALUE obj)
{
return type_name(TYPE(obj), obj);
}
const char *
rb_method_type_name(rb_method_type_t type)
{
switch (type) {
case VM_METHOD_TYPE_ISEQ: return "iseq";
case VM_METHOD_TYPE_ATTRSET: return "attrest";
case VM_METHOD_TYPE_IVAR: return "ivar";
case VM_METHOD_TYPE_BMETHOD: return "bmethod";
case VM_METHOD_TYPE_ALIAS: return "alias";
case VM_METHOD_TYPE_REFINED: return "refined";
case VM_METHOD_TYPE_CFUNC: return "cfunc";
case VM_METHOD_TYPE_ZSUPER: return "zsuper";
case VM_METHOD_TYPE_MISSING: return "missing";
case VM_METHOD_TYPE_OPTIMIZED: return "optimized";
case VM_METHOD_TYPE_UNDEF: return "undef";
case VM_METHOD_TYPE_NOTIMPLEMENTED: return "notimplemented";
}
rb_bug("rb_method_type_name: unreachable (type: %d)", type);
}
static void
rb_raw_iseq_info(char *const buff, const size_t buff_size, const rb_iseq_t *iseq)
{
if (buff_size > 0 && ISEQ_BODY(iseq) && ISEQ_BODY(iseq)->location.label && !RB_TYPE_P(ISEQ_BODY(iseq)->location.pathobj, T_MOVED)) {
VALUE path = rb_iseq_path(iseq);
int n = ISEQ_BODY(iseq)->location.first_lineno;
snprintf(buff, buff_size, " %s@%s:%d",
RSTRING_PTR(ISEQ_BODY(iseq)->location.label),
RSTRING_PTR(path), n);
}
}
static int
str_len_no_raise(VALUE str)
{
long len = RSTRING_LEN(str);
if (len < 0) return 0;
if (len > INT_MAX) return INT_MAX;
return (int)len;
}
#define BUFF_ARGS buff + pos, buff_size - pos
#define APPEND_F(...) if ((pos += snprintf(BUFF_ARGS, "" __VA_ARGS__)) >= buff_size) goto end
#define APPEND_S(s) do { \
if ((pos + (int)rb_strlen_lit(s)) >= buff_size) { \
goto end; \
} \
else { \
memcpy(buff + pos, (s), rb_strlen_lit(s) + 1); \
} \
} while (0)
#define TF(c) ((c) != 0 ? "true" : "false")
#define C(c, s) ((c) != 0 ? (s) : " ")
static size_t
rb_raw_obj_info_common(char *const buff, const size_t buff_size, const VALUE obj)
{
size_t pos = 0;
if (SPECIAL_CONST_P(obj)) {
APPEND_F("%s", obj_type_name(obj));
if (FIXNUM_P(obj)) {
APPEND_F(" %ld", FIX2LONG(obj));
}
else if (SYMBOL_P(obj)) {
APPEND_F(" %s", rb_id2name(SYM2ID(obj)));
}
}
else {
const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
if (is_pointer_to_heap(&rb_objspace, (void *)obj)) {
APPEND_F("%p [%d%s%s%s%s%s%s] %s ",
(void *)obj, age,
C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"),
C(RVALUE_MARK_BITMAP(obj), "M"),
C(RVALUE_PIN_BITMAP(obj), "P"),
C(RVALUE_MARKING_BITMAP(obj), "R"),
C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"),
C(rb_objspace_garbage_object_p(obj), "G"),
obj_type_name(obj));
}
else {
/* fake */
APPEND_F("%p [%dXXXX] %s",
(void *)obj, age,
obj_type_name(obj));
}
if (internal_object_p(obj)) {
/* ignore */
}
else if (RBASIC(obj)->klass == 0) {
APPEND_S("(temporary internal)");
}
else if (RTEST(RBASIC(obj)->klass)) {
VALUE class_path = rb_class_path_cached(RBASIC(obj)->klass);
if (!NIL_P(class_path)) {
APPEND_F("(%s)", RSTRING_PTR(class_path));
}
}
#if GC_DEBUG
APPEND_F("@%s:%d", RANY(obj)->file, RANY(obj)->line);
#endif
}
end:
return pos;
}
static size_t
rb_raw_obj_info_buitin_type(char *const buff, const size_t buff_size, const VALUE obj, size_t pos)
{
if (LIKELY(pos < buff_size) && !SPECIAL_CONST_P(obj)) {
const enum ruby_value_type type = BUILTIN_TYPE(obj);
switch (type) {
case T_NODE:
UNEXPECTED_NODE(rb_raw_obj_info);
break;
case T_ARRAY:
if (ARY_SHARED_P(obj)) {
APPEND_S("shared -> ");
rb_raw_obj_info(BUFF_ARGS, ARY_SHARED_ROOT(obj));
}
else if (ARY_EMBED_P(obj)) {
APPEND_F("[%s%s] len: %ld (embed)",
C(ARY_EMBED_P(obj), "E"),
C(ARY_SHARED_P(obj), "S"),
RARRAY_LEN(obj));
}
else {
APPEND_F("[%s%s%s] len: %ld, capa:%ld ptr:%p",
C(ARY_EMBED_P(obj), "E"),
C(ARY_SHARED_P(obj), "S"),
C(RARRAY_TRANSIENT_P(obj), "T"),
RARRAY_LEN(obj),
ARY_EMBED_P(obj) ? -1L : RARRAY(obj)->as.heap.aux.capa,
(void *)RARRAY_CONST_PTR_TRANSIENT(obj));
}
break;
case T_STRING: {
if (STR_SHARED_P(obj)) {
APPEND_F(" [shared] len: %ld", RSTRING_LEN(obj));
}
else {
if (STR_EMBED_P(obj)) APPEND_S(" [embed]");
APPEND_F(" len: %ld, capa: %" PRIdSIZE, RSTRING_LEN(obj), rb_str_capacity(obj));
}
APPEND_F(" \"%.*s\"", str_len_no_raise(obj), RSTRING_PTR(obj));
break;
}
case T_SYMBOL: {
VALUE fstr = RSYMBOL(obj)->fstr;
ID id = RSYMBOL(obj)->id;
if (RB_TYPE_P(fstr, T_STRING)) {
APPEND_F(":%s id:%d", RSTRING_PTR(fstr), (unsigned int)id);
}
else {
APPEND_F("(%p) id:%d", (void *)fstr, (unsigned int)id);
}
break;
}
case T_MOVED: {
APPEND_F("-> %p", (void*)rb_gc_location(obj));
break;
}
case T_HASH: {
APPEND_F("[%c%c] %"PRIdSIZE,
RHASH_AR_TABLE_P(obj) ? 'A' : 'S',
RHASH_TRANSIENT_P(obj) ? 'T' : ' ',
RHASH_SIZE(obj));
break;
}
case T_CLASS:
case T_MODULE:
{
VALUE class_path = rb_class_path_cached(obj);
if (!NIL_P(class_path)) {
APPEND_F("%s", RSTRING_PTR(class_path));
}
else {
APPEND_S("(annon)");
}
break;
}
case T_ICLASS:
{
VALUE class_path = rb_class_path_cached(RBASIC_CLASS(obj));
if (!NIL_P(class_path)) {
APPEND_F("src:%s", RSTRING_PTR(class_path));
}
break;
}
case T_OBJECT:
{
uint32_t len = ROBJECT_NUMIV(obj);
if (RANY(obj)->as.basic.flags & ROBJECT_EMBED) {
APPEND_F("(embed) len:%d", len);
}
else {
VALUE *ptr = ROBJECT_IVPTR(obj);
APPEND_F("len:%d ptr:%p", len, (void *)ptr);
}
}
break;
case T_DATA: {
const struct rb_block *block;
const rb_iseq_t *iseq;
if (rb_obj_is_proc(obj) &&
(block = vm_proc_block(obj)) != NULL &&
(vm_block_type(block) == block_type_iseq) &&
(iseq = vm_block_iseq(block)) != NULL) {
rb_raw_iseq_info(BUFF_ARGS, iseq);
}
else if (rb_ractor_p(obj)) {
rb_ractor_t *r = (void *)DATA_PTR(obj);
if (r) {
APPEND_F("r:%d", r->pub.id);
}
}
else {
const char * const type_name = rb_objspace_data_type_name(obj);
if (type_name) {
APPEND_F("%s", type_name);
}
}
break;
}
case T_IMEMO: {
APPEND_F("<%s> ", rb_imemo_name(imemo_type(obj)));
switch (imemo_type(obj)) {
case imemo_ment:
{
const rb_method_entry_t *me = &RANY(obj)->as.imemo.ment;
APPEND_F(":%s (%s%s%s%s) type:%s alias:%d owner:%p defined_class:%p",
rb_id2name(me->called_id),
METHOD_ENTRY_VISI(me) == METHOD_VISI_PUBLIC ? "pub" :
METHOD_ENTRY_VISI(me) == METHOD_VISI_PRIVATE ? "pri" : "pro",
METHOD_ENTRY_COMPLEMENTED(me) ? ",cmp" : "",
METHOD_ENTRY_CACHED(me) ? ",cc" : "",
METHOD_ENTRY_INVALIDATED(me) ? ",inv" : "",
me->def ? rb_method_type_name(me->def->type) : "NULL",
me->def ? me->def->alias_count : -1,
(void *)me->owner, // obj_info(me->owner),
(void *)me->defined_class); //obj_info(me->defined_class)));
if (me->def) {
switch (me->def->type) {
case VM_METHOD_TYPE_ISEQ:
APPEND_S(" (iseq:");
rb_raw_obj_info(BUFF_ARGS, (VALUE)me->def->body.iseq.iseqptr);
APPEND_S(")");
break;
default:
break;
}
}
break;
}
case imemo_iseq: {
const rb_iseq_t *iseq = (const rb_iseq_t *)obj;
rb_raw_iseq_info(BUFF_ARGS, iseq);
break;
}
case imemo_callinfo:
{
const struct rb_callinfo *ci = (const struct rb_callinfo *)obj;
APPEND_F("(mid:%s, flag:%x argc:%d, kwarg:%s)",
rb_id2name(vm_ci_mid(ci)),
vm_ci_flag(ci),
vm_ci_argc(ci),
vm_ci_kwarg(ci) ? "available" : "NULL");
break;
}
case imemo_callcache:
{
const struct rb_callcache *cc = (const struct rb_callcache *)obj;
VALUE class_path = cc->klass ? rb_class_path_cached(cc->klass) : Qnil;
const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
APPEND_F("(klass:%s cme:%s%s (%p) call:%p",
NIL_P(class_path) ? (cc->klass ? "??" : "<NULL>") : RSTRING_PTR(class_path),
cme ? rb_id2name(cme->called_id) : "<NULL>",
cme ? (METHOD_ENTRY_INVALIDATED(cme) ? " [inv]" : "") : "",
(void *)cme,
(void *)vm_cc_call(cc));
break;
}
default:
break;
}
}
default:
break;
}
}
end:
return pos;
}
#undef TF
#undef C
const char *
rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj)
{
asan_unpoisoning_object(obj) {
size_t pos = rb_raw_obj_info_common(buff, buff_size, obj);
pos = rb_raw_obj_info_buitin_type(buff, buff_size, obj, pos);
if (pos >= buff_size) {} // truncated
}
return buff;
}
#undef APPEND_S
#undef APPEND_F
#undef BUFF_ARGS
#if RGENGC_OBJ_INFO
#define OBJ_INFO_BUFFERS_NUM 10
#define OBJ_INFO_BUFFERS_SIZE 0x100
static rb_atomic_t obj_info_buffers_index = 0;
static char obj_info_buffers[OBJ_INFO_BUFFERS_NUM][OBJ_INFO_BUFFERS_SIZE];
/* Increments *var atomically and resets *var to 0 when maxval is
* reached. Returns the wraparound old *var value (0...maxval). */
static rb_atomic_t
atomic_inc_wraparound(rb_atomic_t *var, const rb_atomic_t maxval)
{
rb_atomic_t oldval = RUBY_ATOMIC_FETCH_ADD(*var, 1);
if (UNLIKELY(oldval >= maxval - 1)) { // wraparound *var
const rb_atomic_t newval = oldval + 1;
RUBY_ATOMIC_CAS(*var, newval, newval % maxval);
oldval %= maxval;
}
return oldval;
}
static const char *
obj_info(VALUE obj)
{
rb_atomic_t index = atomic_inc_wraparound(&obj_info_buffers_index, OBJ_INFO_BUFFERS_NUM);
char *const buff = obj_info_buffers[index];
return rb_raw_obj_info(buff, OBJ_INFO_BUFFERS_SIZE, obj);
}
#else
static const char *
obj_info(VALUE obj)
{
return obj_type_name(obj);
}
#endif
MJIT_FUNC_EXPORTED const char *
rb_obj_info(VALUE obj)
{
return obj_info(obj);
}
void
rb_obj_info_dump(VALUE obj)
{
char buff[0x100];
fprintf(stderr, "rb_obj_info_dump: %s\n", rb_raw_obj_info(buff, 0x100, obj));
}
MJIT_FUNC_EXPORTED void
rb_obj_info_dump_loc(VALUE obj, const char *file, int line, const char *func)
{
char buff[0x100];
fprintf(stderr, "<OBJ_INFO:%s@%s:%d> %s\n", func, file, line, rb_raw_obj_info(buff, 0x100, obj));
}
#if GC_DEBUG
void
rb_gcdebug_print_obj_condition(VALUE obj)
{
rb_objspace_t *objspace = &rb_objspace;
fprintf(stderr, "created at: %s:%d\n", RANY(obj)->file, RANY(obj)->line);
if (BUILTIN_TYPE(obj) == T_MOVED) {
fprintf(stderr, "moved?: true\n");
}
else {
fprintf(stderr, "moved?: false\n");
}
if (is_pointer_to_heap(objspace, (void *)obj)) {
fprintf(stderr, "pointer to heap?: true\n");
}
else {
fprintf(stderr, "pointer to heap?: false\n");
return;
}
fprintf(stderr, "marked? : %s\n", MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj) ? "true" : "false");
fprintf(stderr, "pinned? : %s\n", MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj) ? "true" : "false");
fprintf(stderr, "age? : %d\n", RVALUE_AGE(obj));
fprintf(stderr, "old? : %s\n", RVALUE_OLD_P(obj) ? "true" : "false");
fprintf(stderr, "WB-protected?: %s\n", RVALUE_WB_UNPROTECTED(obj) ? "false" : "true");
fprintf(stderr, "remembered? : %s\n", RVALUE_REMEMBERED(obj) ? "true" : "false");
if (is_lazy_sweeping(objspace)) {
fprintf(stderr, "lazy sweeping?: true\n");
fprintf(stderr, "swept?: %s\n", is_swept_object(objspace, obj) ? "done" : "not yet");
}
else {
fprintf(stderr, "lazy sweeping?: false\n");
}
}
static VALUE
gcdebug_sentinel(RB_BLOCK_CALL_FUNC_ARGLIST(obj, name))
{
fprintf(stderr, "WARNING: object %s(%p) is inadvertently collected\n", (char *)name, (void *)obj);
return Qnil;
}
void
rb_gcdebug_sentinel(VALUE obj, const char *name)
{
rb_define_finalizer(obj, rb_proc_new(gcdebug_sentinel, (VALUE)name));
}
#endif /* GC_DEBUG */
#if GC_DEBUG_STRESS_TO_CLASS
/*
* call-seq:
* GC.add_stress_to_class(class[, ...])
*
* Raises NoMemoryError when allocating an instance of the given classes.
*
*/
static VALUE
rb_gcdebug_add_stress_to_class(int argc, VALUE *argv, VALUE self)
{
rb_objspace_t *objspace = &rb_objspace;
if (!stress_to_class) {
stress_to_class = rb_ary_hidden_new(argc);
}
rb_ary_cat(stress_to_class, argv, argc);
return self;
}
/*
* call-seq:
* GC.remove_stress_to_class(class[, ...])
*
* No longer raises NoMemoryError when allocating an instance of the
* given classes.
*
*/
static VALUE
rb_gcdebug_remove_stress_to_class(int argc, VALUE *argv, VALUE self)
{
rb_objspace_t *objspace = &rb_objspace;
int i;
if (stress_to_class) {
for (i = 0; i < argc; ++i) {
rb_ary_delete_same(stress_to_class, argv[i]);
}
if (RARRAY_LEN(stress_to_class) == 0) {
stress_to_class = 0;
}
}
return Qnil;
}
#endif
/*
* Document-module: ObjectSpace
*
* The ObjectSpace module contains a number of routines
* that interact with the garbage collection facility and allow you to
* traverse all living objects with an iterator.
*
* ObjectSpace also provides support for object finalizers, procs that will be
* called when a specific object is about to be destroyed by garbage
* collection. See the documentation for
* <code>ObjectSpace.define_finalizer</code> for important information on
* how to use this method correctly.
*
* a = "A"
* b = "B"
*
* ObjectSpace.define_finalizer(a, proc {|id| puts "Finalizer one on #{id}" })
* ObjectSpace.define_finalizer(b, proc {|id| puts "Finalizer two on #{id}" })
*
* a = nil
* b = nil
*
* _produces:_
*
* Finalizer two on 537763470
* Finalizer one on 537763480
*/
/*
* Document-class: ObjectSpace::WeakMap
*
* An ObjectSpace::WeakMap object holds references to
* any objects, but those objects can get garbage collected.
*
* This class is mostly used internally by WeakRef, please use
* +lib/weakref.rb+ for the public interface.
*/
/* Document-class: GC::Profiler
*
* The GC profiler provides access to information on GC runs including time,
* length and object space size.
*
* Example:
*
* GC::Profiler.enable
*
* require 'rdoc/rdoc'
*
* GC::Profiler.report
*
* GC::Profiler.disable
*
* See also GC.count, GC.malloc_allocated_size and GC.malloc_allocations
*/
#include "gc.rbinc"
void
Init_GC(void)
{
#undef rb_intern
VALUE rb_mObjSpace;
VALUE rb_mProfiler;
VALUE gc_constants;
rb_mGC = rb_define_module("GC");
gc_constants = rb_hash_new();
rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), RBOOL(GC_DEBUG));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(sizeof(RVALUE)));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_OBJ_LIMIT")), SIZET2NUM(HEAP_PAGE_OBJ_LIMIT));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(SIZE_POOL_COUNT));
rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(size_pool_slot_size(SIZE_POOL_COUNT - 1)));
OBJ_FREEZE(gc_constants);
/* internal constants */
rb_define_const(rb_mGC, "INTERNAL_CONSTANTS", gc_constants);
rb_mProfiler = rb_define_module_under(rb_mGC, "Profiler");
rb_define_singleton_method(rb_mProfiler, "enabled?", gc_profile_enable_get, 0);
rb_define_singleton_method(rb_mProfiler, "enable", gc_profile_enable, 0);
rb_define_singleton_method(rb_mProfiler, "raw_data", gc_profile_record_get, 0);
rb_define_singleton_method(rb_mProfiler, "disable", gc_profile_disable, 0);
rb_define_singleton_method(rb_mProfiler, "clear", gc_profile_clear, 0);
rb_define_singleton_method(rb_mProfiler, "result", gc_profile_result, 0);
rb_define_singleton_method(rb_mProfiler, "report", gc_profile_report, -1);
rb_define_singleton_method(rb_mProfiler, "total_time", gc_profile_total_time, 0);
rb_mObjSpace = rb_define_module("ObjectSpace");
rb_define_module_function(rb_mObjSpace, "each_object", os_each_obj, -1);
rb_define_module_function(rb_mObjSpace, "define_finalizer", define_final, -1);
rb_define_module_function(rb_mObjSpace, "undefine_finalizer", undefine_final, 1);
rb_define_module_function(rb_mObjSpace, "_id2ref", os_id2ref, 1);
rb_vm_register_special_exception(ruby_error_nomemory, rb_eNoMemError, "failed to allocate memory");
rb_define_method(rb_cBasicObject, "__id__", rb_obj_id, 0);
rb_define_method(rb_mKernel, "object_id", rb_obj_id, 0);
rb_define_module_function(rb_mObjSpace, "count_objects", count_objects, -1);
{
VALUE rb_cWeakMap = rb_define_class_under(rb_mObjSpace, "WeakMap", rb_cObject);
rb_define_alloc_func(rb_cWeakMap, wmap_allocate);
rb_define_method(rb_cWeakMap, "[]=", wmap_aset, 2);
rb_define_method(rb_cWeakMap, "[]", wmap_aref, 1);
rb_define_method(rb_cWeakMap, "include?", wmap_has_key, 1);
rb_define_method(rb_cWeakMap, "member?", wmap_has_key, 1);
rb_define_method(rb_cWeakMap, "key?", wmap_has_key, 1);
rb_define_method(rb_cWeakMap, "inspect", wmap_inspect, 0);
rb_define_method(rb_cWeakMap, "each", wmap_each, 0);
rb_define_method(rb_cWeakMap, "each_pair", wmap_each, 0);
rb_define_method(rb_cWeakMap, "each_key", wmap_each_key, 0);
rb_define_method(rb_cWeakMap, "each_value", wmap_each_value, 0);
rb_define_method(rb_cWeakMap, "keys", wmap_keys, 0);
rb_define_method(rb_cWeakMap, "values", wmap_values, 0);
rb_define_method(rb_cWeakMap, "size", wmap_size, 0);
rb_define_method(rb_cWeakMap, "length", wmap_size, 0);
rb_include_module(rb_cWeakMap, rb_mEnumerable);
}
/* internal methods */
rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency_m, 0);
rb_define_singleton_method(rb_mGC, "verify_transient_heap_internal_consistency", gc_verify_transient_heap_internal_consistency, 0);
#if MALLOC_ALLOCATED_SIZE
rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0);
rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0);
#endif
if (GC_COMPACTION_SUPPORTED) {
rb_define_singleton_method(rb_mGC, "compact", gc_compact, 0);
rb_define_singleton_method(rb_mGC, "auto_compact", gc_get_auto_compact, 0);
rb_define_singleton_method(rb_mGC, "auto_compact=", gc_set_auto_compact, 1);
rb_define_singleton_method(rb_mGC, "latest_compact_info", gc_compact_stats, 0);
}
else {
rb_define_singleton_method(rb_mGC, "compact", rb_f_notimplement, 0);
rb_define_singleton_method(rb_mGC, "auto_compact", rb_f_notimplement, 0);
rb_define_singleton_method(rb_mGC, "auto_compact=", rb_f_notimplement, 1);
rb_define_singleton_method(rb_mGC, "latest_compact_info", rb_f_notimplement, 0);
/* When !GC_COMPACTION_SUPPORTED, this method is not defined in gc.rb */
rb_define_singleton_method(rb_mGC, "verify_compaction_references", rb_f_notimplement, -1);
}
#if GC_DEBUG_STRESS_TO_CLASS
rb_define_singleton_method(rb_mGC, "add_stress_to_class", rb_gcdebug_add_stress_to_class, -1);
rb_define_singleton_method(rb_mGC, "remove_stress_to_class", rb_gcdebug_remove_stress_to_class, -1);
#endif
{
VALUE opts;
/* GC build options */
rb_define_const(rb_mGC, "OPTS", opts = rb_ary_new());
#define OPT(o) if (o) rb_ary_push(opts, rb_fstring_lit(#o))
OPT(GC_DEBUG);
OPT(USE_RGENGC);
OPT(RGENGC_DEBUG);
OPT(RGENGC_CHECK_MODE);
OPT(RGENGC_PROFILE);
OPT(RGENGC_ESTIMATE_OLDMALLOC);
OPT(GC_PROFILE_MORE_DETAIL);
OPT(GC_ENABLE_LAZY_SWEEP);
OPT(CALC_EXACT_MALLOC_SIZE);
OPT(MALLOC_ALLOCATED_SIZE);
OPT(MALLOC_ALLOCATED_SIZE_CHECK);
OPT(GC_PROFILE_DETAIL_MEMORY);
OPT(GC_COMPACTION_SUPPORTED);
#undef OPT
OBJ_FREEZE(opts);
}
}
#ifdef ruby_xmalloc
#undef ruby_xmalloc
#endif
#ifdef ruby_xmalloc2
#undef ruby_xmalloc2
#endif
#ifdef ruby_xcalloc
#undef ruby_xcalloc
#endif
#ifdef ruby_xrealloc
#undef ruby_xrealloc
#endif
#ifdef ruby_xrealloc2
#undef ruby_xrealloc2
#endif
void *
ruby_xmalloc(size_t size)
{
#if USE_GC_MALLOC_OBJ_INFO_DETAILS
ruby_malloc_info_file = __FILE__;
ruby_malloc_info_line = __LINE__;
#endif
return ruby_xmalloc_body(size);
}
void *
ruby_xmalloc2(size_t n, size_t size)
{
#if USE_GC_MALLOC_OBJ_INFO_DETAILS
ruby_malloc_info_file = __FILE__;
ruby_malloc_info_line = __LINE__;
#endif
return ruby_xmalloc2_body(n, size);
}
void *
ruby_xcalloc(size_t n, size_t size)
{
#if USE_GC_MALLOC_OBJ_INFO_DETAILS
ruby_malloc_info_file = __FILE__;
ruby_malloc_info_line = __LINE__;
#endif
return ruby_xcalloc_body(n, size);
}
void *
ruby_xrealloc(void *ptr, size_t new_size)
{
#if USE_GC_MALLOC_OBJ_INFO_DETAILS
ruby_malloc_info_file = __FILE__;
ruby_malloc_info_line = __LINE__;
#endif
return ruby_xrealloc_body(ptr, new_size);
}
void *
ruby_xrealloc2(void *ptr, size_t n, size_t new_size)
{
#if USE_GC_MALLOC_OBJ_INFO_DETAILS
ruby_malloc_info_file = __FILE__;
ruby_malloc_info_line = __LINE__;
#endif
return ruby_xrealloc2_body(ptr, n, new_size);
}
View git blame Copy permalink