kotovalexarian-likes-github/ruby--ruby
1
0
Fork 0
mirror of https://github.com/ruby/ruby.git synced 2022-11-09 12:17:21 -05:00
Code Releases Activity

variable.c: fix multiple autoload with identical file (again)

Browse source 
We need to ensure autoload declarations pointing to the same
feature (aka "file") can wait on each other to avoid deadlock
situations.

So, reorganize autoload data structures to maintain a
feature => autoload_data_i mapping, and have module constant
tables point to the new autoload_const struct instead of
directly to autoload_data_i.  This allows multiple
autoload_const structs to refer to the SAME autoload_data_i
struct, and with it, the on-stack autoload_state.waitq.

The end result is different constants can share the same waitq
(tied to the feature name), and not deadlock each other during
loading.

Thanks to Eugene Kenny for the bug report and reproducible test case.

Reported-by: Eugene Kenny <elkenny@gmail.com>

* variable.c (autoload_featuremap): new global
  (struct autoload_const): new per-const struct
  (struct autoload_state): reference autoload_const instead of autoload_data_i
  (struct autoload_data_i): remove per-const
  (autoload_i_mark): delete from autoload_featuremap if unreferenced
  (autoload_c_mark): new dmark callback
  (autoload_c_free): new dfree callback
  (autoload_c_memsize): new memsize callback
  (autoload_const_type): new data type
  (get_autoload_data): set autoload_const as well
  (rb_autoload_str): use new data structures
  (autoload_delete): cleanup from autoload_featuremap
  (check_autoload_required): adjust for new internals
  (rb_autoloading_value): ditto
  (struct autoload_const_set_args): remove, redundant with autoload_const
  (const_tbl_update): adjust for new internals
  (autoload_const_set): ditto
  (autoload_require): ditto
  (autoload_reset): ditto
  (rb_autoload_load): ditto
  (rb_const_set): ditto
  (current_autoload_data): ditto
  (set_const_visibility): ditto
* test/ruby/test_autoload.rb (test_autoload_same_file): new test
  (test_no_leak): new test
  [ruby-core:86935] [Bug #14742]

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@63392 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
This commit is contained in:
normal 2018-05-10 08:54:26 +00:00
parent 5afa84b028
commit ec959fbb4b
2 changed files with 209 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
test/ruby/test_autoload.rb
View file

@ -311,6 +311,42 @@ p Foo::Bar
end end
end if Process.respond_to?(:fork) end if Process.respond_to?(:fork)
def test_autoload_same_file
Dir.mktmpdir('autoload') do |tmpdir|
File.write("#{tmpdir}/b.rb", "#{<<~'begin;'}\n#{<<~'end;'}")
begin;
module Foo; end
module Bar; end
end;
3.times do # timing-dependent, needs a few times to hit [Bug #14742]
assert_separately(%W[-I #{tmpdir}], "#{<<-'begin;'}\n#{<<-'end;'}")
begin;
autoload :Foo, 'b'
autoload :Bar, 'b'
t1 = Thread.new do Foo end
t2 = Thread.new do Bar end
t1.join
t2.join
bug = '[ruby-core:86935] [Bug #14742]'
assert_instance_of Module, t1.value, bug
assert_instance_of Module, t2.value, bug
end;
end
end
end
def test_no_leak
assert_no_memory_leak([], '', <<~'end;', 'many autoloads', timeout: 30)
200000.times do |i|
m = Module.new
m.instance_eval do
autoload :Foo, 'x'
autoload :Bar, i.to_s
end
end
end;
end
def add_autoload(path) def add_autoload(path)
(@autoload_paths ||= []) << path (@autoload_paths ||= []) << path
::Object.class_eval {autoload(:AutoloadTest, path)} ::Object.class_eval {autoload(:AutoloadTest, path)}

244
variable.c
View file

@ -25,6 +25,7 @@
static struct rb_id_table *rb_global_tbl; static struct rb_id_table *rb_global_tbl;
static ID autoload, classpath, tmp_classpath, classid; static ID autoload, classpath, tmp_classpath, classid;
static VALUE autoload_featuremap; /* feature => autoload_i */
static void check_before_mod_set(VALUE, ID, VALUE, const char *); static void check_before_mod_set(VALUE, ID, VALUE, const char *);
static void setup_const_entry(rb_const_entry_t *, VALUE, VALUE, rb_const_flag_t); static void setup_const_entry(rb_const_entry_t *, VALUE, VALUE, rb_const_flag_t);
@ -1842,31 +1843,53 @@ autoload_data(VALUE mod, ID id)
return (VALUE)val; return (VALUE)val;
} }
struct autoload_const {
struct list_node cnode; /* <=> autoload_data_i.constants */
VALUE mod;
VALUE ad; /* autoload_data_i */
VALUE value;
ID id;
int safe_level;
rb_const_flag_t flag;
};
/* always on stack, no need to mark */ /* always on stack, no need to mark */
struct autoload_state { struct autoload_state {
struct autoload_data_i *ele; struct autoload_const *ac;
VALUE mod;
VALUE result; VALUE result;
ID id;
VALUE thread; VALUE thread;
struct list_node waitq; struct list_node waitq;
}; };
struct autoload_data_i { struct autoload_data_i {
VALUE feature; VALUE feature;
int safe_level;
rb_const_flag_t flag;
VALUE value;
struct autoload_state *state; /* points to on-stack struct */ struct autoload_state *state; /* points to on-stack struct */
rb_serial_t fork_gen; rb_serial_t fork_gen;
struct list_head constants; /* <=> autoload_const.cnode */
}; };
static void static void
autoload_i_mark(void *ptr) autoload_i_mark(void *ptr)
{ {
struct autoload_data_i *p = ptr; struct autoload_data_i *p = ptr;
rb_gc_mark(p->feature); rb_gc_mark(p->feature);
rb_gc_mark(p->value);
/* allow GC to free us if no modules refer to this via autoload_const.ad */
if (list_empty(&p->constants)) {
rb_hash_delete(autoload_featuremap, p->feature);
}
}
static void
autoload_i_free(void *ptr)
{
struct autoload_data_i *p = ptr;
/* we may leak some memory at VM shutdown time, no big deal */
if (list_empty(&p->constants)) {
xfree(p);
}
} }
static size_t static size_t
@ -1877,20 +1900,53 @@ autoload_i_memsize(const void *ptr)
static const rb_data_type_t autoload_data_i_type = { static const rb_data_type_t autoload_data_i_type = {
"autoload_i", "autoload_i",
{autoload_i_mark, RUBY_TYPED_DEFAULT_FREE, autoload_i_memsize,}, {autoload_i_mark, autoload_i_free, autoload_i_memsize,},
0, 0, RUBY_TYPED_FREE_IMMEDIATELY
};
static void
autoload_c_mark(void *ptr)
{
struct autoload_const *ac = ptr;
rb_gc_mark(ac->mod);
rb_gc_mark(ac->ad);
rb_gc_mark(ac->value);
}
static void
autoload_c_free(void *ptr)
{
struct autoload_const *ac = ptr;
list_del(&ac->cnode);
xfree(ac);
}
static size_t
autoload_c_memsize(const void *ptr)
{
return sizeof(struct autoload_const);
}
static const rb_data_type_t autoload_const_type = {
"autoload_const",
{autoload_c_mark, autoload_c_free, autoload_c_memsize,},
0, 0, RUBY_TYPED_FREE_IMMEDIATELY 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
}; };
static struct autoload_data_i * static struct autoload_data_i *
get_autoload_data(VALUE av) get_autoload_data(VALUE acv, struct autoload_const **acp)
{ {
struct autoload_data_i *ele = rb_check_typeddata(av, &autoload_data_i_type); struct autoload_const *ac = rb_check_typeddata(acv, &autoload_const_type);
struct autoload_data_i *ele;
ele = rb_check_typeddata(ac->ad, &autoload_data_i_type);
/* do not reach across stack for ->state after forking: */ /* do not reach across stack for ->state after forking: */
if (ele && ele->state && ele->fork_gen != GET_VM()->fork_gen) { if (ele && ele->state && ele->fork_gen != GET_VM()->fork_gen) {
ele->state = 0; ele->state = 0;
ele->fork_gen = 0; ele->fork_gen = 0;
} }
if (acp) *acp = ac;
return ele; return ele;
} }
@ -1940,17 +1996,42 @@ rb_autoload_str(VALUE mod, ID id, VALUE file)
DATA_PTR(av) = tbl = st_init_numtable(); DATA_PTR(av) = tbl = st_init_numtable();
} }
ad = TypedData_Make_Struct(0, struct autoload_data_i, &autoload_data_i_type, ele);
if (OBJ_TAINTED(file)) { if (OBJ_TAINTED(file)) {
file = rb_str_dup(file); file = rb_str_dup(file);
FL_UNSET(file, FL_TAINT); FL_UNSET(file, FL_TAINT);
} }
ele->feature = rb_fstring(file); file = rb_fstring(file);
ele->safe_level = rb_safe_level(); if (!autoload_featuremap) {
ele->value = Qundef; autoload_featuremap = rb_hash_new_compare_by_id();
rb_obj_hide(autoload_featuremap);
rb_gc_register_mark_object(autoload_featuremap);
}
ad = rb_hash_aref(autoload_featuremap, file);
if (NIL_P(ad)) {
ad = TypedData_Make_Struct(0, struct autoload_data_i,
&autoload_data_i_type, ele);
ele->feature = file;
ele->state = 0; ele->state = 0;
ele->flag = CONST_PUBLIC; list_head_init(&ele->constants);
st_insert(tbl, (st_data_t)id, (st_data_t)ad); rb_hash_aset(autoload_featuremap, file, ad);
}
else {
ele = rb_check_typeddata(ad, &autoload_data_i_type);
}
{
VALUE acv;
struct autoload_const *ac;
acv = TypedData_Make_Struct(0, struct autoload_const,
&autoload_const_type, ac);
ac->mod = mod;
ac->id = id;
ac->value = Qundef;
ac->safe_level = rb_safe_level();
ac->flag = CONST_PUBLIC;
ac->ad = ad;
list_add_tail(&ele->constants, &ac->cnode);
st_insert(tbl, (st_data_t)id, (st_data_t)acv);
}
} }
static void static void
@ -1960,8 +2041,22 @@ autoload_delete(VALUE mod, ID id)
if (st_lookup(RCLASS_IV_TBL(mod), (st_data_t)autoload, &val)) { if (st_lookup(RCLASS_IV_TBL(mod), (st_data_t)autoload, &val)) {
struct st_table *tbl = check_autoload_table((VALUE)val); struct st_table *tbl = check_autoload_table((VALUE)val);
struct autoload_data_i *ele;
struct autoload_const *ac;
st_delete(tbl, &n, &load); st_delete(tbl, &n, &load);
ele = get_autoload_data((VALUE)load, &ac);
VM_ASSERT(ele);
if (ele) {
VM_ASSERT(!list_empty(&ele->constants));
}
/*
* we must delete here to avoid "already initialized" warnings
* with parallel autoload. Using list_del_init here so list_del
* works in autoload_c_free
*/
list_del_init(&ac->cnode);
if (tbl->num_entries == 0) { if (tbl->num_entries == 0) {
n = autoload; n = autoload;
@ -1987,12 +2082,13 @@ reset_safe(VALUE safe)
static VALUE static VALUE
check_autoload_required(VALUE mod, ID id, const char **loadingpath) check_autoload_required(VALUE mod, ID id, const char **loadingpath)
{ {
VALUE file, load; VALUE file;
VALUE load = autoload_data(mod, id);
struct autoload_data_i *ele; struct autoload_data_i *ele;
const char *loading; const char *loading;
int safe; int safe;
if (!(load = autoload_data(mod, id)) || !(ele = get_autoload_data(load))) { if (!load || !(ele = get_autoload_data(load, 0))) {
return 0; return 0;
} }
file = ele->feature; file = ele->feature;
@ -2027,19 +2123,21 @@ check_autoload_required(VALUE mod, ID id, const char **loadingpath)
MJIT_FUNC_EXPORTED int MJIT_FUNC_EXPORTED int
rb_autoloading_value(VALUE mod, ID id, VALUE* value, rb_const_flag_t *flag) rb_autoloading_value(VALUE mod, ID id, VALUE* value, rb_const_flag_t *flag)
{ {
VALUE load; VALUE load = autoload_data(mod, id);
struct autoload_data_i *ele; struct autoload_data_i *ele;
struct autoload_const *ac;
if (!(load = autoload_data(mod, id)) || !(ele = get_autoload_data(load))) { if (!load || !(ele = get_autoload_data(load, &ac))) {
return 0; return 0;
} }
if (ele->state && ele->state->thread == rb_thread_current()) { if (ele->state && ele->state->thread == rb_thread_current()) {
if (ele->value != Qundef) { if (ac->value != Qundef) {
if (value) { if (value) {
*value = ele->value; *value = ac->value;
} }
if (flag) { if (flag) {
*flag = ele->flag; *flag = ac->flag;
} }
return 1; return 1;
} }
@ -2058,23 +2156,16 @@ autoload_defined_p(VALUE mod, ID id)
return !rb_autoloading_value(mod, id, NULL, NULL); return !rb_autoloading_value(mod, id, NULL, NULL);
} }
struct autoload_const_set_args { static void const_tbl_update(struct autoload_const *);
VALUE mod;
ID id;
VALUE value;
rb_const_flag_t flag;
};
static void const_tbl_update(struct autoload_const_set_args *);
static VALUE static VALUE
autoload_const_set(VALUE arg) autoload_const_set(VALUE arg)
{ {
struct autoload_const_set_args* args = (struct autoload_const_set_args *)arg; struct autoload_const *ac = (struct autoload_const *)arg;
VALUE klass = args->mod; VALUE klass = ac->mod;
ID id = args->id; ID id = ac->id;
check_before_mod_set(klass, id, args->value, "constant"); check_before_mod_set(klass, id, ac->value, "constant");
const_tbl_update(args); const_tbl_update(ac);
return 0; /* ignored */ return 0; /* ignored */
} }
@ -2082,10 +2173,13 @@ static VALUE
autoload_require(VALUE arg) autoload_require(VALUE arg)
{ {
struct autoload_state *state = (struct autoload_state *)arg; struct autoload_state *state = (struct autoload_state *)arg;
struct autoload_const *ac = state->ac;
struct autoload_data_i *ele;
ele = rb_check_typeddata(ac->ad, &autoload_data_i_type);
/* this may release GVL and switch threads: */ /* this may release GVL and switch threads: */
state->result = rb_funcall(rb_vm_top_self(), rb_intern("require"), 1, state->result = rb_funcall(rb_vm_top_self(), rb_intern("require"), 1,
state->ele->feature); ele->feature);
return state->result; return state->result;
} }
@ -2095,27 +2189,28 @@ autoload_reset(VALUE arg)
{ {
struct autoload_state *state = (struct autoload_state *)arg; struct autoload_state *state = (struct autoload_state *)arg;
int need_wakeups = 0; int need_wakeups = 0;
struct autoload_const *ac = state->ac;
struct autoload_data_i *ele;
if (state->ele->state == state) { ele = rb_check_typeddata(ac->ad, &autoload_data_i_type);
if (ele->state == state) {
need_wakeups = 1; need_wakeups = 1;
state->ele->state = 0; ele->state = 0;
state->ele->fork_gen = 0; ele->fork_gen = 0;
} }
/* At the last, move a value defined in autoload to constant table */ /* At the last, move a value defined in autoload to constant table */
if (RTEST(state->result) && state->ele->value != Qundef) { if (RTEST(state->result)) {
int safe_backup; struct autoload_const *next;
struct autoload_const_set_args args; int safe_backup = rb_safe_level();
args.mod = state->mod; list_for_each_safe(&ele->constants, ac, next, cnode) {
args.id = state->id; if (ac->value != Qundef) {
args.value = state->ele->value; rb_ensure(autoload_const_set, (VALUE)ac,
args.flag = state->ele->flag;
safe_backup = rb_safe_level();
rb_set_safe_level_force(state->ele->safe_level);
rb_ensure(autoload_const_set, (VALUE)&args,
reset_safe, (VALUE)safe_backup); reset_safe, (VALUE)safe_backup);
} }
}
}
/* wakeup any waiters we had */ /* wakeup any waiters we had */
if (need_wakeups) { if (need_wakeups) {
@ -2172,6 +2267,7 @@ rb_autoload_load(VALUE mod, ID id)
VALUE load, result; VALUE load, result;
const char *loading = 0, *src; const char *loading = 0, *src;
struct autoload_data_i *ele; struct autoload_data_i *ele;
struct autoload_const *ac;
struct autoload_state state; struct autoload_state state;
if (!autoload_defined_p(mod, id)) return Qfalse; if (!autoload_defined_p(mod, id)) return Qfalse;
@ -2181,13 +2277,10 @@ rb_autoload_load(VALUE mod, ID id)
if (src && loading && strcmp(src, loading) == 0) return Qfalse; if (src && loading && strcmp(src, loading) == 0) return Qfalse;
/* set ele->state for a marker of autoloading thread */ /* set ele->state for a marker of autoloading thread */
if (!(ele = get_autoload_data(load))) { if (!(ele = get_autoload_data(load, &ac))) {
return Qfalse; return Qfalse;
} }
state.ac = ac;
state.ele = ele;
state.mod = mod;
state.id = id;
state.thread = rb_thread_current(); state.thread = rb_thread_current();
if (!ele->state) { if (!ele->state) {
ele->state = &state; ele->state = &state;
@ -2229,7 +2322,7 @@ rb_autoload_p(VALUE mod, ID id)
} }
load = check_autoload_required(mod, id, 0); load = check_autoload_required(mod, id, 0);
if (!load) return Qnil; if (!load) return Qnil;
return (ele = get_autoload_data(load)) ? ele->feature : Qnil; return (ele = get_autoload_data(load, 0)) ? ele->feature : Qnil;
} }
MJIT_FUNC_EXPORTED void MJIT_FUNC_EXPORTED void
@ -2610,12 +2703,12 @@ rb_const_set(VALUE klass, ID id, VALUE val)
setup_const_entry(ce, klass, val, CONST_PUBLIC); setup_const_entry(ce, klass, val, CONST_PUBLIC);
} }
else { else {
struct autoload_const_set_args args; struct autoload_const ac;
args.mod = klass; ac.mod = klass;
args.id = id; ac.id = id;
args.value = val; ac.value = val;
args.flag = CONST_PUBLIC; ac.flag = CONST_PUBLIC;
const_tbl_update(&args); const_tbl_update(&ac);
} }
/* /*
* Resolve and cache class name immediately to resolve ambiguity * Resolve and cache class name immediately to resolve ambiguity
@ -2647,12 +2740,12 @@ rb_const_set(VALUE klass, ID id, VALUE val)
} }
static struct autoload_data_i * static struct autoload_data_i *
current_autoload_data(VALUE mod, ID id) current_autoload_data(VALUE mod, ID id, struct autoload_const **acp)
{ {
struct autoload_data_i *ele; struct autoload_data_i *ele;
VALUE load = autoload_data(mod, id); VALUE load = autoload_data(mod, id);
if (!load) return 0; if (!load) return 0;
ele = get_autoload_data(load); ele = get_autoload_data(load, acp);
if (!ele) return 0; if (!ele) return 0;
/* for autoloading thread, keep the defined value to autoloading storage */ /* for autoloading thread, keep the defined value to autoloading storage */
if (ele->state && (ele->state->thread == rb_thread_current())) { if (ele->state && (ele->state->thread == rb_thread_current())) {
@ -2662,25 +2755,25 @@ current_autoload_data(VALUE mod, ID id)
} }
static void static void
const_tbl_update(struct autoload_const_set_args *args) const_tbl_update(struct autoload_const *ac)
{ {
VALUE value; VALUE value;
VALUE klass = args->mod; VALUE klass = ac->mod;
VALUE val = args->value; VALUE val = ac->value;
ID id = args->id; ID id = ac->id;
struct rb_id_table *tbl = RCLASS_CONST_TBL(klass); struct rb_id_table *tbl = RCLASS_CONST_TBL(klass);
rb_const_flag_t visibility = args->flag; rb_const_flag_t visibility = ac->flag;
rb_const_entry_t *ce; rb_const_entry_t *ce;
if (rb_id_table_lookup(tbl, id, &value)) { if (rb_id_table_lookup(tbl, id, &value)) {
ce = (rb_const_entry_t *)value; ce = (rb_const_entry_t *)value;
if (ce->value == Qundef) { if (ce->value == Qundef) {
struct autoload_data_i *ele = current_autoload_data(klass, id); struct autoload_data_i *ele = current_autoload_data(klass, id, &ac);
if (ele) { if (ele) {
rb_clear_constant_cache(); rb_clear_constant_cache();
ele->value = val; /* autoload_i is non-WB-protected */ ac->value = val; /* autoload_i is non-WB-protected */
return; return;
} }
/* otherwise, allow to override */ /* otherwise, allow to override */
@ -2753,6 +2846,7 @@ set_const_visibility(VALUE mod, int argc, const VALUE *argv,
} }
for (i = 0; i < argc; i++) { for (i = 0; i < argc; i++) {
struct autoload_const *ac;
VALUE val = argv[i]; VALUE val = argv[i];
id = rb_check_id(&val); id = rb_check_id(&val);
if (!id) { if (!id) {
@ -2767,10 +2861,12 @@ set_const_visibility(VALUE mod, int argc, const VALUE *argv,
ce->flag &= ~mask; ce->flag &= ~mask;
ce->flag |= flag; ce->flag |= flag;
if (ce->value == Qundef) { if (ce->value == Qundef) {
struct autoload_data_i *ele = current_autoload_data(mod, id); struct autoload_data_i *ele;
ele = current_autoload_data(mod, id, &ac);
if (ele) { if (ele) {
ele->flag &= ~mask; ac->flag &= ~mask;
ele->flag |= flag; ac->flag |= flag;
} }
} }
} }