[Backport #13150]

this patch contains r54158, r57410, r57631 and r57954.

	Prevent GC by volatile [Bug #13150]

	test/ruby/test_marshal.rb test_context_switch (load) and test_gc (dump)
	are failed on FreeBSD 10.3 and gcc7 (FreeBSD Ports Collection) 7.0.0
	20170115 (experimental); RB_GC_GUARD looks not worked well.

	* include/ruby/ruby.h (RB_GC_GUARD): prevent guarded pointer from
	  optimization by using as an input to inline asm.

	* ruby.h: remove comment

	* include/ruby/ruby.h (RB_GC_GUARD): remove comment unsupported by
	  Solaris AS.

	Hidden objects (klass == 0) are not visible to Ruby code invoked
	from other threads or signal handlers, so they can never be
	accessed from other contexts.  This makes it safe to call
	rb_gc_force_recycle on the object slot after releasing malloc
	memory.

	* marshal.c (rb_marshal_dump_limited): hide dump_arg and recycle when
	  done (rb_marshal_load_with_proc): hide load_arg and recycle when done
	  [ruby-core:79518]

	* marshal.c (rb_marshal_dump_limited): do not free dump_arg, which
	  may be dereferenced in check_dump_arg due to continuation, and
	  get rid of dangling pointers.

	* marshal.c (rb_marshal_load_with_proc): ditto for load_arg.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_2_3@59539 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,37 @@
+Wed Aug  9 17:35:47 2017  SHIBATA Hiroshi  <hsbt@ruby-lang.org>
+
+	this patch contains r54158, r57410, r57631 and r57954.
+
+	Prevent GC by volatile [Bug #13150]
+
+	test/ruby/test_marshal.rb test_context_switch (load) and test_gc (dump)
+	are failed on FreeBSD 10.3 and gcc7 (FreeBSD Ports Collection) 7.0.0
+	20170115 (experimental); RB_GC_GUARD looks not worked well.
+
+	* include/ruby/ruby.h (RB_GC_GUARD): prevent guarded pointer from
+	  optimization by using as an input to inline asm.
+
+	* ruby.h: remove comment
+
+	* include/ruby/ruby.h (RB_GC_GUARD): remove comment unsupported by
+	  Solaris AS.
+
+	Hidden objects (klass == 0) are not visible to Ruby code invoked
+	from other threads or signal handlers, so they can never be
+	accessed from other contexts.  This makes it safe to call
+	rb_gc_force_recycle on the object slot after releasing malloc
+	memory.
+
+	* marshal.c (rb_marshal_dump_limited): hide dump_arg and recycle when
+	  done (rb_marshal_load_with_proc): hide load_arg and recycle when done
+	  [ruby-core:79518]
+
+	* marshal.c (rb_marshal_dump_limited): do not free dump_arg, which
+	  may be dereferenced in check_dump_arg due to continuation, and
+	  get rid of dangling pointers.
+
+	* marshal.c (rb_marshal_load_with_proc): ditto for load_arg.
+
 Wed Aug  9 17:28:35 2017  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* eval.c (setup_exception): make unfrozen copy of special

include/ruby/ruby.h

@@ -551,27 +551,23 @@ static inline int rb_type(VALUE obj);
 	((type) == RUBY_T_FLOAT) ? RB_FLOAT_TYPE_P(obj) : \
 	(!RB_SPECIAL_CONST_P(obj) && RB_BUILTIN_TYPE(obj) == (type)))
 
-/* RB_GC_GUARD_PTR() is an intermediate macro, and has no effect by
- * itself.  don't use it directly */
 #ifdef __GNUC__
-#define RB_GC_GUARD_PTR(ptr) \
-    __extension__ ({volatile VALUE *rb_gc_guarded_ptr = (ptr); rb_gc_guarded_ptr;})
-#else
-#ifdef _MSC_VER
+#define RB_GC_GUARD(v) \
+    (*__extension__ ({ \
+	volatile VALUE *rb_gc_guarded_ptr = &(v); \
+	__asm__("" : : "m"(rb_gc_guarded_ptr)); \
+	rb_gc_guarded_ptr; \
+    }))
+#elif defined _MSC_VER
 #pragma optimize("", off)
 static inline volatile VALUE *rb_gc_guarded_ptr(volatile VALUE *ptr) {return ptr;}
 #pragma optimize("", on)
+#define RB_GC_GUARD(v) (*rb_gc_guarded_ptr(&(v)))
 #else
 volatile VALUE *rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val);
 #define HAVE_RB_GC_GUARDED_PTR_VAL 1
 #define RB_GC_GUARD(v) (*rb_gc_guarded_ptr_val(&(v),(v)))
 #endif
-#define RB_GC_GUARD_PTR(ptr) rb_gc_guarded_ptr(ptr)
-#endif
-
-#ifndef RB_GC_GUARD
-#define RB_GC_GUARD(v) (*RB_GC_GUARD_PTR(&(v)))
-#endif
 
 #ifdef __GNUC__
 #define RB_UNUSED_VAR(x) x __attribute__ ((unused))

marshal.c

@@ -1022,7 +1022,7 @@ VALUE
 rb_marshal_dump_limited(VALUE obj, VALUE port, int limit)
 {
     struct dump_arg *arg;
-    VALUE wrapper; /* used to avoid memory leak in case of exception */
+    volatile VALUE wrapper; /* used to avoid memory leak in case of exception */
 
     wrapper = TypedData_Make_Struct(rb_cData, struct dump_arg, &dump_arg_data, arg);
     arg->dest = 0;
@@ -1051,8 +1051,8 @@ rb_marshal_dump_limited(VALUE obj, VALUE port, int limit)
 	rb_io_write(arg->dest, arg->str);
 	rb_str_resize(arg->str, 0);
     }
-    clear_dump_arg(arg);
-    RB_GC_GUARD(wrapper);
+    free_dump_arg(arg);
+    rb_gc_force_recycle(wrapper);
 
     return port;
 }
@@ -2044,7 +2044,7 @@ rb_marshal_load_with_proc(VALUE port, VALUE proc)
 {
     int major, minor, infection = 0;
     VALUE v;
-    VALUE wrapper; /* used to avoid memory leak in case of exception */
+    volatile VALUE wrapper; /* used to avoid memory leak in case of exception */
     struct load_arg *arg;
 
     v = rb_check_string_type(port);
@@ -2090,8 +2090,8 @@ rb_marshal_load_with_proc(VALUE port, VALUE proc)
 
     if (!NIL_P(proc)) arg->proc = proc;
     v = r_object(arg);
-    clear_load_arg(arg);
-    RB_GC_GUARD(wrapper);
+    free_load_arg(arg);
+    rb_gc_force_recycle(wrapper);
 
     return v;
 }

test/ruby/test_marshal.rb

@@ -645,6 +645,9 @@ class TestMarshal < Test::Unit::TestCase
     c = Bug9523.new
     assert_raise_with_message(RuntimeError, /Marshal\.dump reentered at marshal_dump/) do
       Marshal.dump(c)
+      GC.start
+      1000.times {"x"*1000}
+      GC.start
       c.cc.call
     end
   end

version.h

@@ -1,6 +1,6 @@
 #define RUBY_VERSION "2.3.5"
 #define RUBY_RELEASE_DATE "2017-08-09"
-#define RUBY_PATCHLEVEL 349
+#define RUBY_PATCHLEVEL 350
 
 #define RUBY_RELEASE_YEAR 2017
 #define RUBY_RELEASE_MONTH 8