* vm.c: support variable VM/Machine stack sizes.

Specified by the following environment variaables:
  - RUBY_THREAD_VM_STACK_SIZE: vm stack size used at thread creation.
  default: 128KB (32bit CPU) or 256KB (64bit CPU).
  - RUBY_THREAD_MACHINE_STACK_SIZE: machine stack size used at thread
  creation. default: 512KB or 1024KB.
  - RUBY_FIBER_VM_STACK_SIZE: vm stack size used at fiber creation.
  default: 64KB or 128KB.
  - RUBY_FIBER_MACHINE_STACK_SIZE: machine stack size used at fiber
  creation. default: 256KB or 256KB.
  This values are specified at launched timing. You can not change
  these values at running time.
  Environ variables are only *hints* because:
  - They are aligned to 4KB.
  - They have minimum values (depend on OSs).
  - Machine stack settings are ignored by some OSs.
  Default values especially fiber stack sizes are increased.
  This change affect Fiber's behavior:
  (1) You can run more complex program on a Fiber.
  (2) You can not make many (thousands) Fibers because of
  lack of address space (on 32bit CPU).
  If (2) bothers you,
  (a) Use 64bit CPU with big memory, or
  (b) Specify RUBY_FIBER_(VM|MACHINE)_STACK_SIZE correctly.
  You need to choose correct stack size carefully. These values
  are completely rely on systems (OS/compiler and so on).
* vm_core.h (rb_vm_t::default_params): add to record above settings.
* vm.c (RubyVM::DEFAULT_PARAMS): add new constant to see
  above setting.
* thread_pthread.c: support RUBY_THREAD_MACHINE_STACK_SIZE.
* cont.c: support RUBY_FIBER_(VM|MACHINE)_STACK_SIZE.
* test/ruby/test_fiber.rb: add tests for above.
* test/ruby/test_thread.rb: ditto.



git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@38478 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

ChangeLog

@@ -1,3 +1,45 @@
+Thu Dec 20 06:59:52 2012  Koichi Sasada  <ko1@atdot.net>
+
+	* vm.c: support variable VM/Machine stack sizes.
+	  Specified by the following environment variaables:
+	    - RUBY_THREAD_VM_STACK_SIZE: vm stack size used at thread creation.
+	      default: 128KB (32bit CPU) or 256KB (64bit CPU).
+	    - RUBY_THREAD_MACHINE_STACK_SIZE: machine stack size used at thread
+	      creation. default: 512KB or 1024KB.
+	    - RUBY_FIBER_VM_STACK_SIZE: vm stack size used at fiber creation.
+	      default: 64KB or 128KB.
+	    - RUBY_FIBER_MACHINE_STACK_SIZE: machine stack size used at fiber
+	      creation. default: 256KB or 256KB.
+	  This values are specified at launched timing. You can not change
+	  these values at running time.
+	  Environ variables are only *hints* because:
+	    - They are aligned to 4KB.
+	    - They have minimum values (depend on OSs).
+	    - Machine stack settings are ignored by some OSs.
+	  Default values especially fiber stack sizes are increased.
+	  This change affect Fiber's behavior:
+	    (1) You can run more complex program on a Fiber.
+	    (2) You can not make many (thousands) Fibers because of
+	        lack of address space (on 32bit CPU).
+	  If (2) bothers you, 
+            (a) Use 64bit CPU with big memory, or
+	    (b) Specify RUBY_FIBER_(VM|MACHINE)_STACK_SIZE correctly.
+	  You need to choose correct stack size carefully. These values
+	  are completely rely on systems (OS/compiler and so on).
+
+	* vm_core.h (rb_vm_t::default_params): add to record above settings.
+
+	* vm.c (RubyVM::DEFAULT_PARAMS): add new constant to see
+	  above setting.
+
+	* thread_pthread.c: support RUBY_THREAD_MACHINE_STACK_SIZE.
+
+	* cont.c: support RUBY_FIBER_(VM|MACHINE)_STACK_SIZE.
+
+	* test/ruby/test_fiber.rb: add tests for above.
+
+	* test/ruby/test_thread.rb: ditto.
+
 Thu Dec 20 06:25:44 2012  Koichi Sasada  <ko1@atdot.net>
 
 	* test/ruby/test_fiber.rb: remove a strange single quote character.

cont.c

@@ -47,12 +47,6 @@
 #define RB_PAGE_SIZE (pagesize)
 #define RB_PAGE_MASK (~(RB_PAGE_SIZE - 1))
 static long pagesize;
-
- #if SIZEOF_VOIDP==8
-  #define FIBER_MACHINE_STACK_ALLOCATION_SIZE  (0x20000)
- #else
-  #define FIBER_MACHINE_STACK_ALLOCATION_SIZE  (0x10000)
- #endif
 #endif /*FIBER_USE_NATIVE*/
 
 #define CAPTURE_JUST_VALID_VM_STACK 1
@@ -631,7 +625,7 @@ fiber_setcontext(rb_fiber_t *newfib, rb_fiber_t *oldfib)
     rb_thread_t *th = GET_THREAD(), *sth = &newfib->cont.saved_thread;
 
     if (newfib->status != RUNNING) {
-	fiber_initialize_machine_stack_context(newfib, FIBER_MACHINE_STACK_ALLOCATION_SIZE);
+	fiber_initialize_machine_stack_context(newfib, th->vm->default_params.fiber_machine_stack_size);
     }
 
     /* restore thread context */
@@ -1002,8 +996,6 @@ rb_cont_call(int argc, VALUE *argv, VALUE contval)
  *
  */
 
-#define FIBER_VM_STACK_SIZE (4 * 1024)
-
 static const rb_data_type_t fiber_data_type = {
     "fiber",
     {fiber_mark, fiber_free, fiber_memsize,},
@@ -1054,7 +1046,7 @@ fiber_init(VALUE fibval, VALUE proc)
 
     fiber_link_join(fib);
 
-    th->stack_size = FIBER_VM_STACK_SIZE;
+    th->stack_size = th->vm->default_params.fiber_vm_stack_size / sizeof(VALUE);
     th->stack = ALLOC_N(VALUE, th->stack_size);
 
     th->cfp = (void *)(th->stack + th->stack_size);

test/ruby/test_fiber.rb

@@ -4,6 +4,7 @@ require 'continuation'
 require_relative './envutil'
 
 class TestFiber < Test::Unit::TestCase
+if false
   def test_normal
     f = Fiber.current
     assert_equal(:ok2,
@@ -280,3 +281,44 @@ class TestFiber < Test::Unit::TestCase
   end
 end
 
+  def invoke_rec script, vm_stack_size, machine_stack_size, use_length = true
+    env = {}
+    env['RUBY_FIBER_VM_STACK_SIZE'] = vm_stack_size.to_s if vm_stack_size
+    env['RUBY_FIBER_MACHINE_STACK_SIZE'] = machine_stack_size.to_s if machine_stack_size
+    out, = EnvUtil.invoke_ruby([env, '-e', script], '', true, true)
+    use_length ? out.length : out
+  end
+
+  def test_stack_size
+    h_default = eval(invoke_rec('p RubyVM::DEFAULT_PARAMS', nil, nil, false))
+    h_0 = eval(invoke_rec('p RubyVM::DEFAULT_PARAMS', 0, 0, false))
+    h_large = eval(invoke_rec('p RubyVM::DEFAULT_PARAMS', 1024 * 1024 * 10, 1024 * 1024 * 10, false))
+
+    assert(h_default[:fiber_vm_stack_size] > h_0[:fiber_vm_stack_size])
+    assert(h_default[:fiber_vm_stack_size] < h_large[:fiber_vm_stack_size])
+    assert(h_default[:fiber_machine_stack_size] >= h_0[:fiber_machine_stack_size])
+    assert(h_default[:fiber_machine_stack_size] <= h_large[:fiber_machine_stack_size])
+
+    # check VM machine stack size
+    script = 'def rec; print "."; rec; end; Fiber.new{rec}.resume'
+    size_default = invoke_rec script, nil, nil
+    assert(size_default > 0, size_default.to_s)
+    size_0 = invoke_rec script, 0, nil
+    assert(size_default > size_0, [size_default, size_0].inspect)
+    size_large = invoke_rec script, 1024 * 1024 * 10, nil
+    assert(size_default < size_large, [size_default, size_large].inspect)
+
+    return if /mswin|mingw/ =~ RUBY_PLATFORM
+
+    # check machine stack size
+    # Note that machine stack size may not change size (depend on OSs)
+    script = 'def rec; print "."; 1.times{1.times{1.times{rec}}}; end; Fiber.new{rec}.resume'
+    vm_stack_size = 1024 * 1024
+    size_default = invoke_rec script, vm_stack_size, nil
+    size_0 = invoke_rec script, vm_stack_size, 0
+    assert(size_default >= size_0, [size_default, size_0].inspect)
+    size_large = invoke_rec script, vm_stack_size, 1024 * 1024 * 10
+    assert(size_default <= size_large, [size_default, size_large].inspect)
+  end
+end
+

test/ruby/test_thread.rb

@@ -870,4 +870,44 @@ Thread.new(Thread.current) {|mth|
       th.kill if th
     end
   end
+
+  def invoke_rec script, vm_stack_size, machine_stack_size, use_length = true
+    env = {}
+    env['RUBY_THREAD_VM_STACK_SIZE'] = vm_stack_size.to_s if vm_stack_size
+    env['RUBY_THREAD_MACHINE_STACK_SIZE'] = machine_stack_size.to_s if machine_stack_size
+    out, = EnvUtil.invoke_ruby([env, '-e', script], '', true, true)
+    use_length ? out.length : out
+  end
+
+  def test_stack_size
+    h_default = eval(invoke_rec('p RubyVM::DEFAULT_PARAMS', nil, nil, false))
+    h_0 = eval(invoke_rec('p RubyVM::DEFAULT_PARAMS', 0, 0, false))
+    h_large = eval(invoke_rec('p RubyVM::DEFAULT_PARAMS', 1024 * 1024 * 10, 1024 * 1024 * 10, false))
+
+    assert(h_default[:thread_vm_stack_size] > h_0[:thread_vm_stack_size])
+    assert(h_default[:thread_vm_stack_size] < h_large[:thread_vm_stack_size])
+    assert(h_default[:thread_machine_stack_size] >= h_0[:thread_machine_stack_size])
+    assert(h_default[:thread_machine_stack_size] <= h_large[:thread_machine_stack_size])
+
+    # check VM machine stack size
+    script = 'def rec; print "."; rec; end; rec'
+    size_default = invoke_rec script, nil, nil
+    assert(size_default > 0, size_default.to_s)
+    size_0 = invoke_rec script, 0, nil
+    assert(size_default > size_0, [size_default, size_0].inspect)
+    size_large = invoke_rec script, 1024 * 1024 * 10, nil
+    assert(size_default < size_large, [size_default, size_large].inspect)
+
+    return if /mswin|mingw/ =~ RUBY_PLATFORM
+
+    # check machine stack size
+    # Note that machine stack size may not change size (depend on OSs)
+    script = 'def rec; print "."; 1.times{1.times{1.times{rec}}}; end; Thread.new{rec}.join'
+    vm_stack_size = 1024 * 1024
+    size_default = invoke_rec script, vm_stack_size, nil
+    size_0 = invoke_rec script, vm_stack_size, 0
+    assert(size_default >= size_0, [size_default, size_0].inspect)
+    size_large = invoke_rec script, vm_stack_size, 1024 * 1024 * 10
+    assert(size_default <= size_large, [size_default, size_large].inspect)
+  end
 end

thread_pthread.c

@@ -584,26 +584,6 @@ static struct {
 #endif
 } native_main_thread;
 
-
-enum {
-#ifdef __SYMBIAN32__
-    RUBY_STACK_MIN_LIMIT = 64 * 1024,  /* 64KB: Let's be slightly more frugal on mobile platform */
-#else
-    RUBY_STACK_MIN_LIMIT = 512 * 1024, /* 512KB */
-#endif
-    RUBY_STACK_SPACE_LIMIT = 1024 * 1024,
-    RUBY_STACK_SPACE_RATIO = 5
-};
-#ifdef PTHREAD_STACK_MIN
-#define RUBY_STACK_MIN ((RUBY_STACK_MIN_LIMIT < PTHREAD_STACK_MIN) ? \
-			PTHREAD_STACK_MIN * 2 : RUBY_STACK_MIN_LIMIT)
-#else
-#define RUBY_STACK_MIN (RUBY_STACK_MIN_LIMIT)
-#endif
-#define RUBY_STACK_MIN_SPACE RUBY_STACK_MIN/RUBY_STACK_SPACE_RATIO
-#define RUBY_STACK_SPACE ((RUBY_STACK_MIN_SPACE > RUBY_STACK_SPACE_LIMIT) ? \
-			  RUBY_STACK_SPACE_LIMIT : RUBY_STACK_MIN_SPACE)
-
 #ifdef STACK_END_ADDRESS
 extern void *STACK_END_ADDRESS;
 #endif
@@ -830,6 +810,23 @@ use_cached_thread(rb_thread_t *th)
     return result;
 }
 
+enum {
+    RUBY_STACK_SPACE_LIMIT = 1024 * 1024, /* 1024KB */
+    RUBY_STACK_SPACE_RATIO = 5
+};
+
+static size_t
+space_size(size_t stack_size)
+{
+    size_t space_size = stack_size / RUBY_STACK_SPACE_RATIO;
+    if (space_size > RUBY_STACK_SPACE_LIMIT) {
+	return RUBY_STACK_SPACE_LIMIT;
+    }
+    else {
+	return space_size;
+    }
+}
+
 static int
 native_thread_create(rb_thread_t *th)
 {
@@ -840,8 +837,8 @@ native_thread_create(rb_thread_t *th)
     }
     else {
 	pthread_attr_t attr;
-	const size_t stack_size = RUBY_STACK_MIN;
-	const size_t space = RUBY_STACK_SPACE;
+	const size_t stack_size = th->vm->default_params.thread_machine_stack_size;
+	const size_t space = space_size(stack_size);
 
         th->machine_stack_maxsize = stack_size - space;
 #ifdef __ia64

vm.c

@@ -1613,6 +1613,82 @@ static const rb_data_type_t vm_data_type = {
     {rb_vm_mark, vm_free, vm_memsize,},
 };
 
+
+static VALUE
+vm_default_params(void)
+{
+    rb_vm_t *vm = GET_VM();
+    VALUE result = rb_hash_new();
+#define SET(name) rb_hash_aset(result, ID2SYM(rb_intern(#name)), SIZET2NUM(vm->default_params.name));
+    SET(thread_vm_stack_size);
+    SET(thread_machine_stack_size);
+    SET(fiber_vm_stack_size);
+    SET(fiber_machine_stack_size);
+#undef SET
+    rb_obj_freeze(result);
+    return result;
+}
+
+static size_t
+get_param(const char *name, size_t default_value, size_t min_value)
+{
+    const char *envval;
+    size_t result = default_value;
+    if ((envval = getenv(name)) != 0) {
+	long val = atol(envval);
+	if (val < (long)min_value) {
+	    val = (long)min_value;
+	}
+	result = (size_t)(((val -1 + RUBY_VM_SIZE_ALIGN) / RUBY_VM_SIZE_ALIGN) * RUBY_VM_SIZE_ALIGN);
+    }
+    if (0) fprintf(stderr, "%s: %d\n", name, result); /* debug print */
+
+    return result;
+}
+
+static void
+check_machine_stack_size(size_t *sizep)
+{
+    size_t size = *sizep;
+#ifdef __SYMBIAN32__
+    *sizep = 64 * 1024; /* 64KB: Let's be slightly more frugal on mobile platform */
+#endif
+
+#ifdef PTHREAD_STACK_MIN
+    if (size < PTHREAD_STACK_MIN) {
+	*sizep = PTHREAD_STACK_MIN * 2;
+    }
+#endif
+}
+
+static void
+vm_default_params_setup(rb_vm_t *vm)
+{
+    vm->default_params.thread_vm_stack_size =
+      get_param("RUBY_THREAD_VM_STACK_SIZE",
+		RUBY_VM_THREAD_VM_STACK_SIZE,
+		RUBY_VM_THREAD_VM_STACK_SIZE_MIN);
+
+    vm->default_params.thread_machine_stack_size =
+      get_param("RUBY_THREAD_MACHINE_STACK_SIZE",
+		RUBY_VM_THREAD_MACHINE_STACK_SIZE,
+		RUBY_VM_THREAD_MACHINE_STACK_SIZE_MIN);
+
+    vm->default_params.fiber_vm_stack_size =
+      get_param("RUBY_FIBER_VM_STACK_SIZE",
+		RUBY_VM_FIBER_VM_STACK_SIZE,
+		RUBY_VM_FIBER_VM_STACK_SIZE_MIN);
+
+    vm->default_params.fiber_machine_stack_size =
+      get_param("RUBY_FIBER_MACHINE_STACK_SIZE",
+		RUBY_VM_FIBER_MACHINE_STACK_SIZE,
+		RUBY_VM_FIBER_MACHINE_STACK_SIZE_MIN);
+
+    /* environment dependent check */
+    check_machine_stack_size(&vm->default_params.thread_machine_stack_size);
+    check_machine_stack_size(&vm->default_params.fiber_machine_stack_size);
+}
+
 static void
 vm_init2(rb_vm_t *vm)
 {
@@ -1620,6 +1696,8 @@ vm_init2(rb_vm_t *vm)
     vm->src_encoding_index = -1;
     vm->at_exit.basic.flags = (T_ARRAY | RARRAY_EMBED_FLAG) & ~RARRAY_EMBED_LEN_MASK; /* len set 0 */
     vm->at_exit.basic.klass = 0;
+
+    vm_default_params_setup(vm);
 }
 
 /* Thread */
@@ -1635,6 +1713,7 @@ static VALUE *
 thread_recycle_stack(size_t size)
 {
     if (thread_recycle_stack_count) {
+	/* TODO: check stack size if stack sizes are variable */
 	return thread_recycle_stack_slot[--thread_recycle_stack_count];
     }
     else {
@@ -1839,7 +1918,10 @@ th_init(rb_thread_t *th, VALUE self)
     /* altstack of main thread is reallocated in another place */
     th->altstack = malloc(rb_sigaltstack_size());
 #endif
-    th->stack_size = RUBY_VM_THREAD_STACK_SIZE;
+    /* th->stack_size is word number.
+     * th->vm->default_params.thread_vm_stack_size is byte size.
+     */
+    th->stack_size = th->vm->default_params.thread_vm_stack_size / sizeof(VALUE);
     th->stack = thread_recycle_stack(th->stack_size);
 
     th->cfp = (void *)(th->stack + th->stack_size);
@@ -1864,9 +1946,9 @@ ruby_thread_init(VALUE self)
     rb_vm_t *vm = GET_THREAD()->vm;
     GetThreadPtr(self, th);
 
+    th->vm = vm;
     th_init(th, self);
     rb_iv_set(self, "locals", rb_hash_new());
-    th->vm = vm;
 
     th->top_wrapper = 0;
     th->top_self = rb_vm_top_self();
@@ -2189,6 +2271,14 @@ Init_VM(void)
     /* ::RubyVM::INSTRUCTION_NAMES */
     rb_define_const(rb_cRubyVM, "INSTRUCTION_NAMES", rb_insns_name_array());
 
+    /* ::RubyVM::DEFAULT_PARAMS
+     * This constant variable shows VM's default parameters.
+     * Note that changing these values does not affect VM exection.
+     * Specification is not stable and you should not depend on this value.
+     * Of course, this constant is MRI specific.
+     */
+    rb_define_const(rb_cRubyVM, "DEFAULT_PARAMS", vm_default_params());
+
     /* debug functions ::RubyVM::SDR(), ::RubyVM::NSDR() */
 #if VMDEBUG
     rb_define_singleton_method(rb_cRubyVM, "SDR", sdr, 0);
@@ -2266,7 +2356,6 @@ Init_BareVM(void)
 	exit(EXIT_FAILURE);
     }
     MEMZERO(th, rb_thread_t, 1);
-
     rb_thread_set_current_raw(th);
 
     vm_init2(vm);
@@ -2276,8 +2365,8 @@ Init_BareVM(void)
     ruby_current_vm = vm;
 
     Init_native_thread();
-    th_init(th, 0);
     th->vm = vm;
+    th_init(th, 0);
     ruby_thread_init_stack(th);
 }
 

vm_core.h

@@ -393,8 +393,30 @@ typedef struct rb_vm_struct {
     struct RArray at_exit;
 
     VALUE *defined_strings;
+
+    /* params */
+    struct { /* size in byte */
+	size_t thread_vm_stack_size;
+	size_t thread_machine_stack_size;
+	size_t fiber_vm_stack_size;
+	size_t fiber_machine_stack_size;
+    } default_params;
 } rb_vm_t;
 
+/* default values */
+
+#define RUBY_VM_SIZE_ALIGN 4096
+
+#define RUBY_VM_THREAD_VM_STACK_SIZE          (  32 * 1024 * sizeof(VALUE)) /*  128 KB or  256 KB */
+#define RUBY_VM_THREAD_VM_STACK_SIZE_MIN      (   2 * 1024 * sizeof(VALUE)) /*    8 KB or   16 KB */
+#define RUBY_VM_THREAD_MACHINE_STACK_SIZE     ( 128 * 1024 * sizeof(VALUE)) /*  512 KB or 1024 KB */
+#define RUBY_VM_THREAD_MACHINE_STACK_SIZE_MIN (  16 * 1024 * sizeof(VALUE)) /*   64 KB or  128 KB */
+
+#define RUBY_VM_FIBER_VM_STACK_SIZE           (  16 * 1024 * sizeof(VALUE)) /*   64 KB or  128 KB */
+#define RUBY_VM_FIBER_VM_STACK_SIZE_MIN       (   2 * 1024 * sizeof(VALUE)) /*    8 KB or   16 KB */
+#define RUBY_VM_FIBER_MACHINE_STACK_SIZE      (  64 * 1024 * sizeof(VALUE)) /*  256 KB or  512 KB */
+#define RUBY_VM_FIBER_MACHINE_STACK_SIZE_MIN  (  16 * 1024 * sizeof(VALUE)) /*   64 KB or  128 KB */
+
 #ifndef VM_DEBUG_BP_CHECK
 #define VM_DEBUG_BP_CHECK 1
 #endif
@@ -469,7 +491,7 @@ typedef struct rb_thread_struct {
 
     /* execution information */
     VALUE *stack;		/* must free, must mark */
-    unsigned long stack_size;
+    size_t stack_size;          /* size in word (byte size / sizeof(VALUE)) */
     rb_control_frame_t *cfp;
     int safe_level;
     int raised_flag;
@@ -620,9 +642,6 @@ RUBY_EXTERN VALUE rb_mRubyVMFrozenCore;
 #pragma GCC visibility pop
 #endif
 
-/* each thread has this size stack : 128KB */
-#define RUBY_VM_THREAD_STACK_SIZE (128 * 1024)
-
 #define GetProcPtr(obj, ptr) \
   GetCoreDataFromValue((obj), rb_proc_t, (ptr))