From 471e3a301624a86f08c86a643321c3656abbc9fd Mon Sep 17 00:00:00 2001
From: nobu <nobu@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>
Date: Tue, 11 Aug 2015 06:22:34 +0000
Subject: [PATCH] ruby.h: check integer overflow

* include/ruby/ruby.h (ALLOCV_N): check integer overflow, as well
  as ruby_xmalloc2.  pointed out by Paul <pawlkt AT gmail.com>.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@51533 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
---
 ChangeLog           |  5 +++++
 gc.c                | 14 +++++++-------
 include/ruby/ruby.h | 21 +++++++++++++++++++--
 3 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index ca4cf44752..7303a768f2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+Tue Aug 11 15:22:31 2015  Nobuyoshi Nakada  <nobu@ruby-lang.org>
+
+	* include/ruby/ruby.h (ALLOCV_N): check integer overflow, as well
+	  as ruby_xmalloc2.  pointed out by Paul <pawlkt AT gmail.com>.
+
 Tue Aug 11 14:57:09 2015  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* array.c (rb_ary_repeated_permutation): fix buffer size, ALLOCV_N
diff --git a/gc.c b/gc.c
index a3af6fefbb..cd8682547a 100644
--- a/gc.c
+++ b/gc.c
@@ -7649,16 +7649,16 @@ ruby_xmalloc(size_t size)
     return objspace_xmalloc(&rb_objspace, size);
 }
 
-static inline size_t
-xmalloc2_size(size_t n, size_t size)
+void
+ruby_malloc_size_overflow(size_t count, size_t elsize)
 {
-    size_t len = size * n;
-    if (n != 0 && size != len / n) {
-	rb_raise(rb_eArgError, "malloc: possible integer overflow");
-    }
-    return len;
+    rb_raise(rb_eArgError,
+	     "malloc: possible integer overflow (%"PRIdSIZE"*%"PRIdSIZE")",
+	     count, elsize);
 }
 
+#define xmalloc2_size ruby_xmalloc2_size
+
 void *
 ruby_xmalloc2(size_t n, size_t size)
 {
diff --git a/include/ruby/ruby.h b/include/ruby/ruby.h
index 01b3c2efc6..cfc1005919 100644
--- a/include/ruby/ruby.h
+++ b/include/ruby/ruby.h
@@ -1431,14 +1431,31 @@ rb_num2char_inline(VALUE x)
 
 void *rb_alloc_tmp_buffer(volatile VALUE *store, long len) RUBY_ATTR_ALLOC_SIZE((2));
 void rb_free_tmp_buffer(volatile VALUE *store);
+NORETURN(void ruby_malloc_size_overflow(size_t, size_t));
+static inline size_t
+ruby_xmalloc2_size(const size_t count, const size_t elsize)
+{
+    if (count > SIZE_MAX / elsize) {
+	ruby_malloc_size_overflow(count, elsize);
+    }
+    return count * elsize;
+}
 /* allocates _n_ bytes temporary buffer and stores VALUE including it
  * in _v_.  _n_ may be evaluated twice. */
 #ifdef C_ALLOCA
 # define ALLOCV(v, n) rb_alloc_tmp_buffer(&(v), (n))
+# define ALLOCV_N(type, v, n) \
+    ((type*)ALLOCV((v), ruby_xmalloc2_size((n), sizeof(type))))
 #else
-# define ALLOCV(v, n) ((n) < 1024 ? (RB_GC_GUARD(v) = 0, alloca(n)) : rb_alloc_tmp_buffer(&(v), (n)))
+# define ALLOCV_LIMIT 1024
+# define ALLOCV(v, n) ((n) < ALLOCV_LIMIT ? \
+		       (RB_GC_GUARD(v) = 0, alloca(n)) : \
+		       rb_alloc_tmp_buffer(&(v), (n)))
+# define ALLOCV_N(type, v, n) \
+    ((type*)(ruby_xmalloc2_size((n), sizeof(type)) < ALLOCV_LIMIT ? \
+	     (RB_GC_GUARD(v) = 0, alloca((n) * sizeof(type))) : \
+	     rb_alloc_tmp_buffer(&(v), (n) * sizeof(type))))
 #endif
-#define ALLOCV_N(type, v, n) ((type*)ALLOCV((v), sizeof(type)*(n)))
 #define ALLOCV_END(v) rb_free_tmp_buffer(&(v))
 
 #define MEMZERO(p,type,n) memset((p), 0, sizeof(type)*(n))