From e75cb61d4645b9833b14a0cc4190cceffc9b6551 Mon Sep 17 00:00:00 2001
From: Alan Wu <XrXr@users.noreply.github.com>
Date: Wed, 14 Jul 2021 19:44:26 -0400
Subject: [PATCH] Fix nested bmethod TracePoint and memory leak

df317151a5b4e0c5a30fcc321a9dc6abad63f7ed removed the code to free
rb_hook_list_t, so repeated targeting of the same bmethod started
to leak the hook list. You can observe how the maximum memory use
scales with input size in the following script with `/usr/bin/time -v`.

```ruby
o = Object.new
o.define_singleton_method(:foo) {}
trace = TracePoint.new(:return) {}

bmethod = o.method(:foo)

ARGV.first.to_i.times { trace.enable(target:bmethod){} }
4.times {GC.start}
```

After this change the maximum doesn't grow as quickly.

To plug the leak, check whether the hook list is already allocated
when enabling the targeting TracePoint for the bmethod. This fix
also allows multiple TracePoints to target the same bmethod, similar
to other valid TracePoint targets.

Finally, free the rb_hook_list_t struct when freeing the method
definition it lives on. Freeing in the GC is a good way to avoid
lifetime problems similar to the one fixed in df31715.

[Bug #18031]
---
 test/ruby/test_settracefunc.rb | 22 ++++++++++++++++++++++
 vm_method.c                    |  4 +++-
 vm_trace.c                     |  4 +++-
 3 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/test/ruby/test_settracefunc.rb b/test/ruby/test_settracefunc.rb
index 37c8b37c59..bbd7579eeb 100644
--- a/test/ruby/test_settracefunc.rb
+++ b/test/ruby/test_settracefunc.rb
@@ -2364,6 +2364,28 @@ CODE
     assert_equal [:tp1, 1, 2, :tp2, 3], events
   end
 
+  def test_multiple_tracepoints_same_bmethod
+    events = []
+    tp1 = TracePoint.new(:return) do |tp|
+      events << :tp1
+    end
+    tp2 = TracePoint.new(:return) do |tp|
+      events << :tp2
+    end
+
+    obj = Object.new
+    obj.define_singleton_method(:foo) {}
+    bmethod = obj.method(:foo)
+
+    tp1.enable(target: bmethod) do
+      tp2.enable(target: bmethod) do
+        obj.foo
+      end
+    end
+
+    assert_equal([:tp2, :tp1], events, '[Bug #18031]')
+  end
+
   def test_script_compiled
     events = []
     tp = TracePoint.new(:script_compiled){|tp|
diff --git a/vm_method.c b/vm_method.c
index b12d134dce..7cd9fee754 100644
--- a/vm_method.c
+++ b/vm_method.c
@@ -405,7 +405,9 @@ rb_method_definition_release(rb_method_definition_t *def, int complemented)
 	if (alias_count + complemented_count == 0) {
             if (METHOD_DEBUG) fprintf(stderr, "-%p-%s:%d,%d (remove)\n", (void *)def,
                                       rb_id2name(def->original_id), alias_count, complemented_count);
-            VM_ASSERT(def->type == VM_METHOD_TYPE_BMETHOD ? def->body.bmethod.hooks == NULL : TRUE);
+            if (def->type == VM_METHOD_TYPE_BMETHOD && def->body.bmethod.hooks) {
+                xfree(def->body.bmethod.hooks);
+            }
 	    xfree(def);
 	}
 	else {
diff --git a/vm_trace.c b/vm_trace.c
index 99d3104b40..7f65f98695 100644
--- a/vm_trace.c
+++ b/vm_trace.c
@@ -1226,7 +1226,9 @@ rb_tracepoint_enable_for_target(VALUE tpval, VALUE target, VALUE target_line)
         rb_method_definition_t *def = (rb_method_definition_t *)rb_method_def(target);
         if (def->type == VM_METHOD_TYPE_BMETHOD &&
             (tp->events & (RUBY_EVENT_CALL | RUBY_EVENT_RETURN))) {
-            def->body.bmethod.hooks = ZALLOC(rb_hook_list_t);
+            if (def->body.bmethod.hooks == NULL) {
+                def->body.bmethod.hooks = ZALLOC(rb_hook_list_t);
+            }
             rb_hook_list_connect_tracepoint(target, def->body.bmethod.hooks, tpval, 0);
             rb_hash_aset(tp->local_target_set, target, Qfalse);
             target_bmethod = true;