From eacedcfe44a0ae22bf54ddb7df193c48d4c857c6 Mon Sep 17 00:00:00 2001
From: Jean byroot Boussier <jean.boussier+github@shopify.com>
Date: Tue, 8 Nov 2022 20:43:16 +0900
Subject: [PATCH] mutex: Raise a ThreadError when detecting a fiber deadlock
 (#6680)

[Bug #19105]

If no fiber scheduler is registered and the fiber that
owns the lock and the one that try to acquire it
both belong to the same thread, we're in a deadlock case.

Co-authored-by: Jean Boussier <byroot@ruby-lang.org>
---
 test/fiber/test_mutex.rb | 22 +++++++++++++++++++++-
 thread_sync.c            |  4 ++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/test/fiber/test_mutex.rb b/test/fiber/test_mutex.rb
index b0655f06a5..449c49f38b 100644
--- a/test/fiber/test_mutex.rb
+++ b/test/fiber/test_mutex.rb
@@ -194,7 +194,7 @@ class TestFiberMutex < Test::Unit::TestCase
   end
 
   def test_mutex_deadlock
-    error_pattern = /No live threads left. Deadlock\?/
+    error_pattern = /lock already owned by another fiber/
 
     assert_in_out_err %W[-I#{__dir__} -], <<-RUBY, ['in synchronize'], error_pattern, success: false
     require 'scheduler'
@@ -217,4 +217,24 @@ class TestFiberMutex < Test::Unit::TestCase
     thread.join
     RUBY
   end
+
+  def test_mutex_fiber_deadlock_no_scheduler
+    thr = Thread.new do
+      loop do
+        sleep 1
+      end
+    end
+
+    mutex = Mutex.new
+    mutex.synchronize do
+      error = assert_raise ThreadError do
+        Fiber.new do
+          mutex.lock
+        end.resume
+      end
+      assert_includes error.message, "deadlock; lock already owned by another fiber belonging to the same thread"
+    end
+  ensure
+    thr&.kill&.join
+  end
 end
diff --git a/thread_sync.c b/thread_sync.c
index 2bcf59137e..2f43896cfb 100644
--- a/thread_sync.c
+++ b/thread_sync.c
@@ -327,6 +327,10 @@ do_mutex_lock(VALUE self, int interruptible_p)
                 }
             }
             else {
+                if (!th->vm->thread_ignore_deadlock && rb_fiber_threadptr(mutex->fiber) == th) {
+                    rb_raise(rb_eThreadError, "deadlock; lock already owned by another fiber belonging to the same thread");
+                }
+
                 enum rb_thread_status prev_status = th->status;
                 rb_hrtime_t *timeout = 0;
                 rb_hrtime_t rel = rb_msec2hrtime(100);