mutex: Raise a ThreadError when detecting a fiber deadlock (#6680)

[Bug #19105]

If no fiber scheduler is registered and the fiber that
owns the lock and the one that try to acquire it
both belong to the same thread, we're in a deadlock case.

Co-authored-by: Jean Boussier <byroot@ruby-lang.org>

test/fiber/test_mutex.rb

@@ -194,7 +194,7 @@ class TestFiberMutex < Test::Unit::TestCase
   end
 
   def test_mutex_deadlock
-    error_pattern = /No live threads left. Deadlock\?/
+    error_pattern = /lock already owned by another fiber/
 
     assert_in_out_err %W[-I#{__dir__} -], <<-RUBY, ['in synchronize'], error_pattern, success: false
     require 'scheduler'
@@ -217,4 +217,24 @@ class TestFiberMutex < Test::Unit::TestCase
     thread.join
     RUBY
   end
+
+  def test_mutex_fiber_deadlock_no_scheduler
+    thr = Thread.new do
+      loop do
+        sleep 1
+      end
+    end
+
+    mutex = Mutex.new
+    mutex.synchronize do
+      error = assert_raise ThreadError do
+        Fiber.new do
+          mutex.lock
+        end.resume
+      end
+      assert_includes error.message, "deadlock; lock already owned by another fiber belonging to the same thread"
+    end
+  ensure
+    thr&.kill&.join
+  end
 end

thread_sync.c

@@ -327,6 +327,10 @@ do_mutex_lock(VALUE self, int interruptible_p)
                 }
             }
             else {
+                if (!th->vm->thread_ignore_deadlock && rb_fiber_threadptr(mutex->fiber) == th) {
+                    rb_raise(rb_eThreadError, "deadlock; lock already owned by another fiber belonging to the same thread");
+                }
+
                 enum rb_thread_status prev_status = th->status;
                 rb_hrtime_t *timeout = 0;
                 rb_hrtime_t rel = rb_msec2hrtime(100);