Add systemd notify and watchdog support (#2438)

* Adds systemd notification support

* Improve systemd notification support

This takes the work by @acmh and improves on it. This is done by
squashing all commits and rebasing it. Then the following changes were
made:

* Dropped SD_NOTIFY env var. There is aleady the NOTIFY_SOCKET env var
  presented by systemd and is redundant.
* Move code is pushed in Puma::Systemd
* on_reload now emits RELOADING=1 notification to systemd
* Drop lower bound check on usec. Systemd can only be configured in
  seconds and it's hard to misconfigure. The actual code should be safe.
* Clean up integration tests and skip on JRuby

Co-authored-by: Artur Montenegro <artur.montenegro@tempest.com.br>

Gemfile

@@ -11,6 +11,7 @@ gem "minitest", "~> 5.11"
 gem "minitest-retry"
 gem "minitest-proveit"
 gem "minitest-stub-const"
+gem "sd_notify"
 
 gem "jruby-openssl", :platform => "jruby"
 

History.md

@@ -2,6 +2,7 @@
 
 * Features
   * Your feature goes here <Most recent on the top, like GitHub> (#Github Number)
+  * Integrate with systemd's watchdog and notification features (#2438)
   * Adds max_fast_inline as a configuration option for the Server object (#2406)
 
 * Bugfixes

docs/systemd.md

@@ -24,8 +24,15 @@ After=network.target
 # Requires=puma.socket
 
 [Service]
-# Foreground process (do not use --daemon in ExecStart or config.rb)
-Type=simple
+# Puma supports systemd's `Type=notify` and watchdog service
+# monitoring, if the [sd_notify](https://github.com/agis/ruby-sdnotify) gem is installed,
+# as of Puma 5.1 or later.
+# On earlier versions of Puma or JRuby, change this to `Type=simple` and remove
+# the `WatchdogSec` line.
+Type=notify
+
+# If your Puma process locks up, systemd's watchdog will restart it within seconds.
+WatchdogSec=10
 
 # Preferably configure a non-privileged user
 # User=

lib/puma/events.rb

@@ -137,10 +137,26 @@ module Puma
       register(:on_booted, &block)
     end
 
+    def on_restart(&block)
+      register(:on_restart, &block)
+    end
+
+    def on_stopped(&block)
+      register(:on_stopped, &block)
+    end
+
     def fire_on_booted!
       fire(:on_booted)
     end
 
+    def fire_on_restart!
+      fire(:on_restart)
+    end
+
+    def fire_on_stopped!
+      fire(:on_stopped)
+    end
+
     DEFAULT = new(STDOUT, STDERR)
 
     # Returns an Events object which writes its status to 2 StringIO

lib/puma/launcher.rb

@@ -130,6 +130,7 @@ module Puma
 
     # Begin async shutdown of the server gracefully
     def stop
+      @events.fire_on_stopped!
       @status = :stop
       @runner.stop
     end
@@ -168,6 +169,7 @@ module Puma
 
       setup_signals
       set_process_title
+      integrate_with_systemd
       @runner.run
 
       case @status
@@ -242,6 +244,7 @@ module Puma
     end
 
     def restart!
+      @events.fire_on_restart!
       @config.run_hooks :on_restart, self, @events
 
       if Puma.jruby?
@@ -316,6 +319,30 @@ module Puma
       end
     end
 
+    #
+    # Puma's systemd integration allows Puma to inform systemd:
+    #  1. when it has successfully started
+    #  2. when it is starting shutdown
+    #  3. periodically for a liveness check with a watchdog thread
+    #
+
+    def integrate_with_systemd
+      return unless ENV["NOTIFY_SOCKET"]
+
+      begin
+        require 'puma/systemd'
+      rescue LoadError
+        log "Systemd integration failed. It looks like you're trying to use systemd notify but don't have sd_notify gem installed"
+        return
+      end
+
+      log "* Enabling systemd notification integration"
+
+      systemd = Systemd.new(@events)
+      systemd.hook_events
+      systemd.start_watchdog
+    end
+
     def spec_for_gem(gem_name)
       Bundler.rubygems.loaded_specs(gem_name)
     end
@@ -338,6 +365,7 @@ module Puma
     end
 
     def graceful_stop
+      @events.fire_on_stopped!
       @runner.stop_blocked
       log "=== puma shutdown: #{Time.now} ==="
       log "- Goodbye!"

lib/puma/systemd.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'sd_notify'
+
+module Puma
+  class Systemd
+    def initialize(events)
+      @events = events
+    end
+
+    def hook_events
+      @events.on_booted { SdNotify.ready }
+      @events.on_stopped { SdNotify.stopping }
+      @events.on_restart { SdNotify.reloading }
+    end
+
+    def start_watchdog
+      return unless SdNotify.watchdog?
+
+      ping_f = watchdog_sleep_time
+
+      log "Pinging systemd watchdog every #{ping_f.round(1)} sec"
+      Thread.new do
+        loop do
+          sleep ping_f
+          SdNotify.watchdog
+        end
+      end
+    end
+
+    private
+
+    def watchdog_sleep_time
+      usec = Integer(ENV["WATCHDOG_USEC"])
+
+      sec_f = usec / 1_000_000.0
+      # "It is recommended that a daemon sends a keep-alive notification message
+      # to the service manager every half of the time returned here."
+      sec_f / 2
+    end
+
+    def log(str)
+      @events.log str
+    end
+  end
+end

test/test_integration_systemd.rb

@@ -0,0 +1,62 @@
+require_relative "helper"
+require_relative "helpers/integration"
+
+require 'sd_notify'
+
+class TestIntegrationSystemd < TestIntegration
+  def setup
+    skip "Skipped because Systemd support is linux-only" if windows? || osx?
+    skip UNIX_SKT_MSG unless UNIX_SKT_EXIST
+    skip_unless_signal_exist? :TERM
+    skip_on :jruby
+
+    super
+
+    ::Dir::Tmpname.create("puma_socket") do |sockaddr|
+      @sockaddr = sockaddr
+      @socket = Socket.new(:UNIX, :DGRAM, 0)
+      socket_ai = Addrinfo.unix(sockaddr)
+      @socket.bind(socket_ai)
+      ENV["NOTIFY_SOCKET"] = sockaddr
+    end
+  end
+
+  def teardown
+    return if skipped?
+    @socket.close if @socket
+    File.unlink(@sockaddr) if @sockaddr
+    @socket = nil
+    @sockaddr = nil
+    ENV["NOTIFY_SOCKET"] = nil
+    ENV["WATCHDOG_USEC"] = nil
+  end
+
+  def socket_message
+    @socket.recvfrom(15)[0]
+  end
+
+  def test_systemd_integration
+    cli_server "test/rackup/hello.ru"
+    assert_equal(socket_message, "READY=1")
+
+    connection = connect
+    restart_server connection
+    assert_equal(socket_message, "RELOADING=1")
+    assert_equal(socket_message, "READY=1")
+
+    stop_server
+    assert_equal(socket_message, "STOPPING=1")
+  end
+
+  def test_systemd_watchdog
+    ENV["WATCHDOG_USEC"] = "1_000_000"
+
+    cli_server "test/rackup/hello.ru"
+    assert_equal(socket_message, "READY=1")
+
+    assert_equal(socket_message, "WATCHDOG=1")
+
+    stop_server
+    assert_match(socket_message, "STOPPING=1")
+  end
+end