daemon: capture output of killed health checks

Add an integration test to verify that health checks are killed on
timeout and that the output is captured.

Co-authored-by: Nicolas De Loof <nicolas.deloof@gmail.com>
Signed-off-by: Cory Snider <csnider@mirantis.com>
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

daemon/health.go

@@ -136,9 +136,16 @@ func (p *cmdProbe) run(ctx context.Context, d *Daemon, cntr *container.Container
 		// Wait for probe to exit (it might take some time to call containerd to kill
 		// the process and we don't want dying probes to pile up).
 		<-execErr
+
+		var msg string
+		if out := output.String(); len(out) > 0 {
+			msg = fmt.Sprintf("Health check exceeded timeout (%v): %s", probeTimeout, out)
+		} else {
+			msg = fmt.Sprintf("Health check exceeded timeout (%v)", probeTimeout)
+		}
 		return &types.HealthcheckResult{
 			ExitCode: -1,
-			Output:   fmt.Sprintf("Health check exceeded timeout (%v)", probeTimeout),
+			Output:   msg,
 			End:      time.Now(),
 		}, nil
 	case err := <-execErr:

integration/container/health_test.go

@@ -103,13 +103,13 @@ func TestHealthCheckProcessKilled(t *testing.T) {
 
 	cID := container.Run(ctx, t, apiClient, func(c *container.TestContainerConfig) {
 		c.Config.Healthcheck = &containertypes.HealthConfig{
-			Test:     []string{"CMD", "sh", "-c", "sleep 60"},
+			Test:     []string{"CMD", "sh", "-c", `echo "logs logs logs"; sleep 60`},
 			Interval: 100 * time.Millisecond,
 			Timeout:  50 * time.Millisecond,
 			Retries:  1,
 		}
 	})
-	poll.WaitOn(t, pollForHealthCheckLog(ctx, apiClient, cID, "Health check exceeded timeout (50ms)"))
+	poll.WaitOn(t, pollForHealthCheckLog(ctx, apiClient, cID, "Health check exceeded timeout (50ms): logs logs logs\n"))
 }
 
 func pollForHealthCheckLog(ctx context.Context, client client.APIClient, containerID string, expected string) func(log poll.LogT) poll.Result {