Merge pull request #28938 from elifa/master

Grace period option to health checks.

api/swagger.yaml

@@ -506,6 +506,9 @@ definitions:
       Retries:
         description: "The number of consecutive failures needed to consider a container as unhealthy. 0 means inherit."
         type: "integer"
+      StartPeriod:
+        description: "Start period for the container to initialize before starting health-retries countdown in nanoseconds. 0 means inherit."
+        type: "integer"
 
   HostConfig:
     description: "Container configuration that depends on the host we are running on"

api/types/container/config.go

@@ -21,6 +21,7 @@ type HealthConfig struct {
 	// Zero means to inherit. Durations are expressed as integer nanoseconds.
 	Interval    time.Duration `json:",omitempty"` // Interval is the time to wait between checks.
 	Timeout     time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung.
+	StartPeriod time.Duration `json:",omitempty"` // The start period for the container to initialize before the retries starts to count down.
 
 	// Retries is the number of consecutive failures needed to consider a container as unhealthy.
 	// Zero means inherit.

builder/dockerfile/dispatchers.go

@@ -540,6 +540,7 @@ func healthcheck(b *Builder, args []string, attributes map[string]bool, original
 
 		flInterval := b.flags.AddString("interval", "")
 		flTimeout := b.flags.AddString("timeout", "")
+		flStartPeriod := b.flags.AddString("start-period", "")
 		flRetries := b.flags.AddString("retries", "")
 
 		if err := b.flags.Parse(); err != nil {
@@ -574,6 +575,12 @@ func healthcheck(b *Builder, args []string, attributes map[string]bool, original
 		}
 		healthcheck.Timeout = timeout
 
+		startPeriod, err := parseOptInterval(flStartPeriod)
+		if err != nil {
+			return err
+		}
+		healthcheck.StartPeriod = startPeriod
+
 		if flRetries.Value != "" {
 			retries, err := strconv.ParseInt(flRetries.Value, 10, 32)
 			if err != nil {

cli/command/container/opts.go

@@ -113,6 +113,7 @@ type containerOptions struct {
 	healthCmd          string
 	healthInterval     time.Duration
 	healthTimeout      time.Duration
+	healthStartPeriod  time.Duration
 	healthRetries      int
 	runtime            string
 	autoRemove         bool
@@ -232,6 +233,8 @@ func addFlags(flags *pflag.FlagSet) *containerOptions {
 	flags.DurationVar(&copts.healthInterval, "health-interval", 0, "Time between running the check (ns|us|ms|s|m|h) (default 0s)")
 	flags.IntVar(&copts.healthRetries, "health-retries", 0, "Consecutive failures needed to report unhealthy")
 	flags.DurationVar(&copts.healthTimeout, "health-timeout", 0, "Maximum time to allow one check to run (ns|us|ms|s|m|h) (default 0s)")
+	flags.DurationVar(&copts.healthStartPeriod, "health-start-period", 0, "Start period for the container to initialize before starting health-retries countdown (ns|us|ms|s|m|h) (default 0s)")
+	flags.SetAnnotation("health-start-period", "version", []string{"1.29"})
 	flags.BoolVar(&copts.noHealthcheck, "no-healthcheck", false, "Disable any container-specified HEALTHCHECK")
 
 	// Resource management
@@ -464,6 +467,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions) (*containerConfig, err
 	haveHealthSettings := copts.healthCmd != "" ||
 		copts.healthInterval != 0 ||
 		copts.healthTimeout != 0 ||
+		copts.healthStartPeriod != 0 ||
 		copts.healthRetries != 0
 	if copts.noHealthcheck {
 		if haveHealthSettings {
@@ -486,11 +490,15 @@ func parse(flags *pflag.FlagSet, copts *containerOptions) (*containerConfig, err
 		if copts.healthRetries < 0 {
 			return nil, errors.Errorf("--health-retries cannot be negative")
 		}
+		if copts.healthStartPeriod < 0 {
+			return nil, fmt.Errorf("--health-start-period cannot be negative")
+		}
 
 		healthConfig = &container.HealthConfig{
 			Test:        probe,
 			Interval:    copts.healthInterval,
 			Timeout:     copts.healthTimeout,
+			StartPeriod: copts.healthStartPeriod,
 			Retries:     copts.healthRetries,
 		}
 	}

cli/command/container/opts_test.go

@@ -501,8 +501,8 @@ func TestParseHealth(t *testing.T) {
 	checkError("--no-healthcheck conflicts with --health-* options",
 		"--no-healthcheck", "--health-cmd=/check.sh -q", "img", "cmd")
 
-	health = checkOk("--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "img", "cmd")
+	health = checkOk("--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "--health-start-period=5s", "img", "cmd")
-	if health.Timeout != 2*time.Second || health.Retries != 3 || health.Interval != 4500*time.Millisecond {
+	if health.Timeout != 2*time.Second || health.Retries != 3 || health.Interval != 4500*time.Millisecond || health.StartPeriod != 5*time.Second {
 		t.Fatalf("--health-*: got %#v", health)
 	}
 }

cli/command/service/opts.go

@@ -282,6 +282,7 @@ type healthCheckOptions struct {
 	interval      PositiveDurationOpt
 	timeout       PositiveDurationOpt
 	retries       int
+	startPeriod   PositiveDurationOpt
 	noHealthcheck bool
 }
 
@@ -301,18 +302,22 @@ func (opts *healthCheckOptions) toHealthConfig() (*container.HealthConfig, error
 		if opts.cmd != "" {
 			test = []string{"CMD-SHELL", opts.cmd}
 		}
-		var interval, timeout time.Duration
+		var interval, timeout, startPeriod time.Duration
 		if ptr := opts.interval.Value(); ptr != nil {
 			interval = *ptr
 		}
 		if ptr := opts.timeout.Value(); ptr != nil {
 			timeout = *ptr
 		}
+		if ptr := opts.startPeriod.Value(); ptr != nil {
+			startPeriod = *ptr
+		}
 		healthConfig = &container.HealthConfig{
 			Test:        test,
 			Interval:    interval,
 			Timeout:     timeout,
 			Retries:     opts.retries,
+			StartPeriod: startPeriod,
 		}
 	}
 	return healthConfig, nil
@@ -555,6 +560,8 @@ func addServiceFlags(flags *pflag.FlagSet, opts *serviceOptions) {
 	flags.SetAnnotation(flagHealthTimeout, "version", []string{"1.25"})
 	flags.IntVar(&opts.healthcheck.retries, flagHealthRetries, 0, "Consecutive failures needed to report unhealthy")
 	flags.SetAnnotation(flagHealthRetries, "version", []string{"1.25"})
+	flags.Var(&opts.healthcheck.startPeriod, flagHealthStartPeriod, "Start period for the container to initialize before counting retries towards unstable (ns|us|ms|s|m|h)")
+	flags.SetAnnotation(flagHealthStartPeriod, "version", []string{"1.29"})
 	flags.BoolVar(&opts.healthcheck.noHealthcheck, flagNoHealthcheck, false, "Disable any container-specified HEALTHCHECK")
 	flags.SetAnnotation(flagNoHealthcheck, "version", []string{"1.25"})
 
@@ -644,6 +651,7 @@ const (
 	flagHealthInterval          = "health-interval"
 	flagHealthRetries           = "health-retries"
 	flagHealthTimeout           = "health-timeout"
+	flagHealthStartPeriod       = "health-start-period"
 	flagNoHealthcheck           = "no-healthcheck"
 	flagSecret                  = "secret"
 	flagSecretAdd               = "secret-add"

cli/command/service/opts_test.go

@@ -74,6 +74,7 @@ func TestHealthCheckOptionsToHealthConfig(t *testing.T) {
 		cmd:         "curl",
 		interval:    PositiveDurationOpt{DurationOpt{value: &dur}},
 		timeout:     PositiveDurationOpt{DurationOpt{value: &dur}},
+		startPeriod: PositiveDurationOpt{DurationOpt{value: &dur}},
 		retries:     10,
 	}
 	config, err := opt.toHealthConfig()
@@ -82,6 +83,7 @@ func TestHealthCheckOptionsToHealthConfig(t *testing.T) {
 		Test:        []string{"CMD-SHELL", "curl"},
 		Interval:    time.Second,
 		Timeout:     time.Second,
+		StartPeriod: time.Second,
 		Retries:     10,
 	}), true)
 }

cli/command/service/update.go

@@ -898,7 +898,7 @@ func updateLogDriver(flags *pflag.FlagSet, taskTemplate *swarm.TaskSpec) error {
 }
 
 func updateHealthcheck(flags *pflag.FlagSet, containerSpec *swarm.ContainerSpec) error {
-	if !anyChanged(flags, flagNoHealthcheck, flagHealthCmd, flagHealthInterval, flagHealthRetries, flagHealthTimeout) {
+	if !anyChanged(flags, flagNoHealthcheck, flagHealthCmd, flagHealthInterval, flagHealthRetries, flagHealthTimeout, flagHealthStartPeriod) {
 		return nil
 	}
 	if containerSpec.Healthcheck == nil {
@@ -909,7 +909,7 @@ func updateHealthcheck(flags *pflag.FlagSet, containerSpec *swarm.ContainerSpec)
 		return err
 	}
 	if noHealthcheck {
-		if !anyChanged(flags, flagHealthCmd, flagHealthInterval, flagHealthRetries, flagHealthTimeout) {
+		if !anyChanged(flags, flagHealthCmd, flagHealthInterval, flagHealthRetries, flagHealthTimeout, flagHealthStartPeriod) {
 			containerSpec.Healthcheck = &container.HealthConfig{
 				Test: []string{"NONE"},
 			}
@@ -928,6 +928,10 @@ func updateHealthcheck(flags *pflag.FlagSet, containerSpec *swarm.ContainerSpec)
 		val := *flags.Lookup(flagHealthTimeout).Value.(*PositiveDurationOpt).Value()
 		containerSpec.Healthcheck.Timeout = val
 	}
+	if flags.Changed(flagHealthStartPeriod) {
+		val := *flags.Lookup(flagHealthStartPeriod).Value.(*PositiveDurationOpt).Value()
+		containerSpec.Healthcheck.StartPeriod = val
+	}
 	if flags.Changed(flagHealthRetries) {
 		containerSpec.Healthcheck.Retries, _ = flags.GetInt(flagHealthRetries)
 	}

cli/command/service/update_test.go

@@ -311,6 +311,11 @@ func TestUpdateHealthcheckTable(t *testing.T) {
 			initial:  &container.HealthConfig{Test: []string{"CMD", "cmd1"}, Retries: 10},
 			expected: &container.HealthConfig{Test: []string{"CMD", "cmd1"}},
 		},
+		{
+			flags:    [][2]string{{"health-start-period", "1m"}},
+			initial:  &container.HealthConfig{Test: []string{"CMD", "cmd1"}},
+			expected: &container.HealthConfig{Test: []string{"CMD", "cmd1"}, StartPeriod: time.Minute},
+		},
 		{
 			flags: [][2]string{{"health-cmd", "cmd1"}, {"no-healthcheck", "true"}},
 			err:   "--no-healthcheck conflicts with --health-* options",

cli/compose/convert/service.go

@@ -256,7 +256,7 @@ func convertHealthcheck(healthcheck *composetypes.HealthCheckConfig) (*container
 	}
 	var (
 		err                            error
-		timeout, interval time.Duration
+		timeout, interval, startPeriod time.Duration
 		retries                        int
 	)
 	if healthcheck.Disable {
@@ -280,6 +280,12 @@ func convertHealthcheck(healthcheck *composetypes.HealthCheckConfig) (*container
 			return nil, err
 		}
 	}
+	if healthcheck.StartPeriod != "" {
+		startPeriod, err = time.ParseDuration(healthcheck.StartPeriod)
+		if err != nil {
+			return nil, err
+		}
+	}
 	if healthcheck.Retries != nil {
 		retries = int(*healthcheck.Retries)
 	}
@@ -288,6 +294,7 @@ func convertHealthcheck(healthcheck *composetypes.HealthCheckConfig) (*container
 		Timeout:     timeout,
 		Interval:    interval,
 		Retries:     retries,
+		StartPeriod: startPeriod,
 	}, nil
 }
 

cli/compose/types/types.go

@@ -167,6 +167,7 @@ type HealthCheckConfig struct {
 	Timeout     string
 	Interval    string
 	Retries     *uint64
+	StartPeriod string
 	Disable     bool
 }
 

daemon/cluster/convert/container.go

@@ -221,11 +221,13 @@ func containerToGRPC(c types.ContainerSpec) (*swarmapi.ContainerSpec, error) {
 func healthConfigFromGRPC(h *swarmapi.HealthConfig) *container.HealthConfig {
 	interval, _ := gogotypes.DurationFromProto(h.Interval)
 	timeout, _ := gogotypes.DurationFromProto(h.Timeout)
+	startPeriod, _ := gogotypes.DurationFromProto(h.Timeout)
 	return &container.HealthConfig{
 		Test:        h.Test,
 		Interval:    interval,
 		Timeout:     timeout,
 		Retries:     int(h.Retries),
+		StartPeriod: startPeriod,
 	}
 }
 
@@ -235,5 +237,6 @@ func healthConfigToGRPC(h *container.HealthConfig) *swarmapi.HealthConfig {
 		Interval:    gogotypes.DurationProto(h.Interval),
 		Timeout:     gogotypes.DurationProto(h.Timeout),
 		Retries:     int32(h.Retries),
+		StartPeriod: gogotypes.DurationProto(h.StartPeriod),
 	}
 }

daemon/cluster/executor/container/container.go

@@ -326,11 +326,13 @@ func (c *containerConfig) healthcheck() *enginecontainer.HealthConfig {
 	}
 	interval, _ := gogotypes.DurationFromProto(hcSpec.Interval)
 	timeout, _ := gogotypes.DurationFromProto(hcSpec.Timeout)
+	startPeriod, _ := gogotypes.DurationFromProto(hcSpec.StartPeriod)
 	return &enginecontainer.HealthConfig{
 		Test:        hcSpec.Test,
 		Interval:    interval,
 		Timeout:     timeout,
 		Retries:     int(hcSpec.Retries),
+		StartPeriod: startPeriod,
 	}
 }
 

daemon/commit.go

@@ -94,6 +94,9 @@ func merge(userConf, imageConf *containertypes.Config) error {
 			if userConf.Healthcheck.Timeout == 0 {
 				userConf.Healthcheck.Timeout = imageConf.Healthcheck.Timeout
 			}
+			if userConf.Healthcheck.StartPeriod == 0 {
+				userConf.Healthcheck.StartPeriod = imageConf.Healthcheck.StartPeriod
+			}
 			if userConf.Healthcheck.Retries == 0 {
 				userConf.Healthcheck.Retries = imageConf.Healthcheck.Retries
 			}

daemon/container.go

@@ -255,6 +255,10 @@ func (daemon *Daemon) verifyContainerSettings(hostConfig *containertypes.HostCon
 			if config.Healthcheck.Retries < 0 {
 				return nil, fmt.Errorf("Retries in Healthcheck cannot be negative")
 			}
+
+			if config.Healthcheck.StartPeriod < 0 {
+				return nil, fmt.Errorf("StartPeriod in Healthcheck cannot be negative")
+			}
 		}
 	}
 

daemon/health.go

@@ -30,6 +30,10 @@ const (
 	// than this, the check is considered to have failed.
 	defaultProbeTimeout = 30 * time.Second
 
+	// The time given for the container to start before the health check starts considering
+	// the container unstable. Defaults to none.
+	defaultStartPeriod = 0 * time.Second
+
 	// Default number of consecutive failures of the health check
 	// for the container to be considered unhealthy.
 	defaultProbeRetries = 3
@@ -133,12 +137,29 @@ func handleProbeResult(d *Daemon, c *container.Container, result *types.Healthch
 	if result.ExitCode == exitStatusHealthy {
 		h.FailingStreak = 0
 		h.Status = types.Healthy
-	} else {
+	} else { // Failure (including invalid exit code)
-		// Failure (including invalid exit code)
+		shouldIncrementStreak := true
+
+		// If the container is starting (i.e. we never had a successful health check)
+		// then we check if we are within the start period of the container in which
+		// case we do not increment the failure streak.
+		if h.Status == types.Starting {
+			startPeriod := timeoutWithDefault(c.Config.Healthcheck.StartPeriod, defaultStartPeriod)
+			timeSinceStart := result.Start.Sub(c.State.StartedAt)
+
+			// If still within the start period, then don't increment failing streak.
+			if timeSinceStart < startPeriod {
+				shouldIncrementStreak = false
+			}
+		}
+
+		if shouldIncrementStreak {
 			h.FailingStreak++
+
 			if h.FailingStreak >= retries {
 				h.Status = types.Unhealthy
 			}
+		}
 		// Else we're starting or healthy. Stay in that state.
 	}
 

daemon/health_test.go

@@ -116,4 +116,30 @@ func TestHealthStates(t *testing.T) {
 	if c.State.Health.FailingStreak != 0 {
 		t.Errorf("Expecting FailingStreak=0, but got %d\n", c.State.Health.FailingStreak)
 	}
+
+	// Test start period
+
+	reset(c)
+	c.Config.Healthcheck.Retries = 2
+	c.Config.Healthcheck.StartPeriod = 30 * time.Second
+
+	handleResult(c.State.StartedAt.Add(20*time.Second), 1)
+	if c.State.Health.Status != types.Starting {
+		t.Errorf("Expecting starting, but got %#v\n", c.State.Health.Status)
+	}
+	if c.State.Health.FailingStreak != 0 {
+		t.Errorf("Expecting FailingStreak=0, but got %d\n", c.State.Health.FailingStreak)
+	}
+	handleResult(c.State.StartedAt.Add(50*time.Second), 1)
+	if c.State.Health.Status != types.Starting {
+		t.Errorf("Expecting starting, but got %#v\n", c.State.Health.Status)
+	}
+	if c.State.Health.FailingStreak != 1 {
+		t.Errorf("Expecting FailingStreak=1, but got %d\n", c.State.Health.FailingStreak)
+	}
+	handleResult(c.State.StartedAt.Add(80*time.Second), 0)
+	expect("health_status: healthy")
+	if c.State.Health.FailingStreak != 0 {
+		t.Errorf("Expecting FailingStreak=0, but got %d\n", c.State.Health.FailingStreak)
+	}
 }

docs/api/version-history.md

@@ -22,6 +22,7 @@ keywords: "API, Docker, rcli, REST, documentation"
 * `POST /networks/create` now supports creating the ingress network, by specifying an `Ingress` boolean field. As of now this is supported only when using the overlay network driver.
 * `GET /networks/(name)` now returns an `Ingress` field showing whether the network is the ingress one.
 * `GET /networks/` now supports a `scope` filter to filter networks based on the network mode (`swarm`, `global`, or `local`).
+* `POST /containers/create`, `POST /service/create` and `POST /services/(id or name)/update` now takes the field `StartPeriod` as a part of the `HealthConfig` allowing for specification of a period during which the container should not be considered unealthy even if health checks do not pass.
 
 ## v1.28 API changes
 

docs/reference/builder.md

@@ -1591,6 +1591,7 @@ The options that can appear before `CMD` are:
 
 * `--interval=DURATION` (default: `30s`)
 * `--timeout=DURATION` (default: `30s`)
+* `--start-period=DURATION` (default: `0s`)
 * `--retries=N` (default: `3`)
 
 The health check will first run **interval** seconds after the container is
@@ -1602,6 +1603,11 @@ is considered to have failed.
 It takes **retries** consecutive failures of the health check for the container
 to be considered `unhealthy`.
 
+**start period** provides initialization time for containers that need time to bootstrap.
+Probe failure during that period will not be counted towards the maximum number of retries.
+However, if a health check succeeds during the start period, the container is considered
+started and all consecutive failures will be counted towards the maximum number of retries.
+
 There can only be one `HEALTHCHECK` instruction in a Dockerfile. If you list
 more than one then only the last `HEALTHCHECK` will take effect.
 

docs/reference/commandline/create.md

@@ -62,6 +62,7 @@ Options:
       --health-interval duration      Time between running the check (ns|us|ms|s|m|h) (default 0s)
       --health-retries int            Consecutive failures needed to report unhealthy
       --health-timeout duration       Maximum time to allow one check to run (ns|us|ms|s|m|h) (default 0s)
+      --health-start-period duration  Start period for the container to initialize before counting retries towards unstable (ns|us|ms|s|m|h) (default 0s)
       --help                          Print usage
   -h, --hostname string               Container host name
       --init                          Run an init inside the container that forwards signals and reaps processes

docs/reference/commandline/run.md

@@ -66,6 +66,7 @@ Options:
       --health-interval duration      Time between running the check (ns|us|ms|s|m|h) (default 0s)
       --health-retries int            Consecutive failures needed to report unhealthy
       --health-timeout duration       Maximum time to allow one check to run (ns|us|ms|s|m|h) (default 0s)
+      --health-start-period duration  Start period for the container to initialize before counting retries towards unstable (ns|us|ms|s|m|h) (default 0s)
       --help                          Print usage
   -h, --hostname string               Container host name
       --init                          Run an init inside the container that forwards signals and reaps processes

docs/reference/commandline/service_create.md

@@ -36,6 +36,7 @@ Options:
       --health-interval duration           Time between running the check (ns|us|ms|s|m|h)
       --health-retries int                 Consecutive failures needed to report unhealthy
       --health-timeout duration            Maximum time to allow one check to run (ns|us|ms|s|m|h)
+      --health-start-period duration       Start period for the container to initialize before counting retries towards unstable (ns|us|ms|s|m|h) (default 0s)
       --help                               Print usage
       --host list                          Set one or more custom host-to-IP mappings (host:ip) (default [])
       --hostname string                    Container hostname

docs/reference/commandline/service_update.md

@@ -44,6 +44,7 @@ Options:
       --health-interval duration           Time between running the check (ns|us|ms|s|m|h)
       --health-retries int                 Consecutive failures needed to report unhealthy
       --health-timeout duration            Maximum time to allow one check to run (ns|us|ms|s|m|h)
+      --health-start-period duration       Start period for the container to initialize before counting retries towards unstable (ns|us|ms|s|m|h) (default 0s)
       --help                               Print usage
       --host-add list                      Add or update a custom host-to-IP mapping (host:ip) (default [])
       --host-rm list                       Remove a custom host-to-IP mapping (host:ip) (default [])

docs/reference/run.md

@@ -1462,6 +1462,7 @@ Similarly the operator can set the **HOSTNAME** (Linux) or **COMPUTERNAME** (Win
   --health-interval       Time between running the check
   --health-retries        Consecutive failures needed to report unhealthy
   --health-timeout        Maximum time to allow one check to run
+  --health-start-period   Start period for the container to initialize before starting health-retries countdown
   --no-healthcheck        Disable any container-specified HEALTHCHECK
 ```