// +build !windows package main import ( "strconv" "strings" "github.com/docker/docker/daemon/cluster/executor/container" "github.com/docker/docker/pkg/integration/checker" "github.com/docker/engine-api/types/swarm" "github.com/go-check/check" ) // start a service, and then make its task unhealthy during running // finally, unhealthy task should be detected and killed func (s *DockerSwarmSuite) TestServiceHealthRun(c *check.C) { testRequires(c, DaemonIsLinux) // busybox doesn't work on Windows d := s.AddDaemon(c, true, true) // build image with health-check // note: use `daemon.buildImageWithOut` to build, do not use `buildImage` to build imageName := "testhealth" _, _, err := d.buildImageWithOut(imageName, `FROM busybox RUN touch /status HEALTHCHECK --interval=1s --timeout=1s --retries=1\ CMD cat /status`, true) c.Check(err, check.IsNil) serviceName := "healthServiceRun" out, err := d.Cmd("service", "create", "--name", serviceName, imageName, "top") c.Assert(err, checker.IsNil, check.Commentf(out)) id := strings.TrimSpace(out) var tasks []swarm.Task waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { tasks = d.getServiceTasks(c, id) return tasks, nil }, checker.HasLen, 1) task := tasks[0] // wait for task to start waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { task = d.getTask(c, task.ID) return task.Status.State, nil }, checker.Equals, swarm.TaskStateStarting) containerID := task.Status.ContainerStatus.ContainerID // wait for container to be healthy waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { out, _ := d.Cmd("inspect", "--format={{.State.Health.Status}}", containerID) return strings.TrimSpace(out), nil }, checker.Equals, "healthy") // make it fail d.Cmd("exec", containerID, "rm", "/status") // wait for container to be unhealthy waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { out, _ := d.Cmd("inspect", "--format={{.State.Health.Status}}", containerID) return strings.TrimSpace(out), nil }, checker.Equals, "unhealthy") // Task should be terminated waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { task = d.getTask(c, task.ID) return task.Status.State, nil }, checker.Equals, swarm.TaskStateFailed) if !strings.Contains(task.Status.Err, container.ErrContainerUnhealthy.Error()) { c.Fatal("unhealthy task exits because of other error") } } // start a service whose task is unhealthy at beginning // its tasks should be blocked in starting stage, until health check is passed func (s *DockerSwarmSuite) TestServiceHealthStart(c *check.C) { testRequires(c, DaemonIsLinux) // busybox doesn't work on Windows d := s.AddDaemon(c, true, true) // service started from this image won't pass health check imageName := "testhealth" _, _, err := d.buildImageWithOut(imageName, `FROM busybox HEALTHCHECK --interval=1s --timeout=1s --retries=1024\ CMD cat /status`, true) c.Check(err, check.IsNil) serviceName := "healthServiceStart" out, err := d.Cmd("service", "create", "--name", serviceName, imageName, "top") c.Assert(err, checker.IsNil, check.Commentf(out)) id := strings.TrimSpace(out) var tasks []swarm.Task waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { tasks = d.getServiceTasks(c, id) return tasks, nil }, checker.HasLen, 1) task := tasks[0] // wait for task to start waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { task = d.getTask(c, task.ID) return task.Status.State, nil }, checker.Equals, swarm.TaskStateStarting) containerID := task.Status.ContainerStatus.ContainerID // wait for health check to work waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { out, _ := d.Cmd("inspect", "--format={{.State.Health.FailingStreak}}", containerID) failingStreak, _ := strconv.Atoi(strings.TrimSpace(out)) return failingStreak, nil }, checker.GreaterThan, 0) // task should be blocked at starting status task = d.getTask(c, task.ID) c.Assert(task.Status.State, check.Equals, swarm.TaskStateStarting) // make it healthy d.Cmd("exec", containerID, "touch", "/status") // Task should be at running status waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { task = d.getTask(c, task.ID) return task.Status.State, nil }, checker.Equals, swarm.TaskStateRunning) } // start a service whose task is unhealthy at beginning // its tasks should be blocked in starting stage, until health check is passed func (s *DockerSwarmSuite) TestServiceHealthUpdate(c *check.C) { testRequires(c, DaemonIsLinux) // busybox doesn't work on Windows d := s.AddDaemon(c, true, true) // service started from this image won't pass health check imageName := "testhealth" _, _, err := d.buildImageWithOut(imageName, `FROM busybox HEALTHCHECK --interval=1s --timeout=1s --retries=1024\ CMD cat /status`, true) c.Check(err, check.IsNil) serviceName := "healthServiceStart" out, err := d.Cmd("service", "create", "--name", serviceName, imageName, "top") c.Assert(err, checker.IsNil, check.Commentf(out)) id := strings.TrimSpace(out) var tasks []swarm.Task waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { tasks = d.getServiceTasks(c, id) return tasks, nil }, checker.HasLen, 1) task := tasks[0] // wait for task to start waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { task = d.getTask(c, task.ID) return task.Status.State, nil }, checker.Equals, swarm.TaskStateStarting) containerID := task.Status.ContainerStatus.ContainerID // wait for health check to work waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { out, _ := d.Cmd("inspect", "--format={{.State.Health.FailingStreak}}", containerID) failingStreak, _ := strconv.Atoi(strings.TrimSpace(out)) return failingStreak, nil }, checker.GreaterThan, 0) // task should be blocked at starting status task = d.getTask(c, task.ID) c.Assert(task.Status.State, check.Equals, swarm.TaskStateStarting) // make it healthy d.Cmd("exec", containerID, "touch", "/status") // Task should be at running status waitAndAssert(c, defaultReconciliationTimeout, func(c *check.C) (interface{}, check.CommentInterface) { task = d.getTask(c, task.ID) return task.Status.State, nil }, checker.Equals, swarm.TaskStateRunning) }