Merge pull request #24943 from aaronlehmann/rolling-updates

Add failure action for rolling updates

api/client/service/inspect.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"time"
 
 	"golang.org/x/net/context"
 
@@ -101,6 +102,17 @@ func printService(out io.Writer, service swarm.Service) {
 			fmt.Fprintf(out, " Replicas:\t%d\n", *service.Spec.Mode.Replicated.Replicas)
 		}
 	}
+
+	if service.UpdateStatus.State != "" {
+		fmt.Fprintln(out, "Update status:")
+		fmt.Fprintf(out, " State:\t\t%s\n", service.UpdateStatus.State)
+		fmt.Fprintf(out, " Started:\t%s ago\n", strings.ToLower(units.HumanDuration(time.Since(service.UpdateStatus.StartedAt))))
+		if service.UpdateStatus.State == swarm.UpdateStateCompleted {
+			fmt.Fprintf(out, " Completed:\t%s ago\n", strings.ToLower(units.HumanDuration(time.Since(service.UpdateStatus.CompletedAt))))
+		}
+		fmt.Fprintf(out, " Message:\t%s\n", service.UpdateStatus.Message)
+	}
+
 	fmt.Fprintln(out, "Placement:")
 	if service.Spec.TaskTemplate.Placement != nil && len(service.Spec.TaskTemplate.Placement.Constraints) > 0 {
 		ioutils.FprintfIfNotEmpty(out, " Constraints\t: %s\n", strings.Join(service.Spec.TaskTemplate.Placement.Constraints, ", "))
@@ -110,6 +122,7 @@ func printService(out io.Writer, service swarm.Service) {
 	if service.Spec.UpdateConfig.Delay.Nanoseconds() > 0 {
 		fmt.Fprintf(out, " Delay:\t\t%s\n", service.Spec.UpdateConfig.Delay)
 	}
+	fmt.Fprintf(out, " On failure:\t%s\n", service.Spec.UpdateConfig.FailureAction)
 	fmt.Fprintf(out, "ContainerSpec:\n")
 	printContainerSpec(out, service.Spec.TaskTemplate.ContainerSpec)
 

api/client/service/opts.go

@@ -274,6 +274,7 @@ func (m *MountOpt) Value() []swarm.Mount {
 type updateOptions struct {
 	parallelism uint64
 	delay       time.Duration
+	onFailure   string
 }
 
 type resourceOptions struct {
@@ -457,6 +458,7 @@ func (opts *serviceOptions) ToService() (swarm.ServiceSpec, error) {
 		UpdateConfig: &swarm.UpdateConfig{
 			Parallelism:   opts.update.parallelism,
 			Delay:         opts.update.delay,
+			FailureAction: opts.update.onFailure,
 		},
 		Networks:     convertNetworks(opts.networks),
 		EndpointSpec: opts.endpoint.ToEndpointSpec(),
@@ -503,6 +505,7 @@ func addServiceFlags(cmd *cobra.Command, opts *serviceOptions) {
 
 	flags.Uint64Var(&opts.update.parallelism, flagUpdateParallelism, 1, "Maximum number of tasks updated simultaneously (0 to update all at once)")
 	flags.DurationVar(&opts.update.delay, flagUpdateDelay, time.Duration(0), "Delay between updates")
+	flags.StringVar(&opts.update.onFailure, flagUpdateFailureAction, "pause", "Action on update failure (pause|continue)")
 
 	flags.StringVar(&opts.endpoint.mode, flagEndpointMode, "", "Endpoint mode (vip or dnsrr)")
 
@@ -545,6 +548,7 @@ const (
 	flagRestartWindow       = "restart-window"
 	flagStopGracePeriod     = "stop-grace-period"
 	flagUpdateDelay         = "update-delay"
+	flagUpdateFailureAction = "update-failure-action"
 	flagUpdateParallelism   = "update-parallelism"
 	flagUser                = "user"
 	flagRegistryAuth        = "with-registry-auth"

api/client/service/update.go

@@ -191,12 +191,13 @@ func updateService(flags *pflag.FlagSet, spec *swarm.ServiceSpec) error {
 		return err
 	}
 
-	if anyChanged(flags, flagUpdateParallelism, flagUpdateDelay) {
+	if anyChanged(flags, flagUpdateParallelism, flagUpdateDelay, flagUpdateFailureAction) {
 		if spec.UpdateConfig == nil {
 			spec.UpdateConfig = &swarm.UpdateConfig{}
 		}
 		updateUint64(flagUpdateParallelism, &spec.UpdateConfig.Parallelism)
 		updateDuration(flagUpdateDelay, &spec.UpdateConfig.Delay)
+		updateString(flagUpdateFailureAction, &spec.UpdateConfig.FailureAction)
 	}
 
 	updateNetworks(flags, &spec.Networks)

contrib/completion/bash/docker

@@ -1726,6 +1726,7 @@ _docker_service_update() {
 		--restart-window
 		--stop-grace-period
 		--update-delay
+		--update-failure-action
 		--update-parallelism
 		--user -u
 		--workdir -w

contrib/completion/zsh/_docker

@@ -1094,6 +1094,7 @@ __docker_service_subcommand() {
         "($help)--restart-window=[Window used to evaluate the restart policy]:window: "
         "($help)--stop-grace-period=[Time to wait before force killing a container]:grace period: "
         "($help)--update-delay=[Delay between updates]:delay: "
+        "($help)--update-failure-action=[Action on update failure]:mode:(pause continue)"
         "($help)--update-parallelism=[Maximum number of tasks updated simultaneously]:number: "
         "($help -u --user)"{-u=,--user=}"[Username or UID]:user:_users"
         "($help)--with-registry-auth[Send registry authentication details to swarm agents]"

daemon/cluster/convert/service.go

@@ -53,9 +53,16 @@ func ServiceFromGRPC(s swarmapi.Service) types.Service {
 		}
 
 		service.Spec.UpdateConfig.Delay, _ = ptypes.Duration(&s.Spec.Update.Delay)
+
+		switch s.Spec.Update.FailureAction {
+		case swarmapi.UpdateConfig_PAUSE:
+			service.Spec.UpdateConfig.FailureAction = types.UpdateFailureActionPause
+		case swarmapi.UpdateConfig_CONTINUE:
+			service.Spec.UpdateConfig.FailureAction = types.UpdateFailureActionContinue
+		}
 	}
 
-	//Mode
+	// Mode
 	switch t := s.Spec.GetMode().(type) {
 	case *swarmapi.ServiceSpec_Global:
 		service.Spec.Mode.Global = &types.GlobalService{}
@@ -65,6 +72,23 @@ func ServiceFromGRPC(s swarmapi.Service) types.Service {
 		}
 	}
 
+	// UpdateStatus
+	service.UpdateStatus = types.UpdateStatus{}
+	if s.UpdateStatus != nil {
+		switch s.UpdateStatus.State {
+		case swarmapi.UpdateStatus_UPDATING:
+			service.UpdateStatus.State = types.UpdateStateUpdating
+		case swarmapi.UpdateStatus_PAUSED:
+			service.UpdateStatus.State = types.UpdateStatePaused
+		case swarmapi.UpdateStatus_COMPLETED:
+			service.UpdateStatus.State = types.UpdateStateCompleted
+		}
+
+		service.UpdateStatus.StartedAt, _ = ptypes.Timestamp(s.UpdateStatus.StartedAt)
+		service.UpdateStatus.CompletedAt, _ = ptypes.Timestamp(s.UpdateStatus.CompletedAt)
+		service.UpdateStatus.Message = s.UpdateStatus.Message
+	}
+
 	return service
 }
 
@@ -111,9 +135,19 @@ func ServiceSpecToGRPC(s types.ServiceSpec) (swarmapi.ServiceSpec, error) {
 	}
 
 	if s.UpdateConfig != nil {
+		var failureAction swarmapi.UpdateConfig_FailureAction
+		switch s.UpdateConfig.FailureAction {
+		case types.UpdateFailureActionPause, "":
+			failureAction = swarmapi.UpdateConfig_PAUSE
+		case types.UpdateFailureActionContinue:
+			failureAction = swarmapi.UpdateConfig_CONTINUE
+		default:
+			return swarmapi.ServiceSpec{}, fmt.Errorf("unrecongized update failure action %s", s.UpdateConfig.FailureAction)
+		}
 		spec.Update = &swarmapi.UpdateConfig{
 			Parallelism:   s.UpdateConfig.Parallelism,
 			Delay:         *ptypes.DurationProto(s.UpdateConfig.Delay),
+			FailureAction: failureAction,
 		}
 	}
 

docs/reference/api/docker_remote_api_v1.24.md

@@ -3966,7 +3966,8 @@ Create a service
       },
       "UpdateConfig": {
         "Delay": 30000000000.0,
-        "Parallelism": 2
+        "Parallelism": 2,
+        "FailureAction": "pause"
       },
       "EndpointSpec": {
         "Ports": [
@@ -4056,6 +4057,8 @@ JSON Parameters:
     - **Parallelism** – Maximum number of tasks to be updated in one iteration (0 means unlimited
       parallelism).
     - **Delay** – Amount of time between updates.
+    - **FailureAction** - Action to take if an updated task fails to run, or stops running during the
+      update. Values are `continue` and `pause`.
 - **Networks** – Array of network names or IDs to attach the service to.
 - **Endpoint** – Properties that can be configured to access and load balance a service.
     - **Spec** –

docs/reference/api/docker_remote_api_v1.25.md

@@ -3967,7 +3967,8 @@ Create a service
       },
       "UpdateConfig": {
         "Delay": 30000000000.0,
-        "Parallelism": 2
+        "Parallelism": 2,
+        "FailureAction": "pause"
       },
       "EndpointSpec": {
         "Ports": [
@@ -4057,6 +4058,8 @@ JSON Parameters:
     - **Parallelism** – Maximum number of tasks to be updated in one iteration (0 means unlimited
       parallelism).
     - **Delay** – Amount of time between updates.
+    - **FailureAction** - Action to take if an updated task fails to run, or stops running during the
+      update. Values are `continue` and `pause`.
 - **Networks** – Array of network names or IDs to attach the service to.
 - **Endpoint** – Properties that can be configured to access and load balance a service.
     - **Spec** –

docs/reference/commandline/service_create.md

@@ -40,6 +40,7 @@ Options:
       --restart-window value           Window used to evaluate the restart policy (default none)
       --stop-grace-period value        Time to wait before force killing a container (default none)
       --update-delay duration          Delay between updates
+      --update-failure-action string   Action on update failure (pause|continue) (default "pause")
       --update-parallelism uint        Maximum number of tasks updated simultaneously (0 to update all at once) (default 1)
   -u, --user string                    Username or UID
       --with-registry-auth             Send registry authentication details to Swarm agents

docs/reference/commandline/service_update.md

@@ -47,6 +47,7 @@ Options:
       --restart-window value           Window used to evaluate the restart policy (default none)
       --stop-grace-period value        Time to wait before force killing a container (default none)
       --update-delay duration          Delay between updates
+      --update-failure-action string   Action on update failure (pause|continue) (default "pause")
       --update-parallelism uint        Maximum number of tasks updated simultaneously (0 to update all at once) (default 1)
   -u, --user string                    Username or UID
       --with-registry-auth             Send registry authentication details to Swarm agents

hack/vendor.sh

@@ -60,7 +60,7 @@ clone git golang.org/x/net 2beffdc2e92c8a3027590f898fe88f69af48a3f8 https://gith
 clone git golang.org/x/sys eb2c74142fd19a79b3f237334c7384d5167b1b46 https://github.com/golang/sys.git
 clone git github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3
 clone git github.com/docker/go-connections fa2850ff103453a9ad190da0df0af134f0314b3d
-clone git github.com/docker/engine-api ebb728a1346926edc2ad9418f9b6045901810b20
+clone git github.com/docker/engine-api 53b6b19ee622c8584c28fdde0e3893383b290da3
 clone git github.com/RackSec/srslog 259aed10dfa74ea2961eddd1d9847619f6e98837
 clone git github.com/imdario/mergo 0.2.1
 

integration-cli/docker_api_swarm_test.go

@@ -793,6 +793,7 @@ func serviceForUpdate(s *swarm.Service) {
 		UpdateConfig: &swarm.UpdateConfig{
 			Parallelism:   2,
 			Delay:         8 * time.Second,
+			FailureAction: swarm.UpdateFailureActionContinue,
 		},
 	}
 	s.Spec.Name = "updatetest"

vendor/src/github.com/docker/engine-api/types/swarm/service.go

@@ -8,6 +8,7 @@ type Service struct {
 	Meta
 	Spec         ServiceSpec  `json:",omitempty"`
 	Endpoint     Endpoint     `json:",omitempty"`
+	UpdateStatus UpdateStatus `json:",omitempty"`
 }
 
 // ServiceSpec represents the spec of a service.
@@ -29,6 +30,26 @@ type ServiceMode struct {
 	Global     *GlobalService     `json:",omitempty"`
 }
 
+// UpdateState is the state of a service update.
+type UpdateState string
+
+const (
+	// UpdateStateUpdating is the updating state.
+	UpdateStateUpdating UpdateState = "updating"
+	// UpdateStatePaused is the paused state.
+	UpdateStatePaused UpdateState = "paused"
+	// UpdateStateCompleted is the completed state.
+	UpdateStateCompleted UpdateState = "completed"
+)
+
+// UpdateStatus reports the status of a service update.
+type UpdateStatus struct {
+	State       UpdateState `json:",omitempty"`
+	StartedAt   time.Time   `json:",omitempty"`
+	CompletedAt time.Time   `json:",omitempty"`
+	Message     string      `json:",omitempty"`
+}
+
 // ReplicatedService is a kind of ServiceMode.
 type ReplicatedService struct {
 	Replicas *uint64 `json:",omitempty"`
@@ -37,8 +58,16 @@ type ReplicatedService struct {
 // GlobalService is a kind of ServiceMode.
 type GlobalService struct{}
 
+const (
+	// UpdateFailureActionPause PAUSE
+	UpdateFailureActionPause = "pause"
+	// UpdateFailureActionContinue CONTINUE
+	UpdateFailureActionContinue = "continue"
+)
+
 // UpdateConfig represents the update configuration.
 type UpdateConfig struct {
 	Parallelism   uint64        `json:",omitempty"`
 	Delay         time.Duration `json:",omitempty"`
+	FailureAction string        `json:",omitempty"`
 }