From 6f4bb796ddb82d04a1c379a950db93bc61c64c04 Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Fri, 31 Mar 2017 14:07:55 -0700 Subject: [PATCH] Daemon to take care of ingress cleanup on leave & shutdown Signed-off-by: Alessandro Boch --- daemon/cluster/executor/backend.go | 4 ++-- daemon/cluster/executor/container/executor.go | 4 +++- daemon/daemon.go | 24 +++++++++++++++++++ daemon/network.go | 24 ++++++++++++------- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/daemon/cluster/executor/backend.go b/daemon/cluster/executor/backend.go index 5fe953ac05..ee7a367fb8 100644 --- a/daemon/cluster/executor/backend.go +++ b/daemon/cluster/executor/backend.go @@ -27,8 +27,8 @@ type Backend interface { CreateManagedNetwork(clustertypes.NetworkCreateRequest) error DeleteManagedNetwork(name string) error FindNetwork(idName string) (libnetwork.Network, error) - SetupIngress(req clustertypes.NetworkCreateRequest, nodeIP string) error - ReleaseIngress() error + SetupIngress(clustertypes.NetworkCreateRequest, string) (<-chan struct{}, error) + ReleaseIngress() (<-chan struct{}, error) PullImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error CreateManagedContainer(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error) ContainerStart(name string, hostConfig *container.HostConfig, checkpoint string, checkpointDir string) error diff --git a/daemon/cluster/executor/container/executor.go b/daemon/cluster/executor/container/executor.go index 4af8bc8f10..6be0f3156c 100644 --- a/daemon/cluster/executor/container/executor.go +++ b/daemon/cluster/executor/container/executor.go @@ -139,13 +139,15 @@ func (e *executor) Configure(ctx context.Context, node *api.Node) error { options.IPAM.Config = append(options.IPAM.Config, c) } - return e.backend.SetupIngress(clustertypes.NetworkCreateRequest{ + _, err := e.backend.SetupIngress(clustertypes.NetworkCreateRequest{ ID: na.Network.ID, NetworkCreateRequest: types.NetworkCreateRequest{ Name: na.Network.Spec.Annotations.Name, NetworkCreate: options, }, }, na.Addresses[0]) + + return err } // Controller returns a docker container runner. diff --git a/daemon/daemon.go b/daemon/daemon.go index 27a79e61a4..778890377a 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -445,7 +445,25 @@ func (daemon *Daemon) DaemonLeavesCluster() { // Daemon is in charge of removing the attachable networks with // connected containers when the node leaves the swarm daemon.clearAttachableNetworks() + // We no longer need the cluster provider, stop it now so that + // the network agent will stop listening to cluster events. daemon.setClusterProvider(nil) + // Wait for the networking cluster agent to stop + daemon.netController.AgentStopWait() + // Daemon is in charge of removing the ingress network when the + // node leaves the swarm. Wait for job to be done or timeout. + // This is called also on graceful daemon shutdown. We need to + // wait, because the ingress release has to happen before the + // network controller is stopped. + if done, err := daemon.ReleaseIngress(); err == nil { + select { + case <-done: + case <-time.After(5 * time.Second): + logrus.Warnf("timeout while waiting for ingress network removal") + } + } else { + logrus.Warnf("failed to initiate ingress network removal: %v", err) + } } // setClusterProvider sets a component for querying the current cluster state. @@ -832,6 +850,12 @@ func (daemon *Daemon) Shutdown() error { } } + // If we are part of a cluster, clean up cluster's stuff + if daemon.clusterProvider != nil { + logrus.Debugf("start clean shutdown of cluster resources...") + daemon.DaemonLeavesCluster() + } + // Shutdown plugins after containers and layerstore. Don't change the order. daemon.pluginShutdown() diff --git a/daemon/network.go b/daemon/network.go index d72fbb6c57..06d3b3eb87 100644 --- a/daemon/network.go +++ b/daemon/network.go @@ -101,8 +101,9 @@ func (daemon *Daemon) getAllNetworks() []libnetwork.Network { } type ingressJob struct { - create *clustertypes.NetworkCreateRequest - ip net.IP + create *clustertypes.NetworkCreateRequest + ip net.IP + jobDone chan struct{} } var ( @@ -124,6 +125,7 @@ func (daemon *Daemon) startIngressWorker() { daemon.releaseIngress(ingressID) ingressID = "" } + close(r.jobDone) } } }() @@ -137,19 +139,23 @@ func (daemon *Daemon) enqueueIngressJob(job *ingressJob) { } // SetupIngress setups ingress networking. -func (daemon *Daemon) SetupIngress(create clustertypes.NetworkCreateRequest, nodeIP string) error { +// The function returns a channel which will signal the caller when the programming is completed. +func (daemon *Daemon) SetupIngress(create clustertypes.NetworkCreateRequest, nodeIP string) (<-chan struct{}, error) { ip, _, err := net.ParseCIDR(nodeIP) if err != nil { - return err + return nil, err } - daemon.enqueueIngressJob(&ingressJob{&create, ip}) - return nil + done := make(chan struct{}) + daemon.enqueueIngressJob(&ingressJob{&create, ip, done}) + return done, nil } // ReleaseIngress releases the ingress networking. -func (daemon *Daemon) ReleaseIngress() error { - daemon.enqueueIngressJob(&ingressJob{nil, nil}) - return nil +// The function returns a channel which will signal the caller when the programming is completed. +func (daemon *Daemon) ReleaseIngress() (<-chan struct{}, error) { + done := make(chan struct{}) + daemon.enqueueIngressJob(&ingressJob{nil, nil, done}) + return done, nil } func (daemon *Daemon) setupIngress(create *clustertypes.NetworkCreateRequest, ip net.IP, staleID string) {