2016-06-13 22:52:49 -04:00
package cluster
2016-11-16 17:17:18 -05:00
//
// ## Swarmkit integration
//
// Cluster - static configurable object for accessing everything swarm related.
// Contains methods for connecting and controlling the cluster. Exists always,
// even if swarm mode is not enabled.
//
// NodeRunner - Manager for starting the swarmkit node. Is present only and
// always if swarm mode is enabled. Implements backoff restart loop in case of
// errors.
//
// NodeState - Information about the current node status including access to
// gRPC clients if a manager is active.
//
// ### Locking
//
// `cluster.controlMutex` - taken for the whole lifecycle of the processes that
// can reconfigure cluster(init/join/leave etc). Protects that one
// reconfiguration action has fully completed before another can start.
//
// `cluster.mu` - taken when the actual changes in cluster configurations
// happen. Different from `controlMutex` because in some cases we need to
// access current cluster state even if the long-running reconfiguration is
// going on. For example network stack may ask for the current cluster state in
// the middle of the shutdown. Any time current cluster state is asked you
// should take the read lock of `cluster.mu`. If you are writing an API
// responder that returns synchronously, hold `cluster.mu.RLock()` for the
// duration of the whole handler function. That ensures that node will not be
// shut down until the handler has finished.
//
// NodeRunner implements its internal locks that should not be used outside of
// the struct. Instead, you should just call `nodeRunner.State()` method to get
// the current state of the cluster(still need `cluster.mu.RLock()` to access
// `cluster.nr` reference itself). Most of the changes in NodeRunner happen
// because of an external event(network problem, unexpected swarmkit error) and
// Docker shouldn't take any locks that delay these changes from happening.
//
2016-06-13 22:52:49 -04:00
import (
"fmt"
2016-06-30 21:07:35 -04:00
"net"
2016-06-13 22:52:49 -04:00
"os"
"path/filepath"
"sync"
"time"
"github.com/Sirupsen/logrus"
2016-08-23 19:50:15 -04:00
"github.com/docker/docker/api/types/network"
2016-09-06 14:18:12 -04:00
types "github.com/docker/docker/api/types/swarm"
2016-06-13 22:52:49 -04:00
executorpkg "github.com/docker/docker/daemon/cluster/executor"
2016-08-30 17:17:32 -04:00
"github.com/docker/docker/pkg/signal"
2016-06-13 22:52:49 -04:00
swarmapi "github.com/docker/swarmkit/api"
2016-10-20 14:26:04 -04:00
swarmnode "github.com/docker/swarmkit/node"
2016-10-21 21:07:55 -04:00
"github.com/pkg/errors"
2016-06-13 22:52:49 -04:00
"golang.org/x/net/context"
)
const swarmDirName = "swarm"
const controlSocket = "control.sock"
2016-06-16 12:42:22 -04:00
const swarmConnectTimeout = 20 * time . Second
2016-07-15 13:58:21 -04:00
const swarmRequestTimeout = 20 * time . Second
2016-06-13 22:52:49 -04:00
const stateFile = "docker-state.json"
2016-06-21 17:27:04 -04:00
const defaultAddr = "0.0.0.0:2377"
2016-06-13 22:52:49 -04:00
const (
initialReconnectDelay = 100 * time . Millisecond
2016-06-16 12:42:22 -04:00
maxReconnectDelay = 30 * time . Second
2016-10-26 04:17:31 -04:00
contextPrefix = "com.docker.swarm"
2016-06-13 22:52:49 -04:00
)
2016-12-02 04:14:32 -05:00
// errNoSwarm is returned on leaving a cluster that was never initialized
2016-12-25 01:37:31 -05:00
var errNoSwarm = errors . New ( "This node is not part of a swarm" )
2016-06-13 22:52:49 -04:00
2016-12-02 04:14:32 -05:00
// errSwarmExists is returned on initialize or join request for a cluster that has already been activated
2016-12-25 01:37:31 -05:00
var errSwarmExists = errors . New ( "This node is already part of a swarm. Use \"docker swarm leave\" to leave this swarm and join another one." )
2016-06-16 12:42:22 -04:00
2016-12-02 04:14:32 -05:00
// errSwarmJoinTimeoutReached is returned when cluster join could not complete before timeout was reached.
2016-12-25 01:37:31 -05:00
var errSwarmJoinTimeoutReached = errors . New ( "Timeout was reached before node was joined. The attempt to join the swarm will continue in the background. Use the \"docker info\" command to see the current swarm status of your node." )
2016-06-13 22:52:49 -04:00
2016-12-02 04:14:32 -05:00
// errSwarmLocked is returned if the swarm is encrypted and needs a key to unlock it.
2016-12-25 01:37:31 -05:00
var errSwarmLocked = errors . New ( "Swarm is encrypted and needs to be unlocked before it can be used. Please use \"docker swarm unlock\" to unlock it." )
2016-10-21 21:07:55 -04:00
2016-12-02 04:14:32 -05:00
// errSwarmCertificatesExpired is returned if docker was not started for the whole validity period and they had no chance to renew automatically.
var errSwarmCertificatesExpired = errors . New ( "Swarm certificates have expired. To replace them, leave the swarm and join again." )
2016-11-08 21:03:47 -05:00
2016-06-30 21:07:35 -04:00
// NetworkSubnetsProvider exposes functions for retrieving the subnets
// of networks managed by Docker, so they can be filtered.
type NetworkSubnetsProvider interface {
2017-02-28 04:51:40 -05:00
Subnets ( ) ( [ ] net . IPNet , [ ] net . IPNet )
2016-06-13 22:52:49 -04:00
}
// Config provides values for Cluster.
type Config struct {
2016-06-30 21:07:35 -04:00
Root string
Name string
Backend executorpkg . Backend
NetworkSubnetsProvider NetworkSubnetsProvider
// DefaultAdvertiseAddr is the default host/IP or network interface to use
// if no AdvertiseAddr value is specified.
DefaultAdvertiseAddr string
2016-08-19 16:06:28 -04:00
// path to store runtime state, such as the swarm control socket
RuntimeRoot string
2016-06-13 22:52:49 -04:00
}
2016-06-24 14:52:28 -04:00
// Cluster provides capabilities to participate in a cluster as a worker or a
// manager.
2016-06-13 22:52:49 -04:00
type Cluster struct {
2016-11-16 17:17:18 -05:00
mu sync . RWMutex
controlMutex sync . RWMutex // protect init/join/leave user operations
nr * nodeRunner
root string
runtimeRoot string
config Config
configEvent chan struct { } // todo: make this array and goroutine safe
attachers map [ string ] * attacher
2016-08-23 19:50:15 -04:00
}
// attacher manages the in-memory attachment state of a container
// attachment to a global scope network managed by swarm manager. It
// helps in identifying the attachment ID via the taskID and the
// corresponding attachment configuration obtained from the manager.
type attacher struct {
2016-09-09 12:55:57 -04:00
taskID string
config * network . NetworkingConfig
2017-03-24 09:43:23 -04:00
inProgress bool
2016-09-09 12:55:57 -04:00
attachWaitCh chan * network . NetworkingConfig
attachCompleteCh chan struct { }
detachWaitCh chan struct { }
2016-06-20 19:35:33 -04:00
}
2016-06-13 22:52:49 -04:00
// New creates a new Cluster instance using provided config.
func New ( config Config ) ( * Cluster , error ) {
root := filepath . Join ( config . Root , swarmDirName )
if err := os . MkdirAll ( root , 0700 ) ; err != nil {
return nil , err
}
2016-08-19 16:06:28 -04:00
if config . RuntimeRoot == "" {
config . RuntimeRoot = root
}
if err := os . MkdirAll ( config . RuntimeRoot , 0700 ) ; err != nil {
return nil , err
}
2016-06-13 22:52:49 -04:00
c := & Cluster {
2016-06-20 19:35:33 -04:00
root : root ,
config : config ,
configEvent : make ( chan struct { } , 10 ) ,
2016-08-19 16:06:28 -04:00
runtimeRoot : config . RuntimeRoot ,
2016-08-23 19:50:15 -04:00
attachers : make ( map [ string ] * attacher ) ,
2016-06-13 22:52:49 -04:00
}
2016-11-16 17:17:18 -05:00
nodeConfig , err := loadPersistentState ( root )
2016-06-13 22:52:49 -04:00
if err != nil {
if os . IsNotExist ( err ) {
return c , nil
}
return nil , err
}
2016-11-16 17:17:18 -05:00
nr , err := c . newNodeRunner ( * nodeConfig )
2016-06-13 22:52:49 -04:00
if err != nil {
return nil , err
}
2016-11-16 17:17:18 -05:00
c . nr = nr
2016-06-13 22:52:49 -04:00
select {
case <- time . After ( swarmConnectTimeout ) :
2016-11-01 00:05:01 -04:00
logrus . Error ( "swarm component could not be started before timeout was reached" )
2016-11-16 17:17:18 -05:00
case err := <- nr . Ready ( ) :
2016-06-13 22:52:49 -04:00
if err != nil {
2017-03-07 19:50:39 -05:00
logrus . WithError ( err ) . Error ( "swarm component could not be started" )
return c , nil
2016-06-13 22:52:49 -04:00
}
}
2016-11-16 17:17:18 -05:00
return c , nil
2016-06-13 22:52:49 -04:00
}
2016-11-16 17:17:18 -05:00
func ( c * Cluster ) newNodeRunner ( conf nodeStartConfig ) ( * nodeRunner , error ) {
2016-06-14 12:13:53 -04:00
if err := c . config . Backend . IsSwarmCompatible ( ) ; err != nil {
2016-06-20 19:35:33 -04:00
return nil , err
2016-06-13 22:52:49 -04:00
}
2016-06-30 21:07:35 -04:00
2016-10-21 16:31:45 -04:00
actualLocalAddr := conf . LocalAddr
2016-06-30 21:07:35 -04:00
if actualLocalAddr == "" {
// If localAddr was not specified, resolve it automatically
// based on the route to joinAddr. localAddr can only be left
// empty on "join".
2016-10-21 16:31:45 -04:00
listenHost , _ , err := net . SplitHostPort ( conf . ListenAddr )
2016-06-30 21:07:35 -04:00
if err != nil {
return nil , fmt . Errorf ( "could not parse listen address: %v" , err )
}
listenAddrIP := net . ParseIP ( listenHost )
if listenAddrIP == nil || ! listenAddrIP . IsUnspecified ( ) {
actualLocalAddr = listenHost
} else {
2016-10-21 16:31:45 -04:00
if conf . RemoteAddr == "" {
2016-06-30 21:07:35 -04:00
// Should never happen except using swarms created by
// old versions that didn't save remoteAddr.
2016-10-21 16:31:45 -04:00
conf . RemoteAddr = "8.8.8.8:53"
2016-06-30 21:07:35 -04:00
}
2016-10-21 16:31:45 -04:00
conn , err := net . Dial ( "udp" , conf . RemoteAddr )
2016-06-30 21:07:35 -04:00
if err != nil {
return nil , fmt . Errorf ( "could not find local IP address: %v" , err )
}
localHostPort := conn . LocalAddr ( ) . String ( )
actualLocalAddr , _ , _ = net . SplitHostPort ( localHostPort )
conn . Close ( )
}
}
2016-11-16 17:17:18 -05:00
nr := & nodeRunner { cluster : c }
nr . actualLocalAddr = actualLocalAddr
2016-10-21 21:07:55 -04:00
2016-11-16 17:17:18 -05:00
if err := nr . Start ( conf ) ; err != nil {
2016-06-20 19:35:33 -04:00
return nil , err
2016-06-13 22:52:49 -04:00
}
2016-06-30 21:07:35 -04:00
2017-01-13 23:14:03 -05:00
c . config . Backend . DaemonJoinsCluster ( c )
2016-06-13 22:52:49 -04:00
2016-11-16 17:17:18 -05:00
return nr , nil
2016-06-13 22:52:49 -04:00
}
2016-07-15 13:58:21 -04:00
func ( c * Cluster ) getRequestContext ( ) ( context . Context , func ( ) ) { // TODO: not needed when requests don't block on qourum lost
return context . WithTimeout ( context . Background ( ) , swarmRequestTimeout )
2016-06-13 22:52:49 -04:00
}
2016-06-24 14:52:28 -04:00
// IsManager returns true if Cluster is participating as a manager.
2016-06-13 22:52:49 -04:00
func ( c * Cluster ) IsManager ( ) bool {
2016-11-16 17:17:18 -05:00
c . mu . RLock ( )
defer c . mu . RUnlock ( )
return c . currentNodeState ( ) . IsActiveManager ( )
2016-06-13 22:52:49 -04:00
}
2016-06-24 14:52:28 -04:00
// IsAgent returns true if Cluster is participating as a worker/agent.
2016-06-13 22:52:49 -04:00
func ( c * Cluster ) IsAgent ( ) bool {
2016-11-16 17:17:18 -05:00
c . mu . RLock ( )
defer c . mu . RUnlock ( )
return c . currentNodeState ( ) . status == types . LocalNodeStateActive
2016-06-13 22:52:49 -04:00
}
2016-06-30 21:07:35 -04:00
// GetLocalAddress returns the local address.
func ( c * Cluster ) GetLocalAddress ( ) string {
2016-11-16 17:17:18 -05:00
c . mu . RLock ( )
defer c . mu . RUnlock ( )
return c . currentNodeState ( ) . actualLocalAddr
2016-06-30 21:07:35 -04:00
}
2016-09-22 21:43:54 -04:00
// GetListenAddress returns the listen address.
func ( c * Cluster ) GetListenAddress ( ) string {
2016-11-16 17:17:18 -05:00
c . mu . RLock ( )
defer c . mu . RUnlock ( )
if c . nr != nil {
return c . nr . config . ListenAddr
2016-10-21 16:31:45 -04:00
}
return ""
2016-09-22 21:43:54 -04:00
}
2016-06-30 21:07:35 -04:00
// GetAdvertiseAddress returns the remotely reachable address of this node.
func ( c * Cluster ) GetAdvertiseAddress ( ) string {
2016-11-16 17:17:18 -05:00
c . mu . RLock ( )
defer c . mu . RUnlock ( )
if c . nr != nil && c . nr . config . AdvertiseAddr != "" {
advertiseHost , _ , _ := net . SplitHostPort ( c . nr . config . AdvertiseAddr )
2016-06-30 21:07:35 -04:00
return advertiseHost
2016-06-13 22:52:49 -04:00
}
2016-11-16 17:17:18 -05:00
return c . currentNodeState ( ) . actualLocalAddr
2016-06-13 22:52:49 -04:00
}
2016-06-24 14:52:28 -04:00
// GetRemoteAddress returns a known advertise address of a remote manager if
2016-06-13 22:52:49 -04:00
// available.
// todo: change to array/connect with info
func ( c * Cluster ) GetRemoteAddress ( ) string {
2016-11-16 17:17:18 -05:00
c . mu . RLock ( )
defer c . mu . RUnlock ( )
2016-06-13 22:52:49 -04:00
return c . getRemoteAddress ( )
}
func ( c * Cluster ) getRemoteAddress ( ) string {
2016-11-16 17:17:18 -05:00
state := c . currentNodeState ( )
if state . swarmNode == nil {
2016-06-13 22:52:49 -04:00
return ""
}
2016-11-16 17:17:18 -05:00
nodeID := state . swarmNode . NodeID ( )
for _ , r := range state . swarmNode . Remotes ( ) {
2016-06-13 22:52:49 -04:00
if r . NodeID != nodeID {
return r . Addr
}
}
return ""
}
// ListenClusterEvents returns a channel that receives messages on cluster
// participation changes.
// todo: make cancelable and accessible to multiple callers
func ( c * Cluster ) ListenClusterEvents ( ) <- chan struct { } {
return c . configEvent
}
2016-11-16 17:17:18 -05:00
// currentNodeState should not be called without a read lock
func ( c * Cluster ) currentNodeState ( ) nodeState {
return c . nr . State ( )
2016-11-08 21:03:47 -05:00
}
2016-06-23 16:52:41 -04:00
// errNoManager returns error describing why manager commands can't be used.
// Call with read lock.
2016-11-16 17:17:18 -05:00
func ( c * Cluster ) errNoManager ( st nodeState ) error {
if st . swarmNode == nil {
2016-12-02 04:14:32 -05:00
if errors . Cause ( st . err ) == errSwarmLocked {
return errSwarmLocked
2016-10-21 21:07:55 -04:00
}
2016-12-02 04:14:32 -05:00
if st . err == errSwarmCertificatesExpired {
return errSwarmCertificatesExpired
2016-11-08 21:03:47 -05:00
}
2016-12-25 01:37:31 -05:00
return errors . New ( "This node is not a swarm manager. Use \"docker swarm init\" or \"docker swarm join\" to connect this node to swarm and try again." )
2016-06-23 16:52:41 -04:00
}
2016-11-16 17:17:18 -05:00
if st . swarmNode . Manager ( ) != nil {
2016-12-25 01:37:31 -05:00
return errors . New ( "This node is not a swarm manager. Manager is being prepared or has trouble connecting to the cluster." )
2016-06-23 16:52:41 -04:00
}
2016-12-25 01:37:31 -05:00
return errors . New ( "This node is not a swarm manager. Worker nodes can't be used to view or modify cluster state. Please run this command on a manager node or promote the current node to a manager." )
2016-06-23 16:52:41 -04:00
}
2016-06-13 22:52:49 -04:00
// Cleanup stops active swarm node. This is run before daemon shutdown.
func ( c * Cluster ) Cleanup ( ) {
2016-11-16 17:17:18 -05:00
c . controlMutex . Lock ( )
defer c . controlMutex . Unlock ( )
c . mu . Lock ( )
node := c . nr
2016-06-13 22:52:49 -04:00
if node == nil {
2016-11-16 17:17:18 -05:00
c . mu . Unlock ( )
2016-06-13 22:52:49 -04:00
return
}
2016-11-16 17:17:18 -05:00
state := c . currentNodeState ( )
2017-04-07 21:27:35 -04:00
c . mu . Unlock ( )
2016-11-16 17:17:18 -05:00
if state . IsActiveManager ( ) {
active , reachable , unreachable , err := managerStats ( state . controlClient , state . NodeID ( ) )
2016-06-13 22:52:49 -04:00
if err == nil {
2016-08-19 16:49:58 -04:00
singlenode := active && isLastManager ( reachable , unreachable )
if active && ! singlenode && removingManagerCausesLossOfQuorum ( reachable , unreachable ) {
2016-06-13 22:52:49 -04:00
logrus . Errorf ( "Leaving cluster with %v managers left out of %v. Raft quorum will be lost." , reachable - 1 , reachable + unreachable )
}
}
}
2017-04-07 21:27:35 -04:00
2016-11-16 17:17:18 -05:00
if err := node . Stop ( ) ; err != nil {
logrus . Errorf ( "failed to shut down cluster node: %v" , err )
signal . DumpStacks ( "" )
}
2017-04-07 21:27:35 -04:00
c . mu . Lock ( )
2016-11-16 17:17:18 -05:00
c . nr = nil
2017-04-07 21:27:35 -04:00
c . mu . Unlock ( )
2016-06-13 22:52:49 -04:00
}
2016-11-16 17:17:18 -05:00
func managerStats ( client swarmapi . ControlClient , currentNodeID string ) ( current bool , reachable int , unreachable int , err error ) {
2016-07-15 13:58:21 -04:00
ctx , cancel := context . WithTimeout ( context . Background ( ) , 5 * time . Second )
defer cancel ( )
2016-11-16 17:17:18 -05:00
nodes , err := client . ListNodes ( ctx , & swarmapi . ListNodesRequest { } )
2016-06-13 22:52:49 -04:00
if err != nil {
return false , 0 , 0 , err
}
for _ , n := range nodes . Nodes {
if n . ManagerStatus != nil {
2016-06-14 20:23:01 -04:00
if n . ManagerStatus . Reachability == swarmapi . RaftMemberStatus_REACHABLE {
2016-06-13 22:52:49 -04:00
reachable ++
2016-11-16 17:17:18 -05:00
if n . ID == currentNodeID {
2016-06-13 22:52:49 -04:00
current = true
}
}
2016-06-14 20:23:01 -04:00
if n . ManagerStatus . Reachability == swarmapi . RaftMemberStatus_UNREACHABLE {
2016-06-13 22:52:49 -04:00
unreachable ++
}
}
}
return
}
2016-10-21 21:07:55 -04:00
func detectLockedError ( err error ) error {
2016-10-27 21:50:49 -04:00
if err == swarmnode . ErrInvalidUnlockKey {
2016-12-02 04:14:32 -05:00
return errors . WithStack ( errSwarmLocked )
2016-10-21 21:07:55 -04:00
}
return err
}
2017-02-28 05:12:11 -05:00
func ( c * Cluster ) lockedManagerAction ( fn func ( ctx context . Context , state nodeState ) error ) error {
c . mu . RLock ( )
defer c . mu . RUnlock ( )
state := c . currentNodeState ( )
if ! state . IsActiveManager ( ) {
return c . errNoManager ( state )
}
ctx , cancel := c . getRequestContext ( )
defer cancel ( )
return fn ( ctx , state )
}