From b96988feb928d295569f82fd0eaed5c16efad41d Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Fri, 14 Apr 2017 16:49:39 -0700 Subject: [PATCH 1/2] Libnetwork vendoring Vendor libnetwork changes for --data-path-addr flag Signed-off-by: Flavio Crisciani --- vendor.conf | 2 +- vendor/github.com/docker/libnetwork/agent.go | 45 +++++++++++++------ .../docker/libnetwork/cluster/provider.go | 1 + .../docker/libnetwork/default_gateway.go | 13 +++++- .../github.com/docker/libnetwork/endpoint.go | 22 ++++----- .../docker/libnetwork/networkdb/broadcast.go | 9 ++++ .../docker/libnetwork/networkdb/cluster.go | 2 +- 7 files changed, 66 insertions(+), 28 deletions(-) diff --git a/vendor.conf b/vendor.conf index 32b3d9fb46..5bab46244d 100644 --- a/vendor.conf +++ b/vendor.conf @@ -25,7 +25,7 @@ github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5 github.com/imdario/mergo 0.2.1 #get libnetwork packages -github.com/docker/libnetwork b13e0604016a4944025aaff521d9c125850b0d04 +github.com/docker/libnetwork 5dc95a3f9ce4b70bf08492ca37ec55c5b6d84975 github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec diff --git a/vendor/github.com/docker/libnetwork/agent.go b/vendor/github.com/docker/libnetwork/agent.go index ff1869055f..785e731c6e 100644 --- a/vendor/github.com/docker/libnetwork/agent.go +++ b/vendor/github.com/docker/libnetwork/agent.go @@ -39,11 +39,21 @@ type agent struct { networkDB *networkdb.NetworkDB bindAddr string advertiseAddr string + dataPathAddr string epTblCancel func() driverCancelFuncs map[string][]func() sync.Mutex } +func (a *agent) dataPathAddress() string { + a.Lock() + defer a.Unlock() + if a.dataPathAddr != "" { + return a.dataPathAddr + } + return a.advertiseAddr +} + const libnetworkEPTable = "endpoint_table" func getBindAddr(ifaceName string) (string, error) { @@ -187,16 +197,25 @@ func (c *controller) agentSetup() error { clusterProvider := c.cfg.Daemon.ClusterProvider agent := c.agent c.Unlock() + + if clusterProvider == nil { + msg := "Aborting initialization of Libnetwork Agent because cluster provider is now unset" + logrus.Errorf(msg) + return fmt.Errorf(msg) + } + bindAddr := clusterProvider.GetLocalAddress() advAddr := clusterProvider.GetAdvertiseAddress() + dataAddr := clusterProvider.GetDataPathAddress() remote := clusterProvider.GetRemoteAddress() remoteAddr, _, _ := net.SplitHostPort(remote) listen := clusterProvider.GetListenAddress() listenAddr, _, _ := net.SplitHostPort(listen) - logrus.Infof("Initializing Libnetwork Agent Listen-Addr=%s Local-addr=%s Adv-addr=%s Remote-addr =%s", listenAddr, bindAddr, advAddr, remoteAddr) + logrus.Infof("Initializing Libnetwork Agent Listen-Addr=%s Local-addr=%s Adv-addr=%s Data-addr=%s Remote-addr=%s", + listenAddr, bindAddr, advAddr, dataAddr, remoteAddr) if advAddr != "" && agent == nil { - if err := c.agentInit(listenAddr, bindAddr, advAddr); err != nil { + if err := c.agentInit(listenAddr, bindAddr, advAddr, dataAddr); err != nil { logrus.Errorf("Error in agentInit : %v", err) } else { c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool { @@ -262,7 +281,7 @@ func (c *controller) getPrimaryKeyTag(subsys string) ([]byte, uint64, error) { return keys[1].Key, keys[1].LamportTime, nil } -func (c *controller) agentInit(listenAddr, bindAddrOrInterface, advertiseAddr string) error { +func (c *controller) agentInit(listenAddr, bindAddrOrInterface, advertiseAddr, dataPathAddr string) error { if !c.isAgent() { return nil } @@ -296,6 +315,7 @@ func (c *controller) agentInit(listenAddr, bindAddrOrInterface, advertiseAddr st networkDB: nDB, bindAddr: bindAddr, advertiseAddr: advertiseAddr, + dataPathAddr: dataPathAddr, epTblCancel: cancel, driverCancelFuncs: make(map[string][]func()), } @@ -336,25 +356,22 @@ func (c *controller) agentDriverNotify(d driverapi.Driver) { return } - d.DiscoverNew(discoverapi.NodeDiscovery, discoverapi.NodeDiscoveryData{ - Address: agent.advertiseAddr, + if err := d.DiscoverNew(discoverapi.NodeDiscovery, discoverapi.NodeDiscoveryData{ + Address: agent.dataPathAddress(), BindAddress: agent.bindAddr, Self: true, - }) + }); err != nil { + logrus.Warnf("Failed the node discovery in driver: %v", err) + } drvEnc := discoverapi.DriverEncryptionConfig{} keys, tags := c.getKeys(subsysIPSec) drvEnc.Keys = keys drvEnc.Tags = tags - c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool { - err := driver.DiscoverNew(discoverapi.EncryptionKeysConfig, drvEnc) - if err != nil { - logrus.Warnf("Failed to set datapath keys in driver %s: %v", name, err) - } - return false - }) - + if err := d.DiscoverNew(discoverapi.EncryptionKeysConfig, drvEnc); err != nil { + logrus.Warnf("Failed to set datapath keys in driver: %v", err) + } } func (c *controller) agentClose() { diff --git a/vendor/github.com/docker/libnetwork/cluster/provider.go b/vendor/github.com/docker/libnetwork/cluster/provider.go index 572bac85a6..24f40c56eb 100644 --- a/vendor/github.com/docker/libnetwork/cluster/provider.go +++ b/vendor/github.com/docker/libnetwork/cluster/provider.go @@ -12,6 +12,7 @@ type Provider interface { GetLocalAddress() string GetListenAddress() string GetAdvertiseAddress() string + GetDataPathAddress() string GetRemoteAddress() string ListenClusterEvents() <-chan struct{} AttachNetwork(string, string, []string) (*network.NetworkingConfig, error) diff --git a/vendor/github.com/docker/libnetwork/default_gateway.go b/vendor/github.com/docker/libnetwork/default_gateway.go index b042539b88..bf1592435f 100644 --- a/vendor/github.com/docker/libnetwork/default_gateway.go +++ b/vendor/github.com/docker/libnetwork/default_gateway.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/types" ) @@ -72,9 +73,19 @@ func (sb *sandbox) setupDefaultGW() error { if err != nil { return fmt.Errorf("container %s: endpoint create on GW Network failed: %v", sb.containerID, err) } + + defer func() { + if err != nil { + if err2 := newEp.Delete(true); err2 != nil { + logrus.Warnf("Failed to remove gw endpoint for container %s after failing to join the gateway network: %v", + sb.containerID, err2) + } + } + }() + epLocal := newEp.(*endpoint) - if err := epLocal.sbJoin(sb); err != nil { + if err = epLocal.sbJoin(sb); err != nil { return fmt.Errorf("container %s: endpoint join on GW Network failed: %v", sb.containerID, err) } diff --git a/vendor/github.com/docker/libnetwork/endpoint.go b/vendor/github.com/docker/libnetwork/endpoint.go index de63cf34e8..8e70c38d76 100644 --- a/vendor/github.com/docker/libnetwork/endpoint.go +++ b/vendor/github.com/docker/libnetwork/endpoint.go @@ -427,7 +427,7 @@ func (ep *endpoint) Join(sbox Sandbox, options ...EndpointOption) error { return ep.sbJoin(sb, options...) } -func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error { +func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) (err error) { n, err := ep.getNetworkFromStore() if err != nil { return fmt.Errorf("failed to get network from store during join: %v", err) @@ -462,7 +462,7 @@ func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error { d, err := n.driver(true) if err != nil { - return fmt.Errorf("failed to join endpoint: %v", err) + return fmt.Errorf("failed to get driver during join: %v", err) } err = d.Join(nid, epid, sb.Key(), ep, sb.Labels()) @@ -471,8 +471,8 @@ func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error { } defer func() { if err != nil { - if err := d.Leave(nid, epid); err != nil { - logrus.Warnf("driver leave failed while rolling back join: %v", err) + if e := d.Leave(nid, epid); e != nil { + logrus.Warnf("driver leave failed while rolling back join: %v", e) } } }() @@ -538,11 +538,11 @@ func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error { logrus.Debugf("Revoking external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID()) extN, err := extEp.getNetworkFromStore() if err != nil { - return fmt.Errorf("failed to get network from store during join: %v", err) + return fmt.Errorf("failed to get network from store for revoking external connectivity during join: %v", err) } extD, err := extN.driver(true) if err != nil { - return fmt.Errorf("failed to join endpoint: %v", err) + return fmt.Errorf("failed to get driver for revoking external connectivity during join: %v", err) } if err = extD.RevokeExternalConnectivity(extEp.network.ID(), extEp.ID()); err != nil { return types.InternalErrorf( @@ -570,9 +570,9 @@ func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error { } if !sb.needDefaultGW() { - if err := sb.clearDefaultGW(); err != nil { + if e := sb.clearDefaultGW(); e != nil { logrus.Warnf("Failure while disconnecting sandbox %s (%s) from gateway network: %v", - sb.ID(), sb.ContainerID(), err) + sb.ID(), sb.ContainerID(), e) } } @@ -705,7 +705,7 @@ func (ep *endpoint) sbLeave(sb *sandbox, force bool, options ...EndpointOption) d, err := n.driver(!force) if err != nil { - return fmt.Errorf("failed to leave endpoint: %v", err) + return fmt.Errorf("failed to get driver during endpoint leave: %v", err) } ep.Lock() @@ -765,11 +765,11 @@ func (ep *endpoint) sbLeave(sb *sandbox, force bool, options ...EndpointOption) logrus.Debugf("Programming external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID()) extN, err := extEp.getNetworkFromStore() if err != nil { - return fmt.Errorf("failed to get network from store during leave: %v", err) + return fmt.Errorf("failed to get network from store for programming external connectivity during leave: %v", err) } extD, err := extN.driver(true) if err != nil { - return fmt.Errorf("failed to leave endpoint: %v", err) + return fmt.Errorf("failed to get driver for programming external connectivity during leave: %v", err) } if err := extD.ProgramExternalConnectivity(extEp.network.ID(), extEp.ID(), sb.Labels()); err != nil { logrus.Warnf("driver failed programming external connectivity on endpoint %s: (%s) %v", diff --git a/vendor/github.com/docker/libnetwork/networkdb/broadcast.go b/vendor/github.com/docker/libnetwork/networkdb/broadcast.go index 5555634179..3fe9f6271a 100644 --- a/vendor/github.com/docker/libnetwork/networkdb/broadcast.go +++ b/vendor/github.com/docker/libnetwork/networkdb/broadcast.go @@ -86,6 +86,15 @@ func (nDB *NetworkDB) sendNodeEvent(event NodeEvent_Type) error { notify: notifyCh, }) + nDB.RLock() + noPeers := len(nDB.nodes) <= 1 + nDB.RUnlock() + + // Message enqueued, do not wait for a send if no peer is present + if noPeers { + return nil + } + // Wait for the broadcast select { case <-notifyCh: diff --git a/vendor/github.com/docker/libnetwork/networkdb/cluster.go b/vendor/github.com/docker/libnetwork/networkdb/cluster.go index 62d05c0aea..54c392c5a4 100644 --- a/vendor/github.com/docker/libnetwork/networkdb/cluster.go +++ b/vendor/github.com/docker/libnetwork/networkdb/cluster.go @@ -17,7 +17,7 @@ import ( ) const ( - reapInterval = 60 * time.Second + reapInterval = 30 * time.Minute reapPeriod = 5 * time.Second retryInterval = 1 * time.Second nodeReapInterval = 24 * time.Hour From 8dc8cd4719f165c01c98e7d3ce1d6cea6a8f60b8 Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Fri, 14 Apr 2017 16:54:17 -0700 Subject: [PATCH 2/2] Inroduce SWARM --data-path-addr flag This new flag will allow the configuration of an interface that can be used for data path traffic to be isolated from control plane traffic. This flag is simply percolated down to libnetwork and will be used by all the global scope drivers (today overlay) Negative test added for invalid flag arguments Signed-off-by: Flavio Crisciani --- api/types/swarm/swarm.go | 2 + cli/command/swarm/init.go | 3 + cli/command/swarm/join.go | 3 + cli/command/swarm/opts.go | 1 + contrib/completion/bash/docker | 4 +- contrib/completion/zsh/_docker | 2 + daemon/cluster/cluster.go | 10 +++ daemon/cluster/listen_addr.go | 95 +++++++++++++++--------- daemon/cluster/noderunner.go | 5 +- daemon/cluster/swarm.go | 12 +++ docs/reference/commandline/swarm_init.md | 10 +++ docs/reference/commandline/swarm_join.md | 10 +++ integration-cli/docker_cli_swarm_test.go | 12 +++ 13 files changed, 131 insertions(+), 38 deletions(-) diff --git a/api/types/swarm/swarm.go b/api/types/swarm/swarm.go index c513274750..08da338439 100644 --- a/api/types/swarm/swarm.go +++ b/api/types/swarm/swarm.go @@ -132,6 +132,7 @@ type ExternalCA struct { type InitRequest struct { ListenAddr string AdvertiseAddr string + DataPathAddr string ForceNewCluster bool Spec Spec AutoLockManagers bool @@ -142,6 +143,7 @@ type InitRequest struct { type JoinRequest struct { ListenAddr string AdvertiseAddr string + DataPathAddr string RemoteAddrs []string JoinToken string // accept by secret Availability NodeAvailability diff --git a/cli/command/swarm/init.go b/cli/command/swarm/init.go index 37d96de113..5d42b0174c 100644 --- a/cli/command/swarm/init.go +++ b/cli/command/swarm/init.go @@ -19,6 +19,7 @@ type initOptions struct { listenAddr NodeAddrOption // Not a NodeAddrOption because it has no default port. advertiseAddr string + dataPathAddr string forceNewCluster bool availability string } @@ -40,6 +41,7 @@ func newInitCommand(dockerCli command.Cli) *cobra.Command { flags := cmd.Flags() flags.Var(&opts.listenAddr, flagListenAddr, "Listen address (format: [:port])") flags.StringVar(&opts.advertiseAddr, flagAdvertiseAddr, "", "Advertised address (format: [:port])") + flags.StringVar(&opts.dataPathAddr, flagDataPathAddr, "", "Address or interface to use for data path traffic (format: )") flags.BoolVar(&opts.forceNewCluster, "force-new-cluster", false, "Force create a new cluster from current state") flags.BoolVar(&opts.autolock, flagAutolock, false, "Enable manager autolocking (requiring an unlock key to start a stopped manager)") flags.StringVar(&opts.availability, flagAvailability, "active", `Availability of the node ("active"|"pause"|"drain")`) @@ -54,6 +56,7 @@ func runInit(dockerCli command.Cli, flags *pflag.FlagSet, opts initOptions) erro req := swarm.InitRequest{ ListenAddr: opts.listenAddr.String(), AdvertiseAddr: opts.advertiseAddr, + DataPathAddr: opts.dataPathAddr, ForceNewCluster: opts.forceNewCluster, Spec: opts.swarmOptions.ToSpec(flags), AutoLockManagers: opts.swarmOptions.autolock, diff --git a/cli/command/swarm/join.go b/cli/command/swarm/join.go index 873eaaefaa..34730ab8c7 100644 --- a/cli/command/swarm/join.go +++ b/cli/command/swarm/join.go @@ -19,6 +19,7 @@ type joinOptions struct { listenAddr NodeAddrOption // Not a NodeAddrOption because it has no default port. advertiseAddr string + dataPathAddr string token string availability string } @@ -41,6 +42,7 @@ func newJoinCommand(dockerCli command.Cli) *cobra.Command { flags := cmd.Flags() flags.Var(&opts.listenAddr, flagListenAddr, "Listen address (format: [:port])") flags.StringVar(&opts.advertiseAddr, flagAdvertiseAddr, "", "Advertised address (format: [:port])") + flags.StringVar(&opts.dataPathAddr, flagDataPathAddr, "", "Address or interface to use for data path traffic (format: )") flags.StringVar(&opts.token, flagToken, "", "Token for entry into the swarm") flags.StringVar(&opts.availability, flagAvailability, "active", `Availability of the node ("active"|"pause"|"drain")`) return cmd @@ -54,6 +56,7 @@ func runJoin(dockerCli command.Cli, flags *pflag.FlagSet, opts joinOptions) erro JoinToken: opts.token, ListenAddr: opts.listenAddr.String(), AdvertiseAddr: opts.advertiseAddr, + DataPathAddr: opts.dataPathAddr, RemoteAddrs: []string{opts.remote}, } if flags.Changed(flagAvailability) { diff --git a/cli/command/swarm/opts.go b/cli/command/swarm/opts.go index 6eddddccae..dc32dc488c 100644 --- a/cli/command/swarm/opts.go +++ b/cli/command/swarm/opts.go @@ -19,6 +19,7 @@ const ( flagDispatcherHeartbeat = "dispatcher-heartbeat" flagListenAddr = "listen-addr" flagAdvertiseAddr = "advertise-addr" + flagDataPathAddr = "data-path-addr" flagQuiet = "quiet" flagRotate = "rotate" flagToken = "token" diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index 813f2b6ccc..1a0e110633 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -3301,7 +3301,7 @@ _docker_swarm_init() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--advertise-addr --autolock --availability --cert-expiry --dispatcher-heartbeat --external-ca --force-new-cluster --help --listen-addr --max-snapshots --snapshot-interval --task-history-limit" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--advertise-addr --data-path-addr --autolock --availability --cert-expiry --dispatcher-heartbeat --external-ca --force-new-cluster --help --listen-addr --max-snapshots --snapshot-interval --task-history-limit" -- "$cur" ) ) ;; esac } @@ -3337,7 +3337,7 @@ _docker_swarm_join() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--advertise-addr --availability --help --listen-addr --token" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--advertise-addr --data-path-addr --availability --help --listen-addr --token" -- "$cur" ) ) ;; *:) COMPREPLY=( $( compgen -W "2377" -- "${cur##*:}" ) ) diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index 7a3a492415..316caf5d43 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -2267,6 +2267,7 @@ __docker_swarm_subcommand() { _arguments $(__docker_arguments) \ $opts_help \ "($help)--advertise-addr=[Advertised address]:ip\:port: " \ + "($help)--data-path-addr=[Data path IP or interface]:ip " \ "($help)--autolock[Enable manager autolocking]" \ "($help)--availability=[Availability of the node]:availability:(active drain pause)" \ "($help)--cert-expiry=[Validity period for node certificates]:duration: " \ @@ -2282,6 +2283,7 @@ __docker_swarm_subcommand() { _arguments $(__docker_arguments) -A '-*' \ $opts_help \ "($help)--advertise-addr=[Advertised address]:ip\:port: " \ + "($help)--data-path-addr=[Data path IP or interface]:ip " \ "($help)--availability=[Availability of the node]:availability:(active drain pause)" \ "($help)--listen-addr=[Listen address]:ip\:port: " \ "($help)--token=[Token for entry into the swarm]:secret: " \ diff --git a/daemon/cluster/cluster.go b/daemon/cluster/cluster.go index aa622d9b1c..8ff527877f 100644 --- a/daemon/cluster/cluster.go +++ b/daemon/cluster/cluster.go @@ -270,6 +270,16 @@ func (c *Cluster) GetAdvertiseAddress() string { return c.currentNodeState().actualLocalAddr } +// GetDataPathAddress returns the address to be used for the data path traffic, if specified. +func (c *Cluster) GetDataPathAddress() string { + c.mu.RLock() + defer c.mu.RUnlock() + if c.nr != nil { + return c.nr.config.DataPathAddr + } + return "" +} + // GetRemoteAddress returns a known advertise address of a remote manager if // available. // todo: change to array/connect with info diff --git a/daemon/cluster/listen_addr.go b/daemon/cluster/listen_addr.go index 83e74ad464..993ccb62ad 100644 --- a/daemon/cluster/listen_addr.go +++ b/daemon/cluster/listen_addr.go @@ -10,8 +10,10 @@ var ( errNoSuchInterface = errors.New("no such interface") errNoIP = errors.New("could not find the system's IP address") errMustSpecifyListenAddr = errors.New("must specify a listening address because the address to advertise is not recognized as a system address, and a system's IP address to use could not be uniquely identified") + errBadNetworkIdentifier = errors.New("must specify a valid IP address or interface name") errBadListenAddr = errors.New("listen address must be an IP address or network interface (with optional port number)") errBadAdvertiseAddr = errors.New("advertise address must be a non-zero IP address or network interface (with optional port number)") + errBadDataPathAddr = errors.New("data path address must be a non-zero IP address or network interface (without a port number)") errBadDefaultAdvertiseAddr = errors.New("default advertise address must be a non-zero IP address or network interface (without a port number)") ) @@ -20,23 +22,17 @@ func resolveListenAddr(specifiedAddr string) (string, string, error) { if err != nil { return "", "", fmt.Errorf("could not parse listen address %s", specifiedAddr) } - // Does the host component match any of the interface names on the // system? If so, use the address from that interface. - interfaceAddr, err := resolveInterfaceAddr(specifiedHost) - if err == nil { - return interfaceAddr.String(), specifiedPort, nil - } - if err != errNoSuchInterface { + specifiedIP, err := resolveInputIPAddr(specifiedHost, true) + if err != nil { + if err == errBadNetworkIdentifier { + err = errBadListenAddr + } return "", "", err } - // If it's not an interface, it must be an IP (for now) - if net.ParseIP(specifiedHost) == nil { - return "", "", errBadListenAddr - } - - return specifiedHost, specifiedPort, nil + return specifiedIP.String(), specifiedPort, nil } func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (string, string, error) { @@ -57,43 +53,32 @@ func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (st advertiseHost = advertiseAddr advertisePort = listenAddrPort } - // Does the host component match any of the interface names on the // system? If so, use the address from that interface. - interfaceAddr, err := resolveInterfaceAddr(advertiseHost) - if err == nil { - return interfaceAddr.String(), advertisePort, nil - } - if err != errNoSuchInterface { + advertiseIP, err := resolveInputIPAddr(advertiseHost, false) + if err != nil { + if err == errBadNetworkIdentifier { + err = errBadAdvertiseAddr + } return "", "", err } - // If it's not an interface, it must be an IP (for now) - if ip := net.ParseIP(advertiseHost); ip == nil || ip.IsUnspecified() { - return "", "", errBadAdvertiseAddr - } - - return advertiseHost, advertisePort, nil + return advertiseIP.String(), advertisePort, nil } if c.config.DefaultAdvertiseAddr != "" { // Does the default advertise address component match any of the // interface names on the system? If so, use the address from // that interface. - interfaceAddr, err := resolveInterfaceAddr(c.config.DefaultAdvertiseAddr) - if err == nil { - return interfaceAddr.String(), listenAddrPort, nil - } - if err != errNoSuchInterface { + defaultAdvertiseIP, err := resolveInputIPAddr(c.config.DefaultAdvertiseAddr, false) + if err != nil { + if err == errBadNetworkIdentifier { + err = errBadDefaultAdvertiseAddr + } return "", "", err } - // If it's not an interface, it must be an IP (for now) - if ip := net.ParseIP(c.config.DefaultAdvertiseAddr); ip == nil || ip.IsUnspecified() { - return "", "", errBadDefaultAdvertiseAddr - } - - return c.config.DefaultAdvertiseAddr, listenAddrPort, nil + return defaultAdvertiseIP.String(), listenAddrPort, nil } systemAddr, err := c.resolveSystemAddr() @@ -103,6 +88,22 @@ func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (st return systemAddr.String(), listenAddrPort, nil } +func resolveDataPathAddr(dataPathAddr string) (string, error) { + if dataPathAddr == "" { + // dataPathAddr is not defined + return "", nil + } + // If a data path flag is specified try to resolve the IP address. + dataPathIP, err := resolveInputIPAddr(dataPathAddr, false) + if err != nil { + if err == errBadNetworkIdentifier { + err = errBadDataPathAddr + } + return "", err + } + return dataPathIP.String(), nil +} + func resolveInterfaceAddr(specifiedInterface string) (net.IP, error) { // Use a specific interface's IP address. intf, err := net.InterfaceByName(specifiedInterface) @@ -149,6 +150,30 @@ func resolveInterfaceAddr(specifiedInterface string) (net.IP, error) { return interfaceAddr6, nil } +// resolveInputIPAddr tries to resolve the IP address from the string passed as input +// - tries to match the string as an interface name, if so returns the IP address associated with it +// - on failure of previous step tries to parse the string as an IP address itself +// if succeeds returns the IP address +func resolveInputIPAddr(input string, isUnspecifiedValid bool) (net.IP, error) { + // Try to see if it is an interface name + interfaceAddr, err := resolveInterfaceAddr(input) + if err == nil { + return interfaceAddr, nil + } + // String matched interface but there is a potential ambiguity to be resolved + if err != errNoSuchInterface { + return nil, err + } + + // String is not an interface check if it is a valid IP + if ip := net.ParseIP(input); ip != nil && (isUnspecifiedValid || !ip.IsUnspecified()) { + return ip, nil + } + + // Not valid IP found + return nil, errBadNetworkIdentifier +} + func (c *Cluster) resolveSystemAddrViaSubnetCheck() (net.IP, error) { // Use the system's only IP address, or fail if there are // multiple addresses to choose from. Skip interfaces which diff --git a/daemon/cluster/noderunner.go b/daemon/cluster/noderunner.go index 49fef1fcc8..6c8ca70376 100644 --- a/daemon/cluster/noderunner.go +++ b/daemon/cluster/noderunner.go @@ -46,7 +46,10 @@ type nodeStartConfig struct { ListenAddr string // AdvertiseAddr is the address other nodes should connect to, // including a port. - AdvertiseAddr string + AdvertiseAddr string + // DataPathAddr is the address that has to be used for the data path + DataPathAddr string + joinAddr string forceNewCluster bool joinToken string diff --git a/daemon/cluster/swarm.go b/daemon/cluster/swarm.go index 6b6a54303a..8db4f3621f 100644 --- a/daemon/cluster/swarm.go +++ b/daemon/cluster/swarm.go @@ -54,6 +54,11 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) { return "", err } + dataPathAddr, err := resolveDataPathAddr(req.DataPathAddr) + if err != nil { + return "", err + } + localAddr := listenHost // If the local address is undetermined, the advertise address @@ -93,6 +98,7 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) { LocalAddr: localAddr, ListenAddr: net.JoinHostPort(listenHost, listenPort), AdvertiseAddr: net.JoinHostPort(advertiseHost, advertisePort), + DataPathAddr: dataPathAddr, availability: req.Availability, }) if err != nil { @@ -155,12 +161,18 @@ func (c *Cluster) Join(req types.JoinRequest) error { } } + dataPathAddr, err := resolveDataPathAddr(req.DataPathAddr) + if err != nil { + return err + } + clearPersistentState(c.root) nr, err := c.newNodeRunner(nodeStartConfig{ RemoteAddr: req.RemoteAddrs[0], ListenAddr: net.JoinHostPort(listenHost, listenPort), AdvertiseAddr: advertiseAddr, + DataPathAddr: dataPathAddr, joinAddr: req.RemoteAddrs[0], joinToken: req.JoinToken, availability: req.Availability, diff --git a/docs/reference/commandline/swarm_init.md b/docs/reference/commandline/swarm_init.md index f4c6348e86..0daace038f 100644 --- a/docs/reference/commandline/swarm_init.md +++ b/docs/reference/commandline/swarm_init.md @@ -25,6 +25,7 @@ Options: --autolock Enable manager autolocking (requiring an unlock key to start a stopped manager) --availability string Availability of the node ("active"|"pause"|"drain") (default "active") --cert-expiry duration Validity period for node certificates (ns|us|ms|s|m|h) (default 2160h0m0s) + --data-path-addr string Address or interface to use for data path traffic (format: ) --dispatcher-heartbeat duration Dispatcher heartbeat period (ns|us|ms|s|m|h) (default 5s) --external-ca external-ca Specifications of one or more certificate signing endpoints --force-new-cluster Force create a new cluster from current state @@ -118,6 +119,15 @@ for example `--advertise-addr eth0:2377`. Specifying a port is optional. If the value is a bare IP address or interface name, the default port 2377 will be used. +### `--data-path-addr` + +This flag specifies the address that global scope network drivers will publish towards +other nodes in order to reach the containers running on this node. +Using this parameter it is then possible to separate the container's data traffic from the +management traffic of the cluster. +If unspecified, Docker will use the same IP address or interface that is used for the +advertise address. + ### `--task-history-limit` This flag sets up task history retention limit. diff --git a/docs/reference/commandline/swarm_join.md b/docs/reference/commandline/swarm_join.md index 4ee11c188b..346fb3532e 100644 --- a/docs/reference/commandline/swarm_join.md +++ b/docs/reference/commandline/swarm_join.md @@ -23,6 +23,7 @@ Join a swarm as a node and/or manager Options: --advertise-addr string Advertised address (format: [:port]) --availability string Availability of the node ("active"|"pause"|"drain") (default "active") + --data-path-addr string Address or interface to use for data path traffic (format: ) --help Print usage --listen-addr node-addr Listen address (format: [:port]) (default 0.0.0.0:2377) --token string Token for entry into the swarm @@ -95,6 +96,15 @@ name, the default port 2377 will be used. This flag is generally not necessary when joining an existing swarm. +### `--data-path-addr` + +This flag specifies the address that global scope network drivers will publish towards +other nodes in order to reach the containers running on this node. +Using this parameter it is then possible to separate the container's data traffic from the +management traffic of the cluster. +If unspecified, Docker will use the same IP address or interface that is used for the +advertise address. + ### `--token string` Secret value required for nodes to join the swarm diff --git a/integration-cli/docker_cli_swarm_test.go b/integration-cli/docker_cli_swarm_test.go index 5d79ee9176..5c1bd2f9ce 100644 --- a/integration-cli/docker_cli_swarm_test.go +++ b/integration-cli/docker_cli_swarm_test.go @@ -1932,3 +1932,15 @@ func (s *DockerSwarmSuite) TestSwarmServiceLsFilterMode(c *check.C) { c.Assert(out, checker.Contains, "top1") c.Assert(out, checker.Not(checker.Contains), "top2") } + +func (s *DockerSwarmSuite) TestSwarmInitUnspecifiedDataPathAddr(c *check.C) { + d := s.AddDaemon(c, false, false) + + out, err := d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0") + c.Assert(err, checker.NotNil) + c.Assert(out, checker.Contains, "data path address must be a non-zero IP") + + out, err = d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0:2000") + c.Assert(err, checker.NotNil) + c.Assert(out, checker.Contains, "data path address must be a non-zero IP") +}