From 8dc8cd4719f165c01c98e7d3ce1d6cea6a8f60b8 Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Fri, 14 Apr 2017 16:54:17 -0700 Subject: [PATCH] Inroduce SWARM --data-path-addr flag This new flag will allow the configuration of an interface that can be used for data path traffic to be isolated from control plane traffic. This flag is simply percolated down to libnetwork and will be used by all the global scope drivers (today overlay) Negative test added for invalid flag arguments Signed-off-by: Flavio Crisciani --- api/types/swarm/swarm.go | 2 + cli/command/swarm/init.go | 3 + cli/command/swarm/join.go | 3 + cli/command/swarm/opts.go | 1 + contrib/completion/bash/docker | 4 +- contrib/completion/zsh/_docker | 2 + daemon/cluster/cluster.go | 10 +++ daemon/cluster/listen_addr.go | 95 +++++++++++++++--------- daemon/cluster/noderunner.go | 5 +- daemon/cluster/swarm.go | 12 +++ docs/reference/commandline/swarm_init.md | 10 +++ docs/reference/commandline/swarm_join.md | 10 +++ integration-cli/docker_cli_swarm_test.go | 12 +++ 13 files changed, 131 insertions(+), 38 deletions(-) diff --git a/api/types/swarm/swarm.go b/api/types/swarm/swarm.go index c513274750..08da338439 100644 --- a/api/types/swarm/swarm.go +++ b/api/types/swarm/swarm.go @@ -132,6 +132,7 @@ type ExternalCA struct { type InitRequest struct { ListenAddr string AdvertiseAddr string + DataPathAddr string ForceNewCluster bool Spec Spec AutoLockManagers bool @@ -142,6 +143,7 @@ type InitRequest struct { type JoinRequest struct { ListenAddr string AdvertiseAddr string + DataPathAddr string RemoteAddrs []string JoinToken string // accept by secret Availability NodeAvailability diff --git a/cli/command/swarm/init.go b/cli/command/swarm/init.go index 37d96de113..5d42b0174c 100644 --- a/cli/command/swarm/init.go +++ b/cli/command/swarm/init.go @@ -19,6 +19,7 @@ type initOptions struct { listenAddr NodeAddrOption // Not a NodeAddrOption because it has no default port. advertiseAddr string + dataPathAddr string forceNewCluster bool availability string } @@ -40,6 +41,7 @@ func newInitCommand(dockerCli command.Cli) *cobra.Command { flags := cmd.Flags() flags.Var(&opts.listenAddr, flagListenAddr, "Listen address (format: [:port])") flags.StringVar(&opts.advertiseAddr, flagAdvertiseAddr, "", "Advertised address (format: [:port])") + flags.StringVar(&opts.dataPathAddr, flagDataPathAddr, "", "Address or interface to use for data path traffic (format: )") flags.BoolVar(&opts.forceNewCluster, "force-new-cluster", false, "Force create a new cluster from current state") flags.BoolVar(&opts.autolock, flagAutolock, false, "Enable manager autolocking (requiring an unlock key to start a stopped manager)") flags.StringVar(&opts.availability, flagAvailability, "active", `Availability of the node ("active"|"pause"|"drain")`) @@ -54,6 +56,7 @@ func runInit(dockerCli command.Cli, flags *pflag.FlagSet, opts initOptions) erro req := swarm.InitRequest{ ListenAddr: opts.listenAddr.String(), AdvertiseAddr: opts.advertiseAddr, + DataPathAddr: opts.dataPathAddr, ForceNewCluster: opts.forceNewCluster, Spec: opts.swarmOptions.ToSpec(flags), AutoLockManagers: opts.swarmOptions.autolock, diff --git a/cli/command/swarm/join.go b/cli/command/swarm/join.go index 873eaaefaa..34730ab8c7 100644 --- a/cli/command/swarm/join.go +++ b/cli/command/swarm/join.go @@ -19,6 +19,7 @@ type joinOptions struct { listenAddr NodeAddrOption // Not a NodeAddrOption because it has no default port. advertiseAddr string + dataPathAddr string token string availability string } @@ -41,6 +42,7 @@ func newJoinCommand(dockerCli command.Cli) *cobra.Command { flags := cmd.Flags() flags.Var(&opts.listenAddr, flagListenAddr, "Listen address (format: [:port])") flags.StringVar(&opts.advertiseAddr, flagAdvertiseAddr, "", "Advertised address (format: [:port])") + flags.StringVar(&opts.dataPathAddr, flagDataPathAddr, "", "Address or interface to use for data path traffic (format: )") flags.StringVar(&opts.token, flagToken, "", "Token for entry into the swarm") flags.StringVar(&opts.availability, flagAvailability, "active", `Availability of the node ("active"|"pause"|"drain")`) return cmd @@ -54,6 +56,7 @@ func runJoin(dockerCli command.Cli, flags *pflag.FlagSet, opts joinOptions) erro JoinToken: opts.token, ListenAddr: opts.listenAddr.String(), AdvertiseAddr: opts.advertiseAddr, + DataPathAddr: opts.dataPathAddr, RemoteAddrs: []string{opts.remote}, } if flags.Changed(flagAvailability) { diff --git a/cli/command/swarm/opts.go b/cli/command/swarm/opts.go index 6eddddccae..dc32dc488c 100644 --- a/cli/command/swarm/opts.go +++ b/cli/command/swarm/opts.go @@ -19,6 +19,7 @@ const ( flagDispatcherHeartbeat = "dispatcher-heartbeat" flagListenAddr = "listen-addr" flagAdvertiseAddr = "advertise-addr" + flagDataPathAddr = "data-path-addr" flagQuiet = "quiet" flagRotate = "rotate" flagToken = "token" diff --git a/contrib/completion/bash/docker b/contrib/completion/bash/docker index 813f2b6ccc..1a0e110633 100644 --- a/contrib/completion/bash/docker +++ b/contrib/completion/bash/docker @@ -3301,7 +3301,7 @@ _docker_swarm_init() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--advertise-addr --autolock --availability --cert-expiry --dispatcher-heartbeat --external-ca --force-new-cluster --help --listen-addr --max-snapshots --snapshot-interval --task-history-limit" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--advertise-addr --data-path-addr --autolock --availability --cert-expiry --dispatcher-heartbeat --external-ca --force-new-cluster --help --listen-addr --max-snapshots --snapshot-interval --task-history-limit" -- "$cur" ) ) ;; esac } @@ -3337,7 +3337,7 @@ _docker_swarm_join() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--advertise-addr --availability --help --listen-addr --token" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--advertise-addr --data-path-addr --availability --help --listen-addr --token" -- "$cur" ) ) ;; *:) COMPREPLY=( $( compgen -W "2377" -- "${cur##*:}" ) ) diff --git a/contrib/completion/zsh/_docker b/contrib/completion/zsh/_docker index 7a3a492415..316caf5d43 100644 --- a/contrib/completion/zsh/_docker +++ b/contrib/completion/zsh/_docker @@ -2267,6 +2267,7 @@ __docker_swarm_subcommand() { _arguments $(__docker_arguments) \ $opts_help \ "($help)--advertise-addr=[Advertised address]:ip\:port: " \ + "($help)--data-path-addr=[Data path IP or interface]:ip " \ "($help)--autolock[Enable manager autolocking]" \ "($help)--availability=[Availability of the node]:availability:(active drain pause)" \ "($help)--cert-expiry=[Validity period for node certificates]:duration: " \ @@ -2282,6 +2283,7 @@ __docker_swarm_subcommand() { _arguments $(__docker_arguments) -A '-*' \ $opts_help \ "($help)--advertise-addr=[Advertised address]:ip\:port: " \ + "($help)--data-path-addr=[Data path IP or interface]:ip " \ "($help)--availability=[Availability of the node]:availability:(active drain pause)" \ "($help)--listen-addr=[Listen address]:ip\:port: " \ "($help)--token=[Token for entry into the swarm]:secret: " \ diff --git a/daemon/cluster/cluster.go b/daemon/cluster/cluster.go index aa622d9b1c..8ff527877f 100644 --- a/daemon/cluster/cluster.go +++ b/daemon/cluster/cluster.go @@ -270,6 +270,16 @@ func (c *Cluster) GetAdvertiseAddress() string { return c.currentNodeState().actualLocalAddr } +// GetDataPathAddress returns the address to be used for the data path traffic, if specified. +func (c *Cluster) GetDataPathAddress() string { + c.mu.RLock() + defer c.mu.RUnlock() + if c.nr != nil { + return c.nr.config.DataPathAddr + } + return "" +} + // GetRemoteAddress returns a known advertise address of a remote manager if // available. // todo: change to array/connect with info diff --git a/daemon/cluster/listen_addr.go b/daemon/cluster/listen_addr.go index 83e74ad464..993ccb62ad 100644 --- a/daemon/cluster/listen_addr.go +++ b/daemon/cluster/listen_addr.go @@ -10,8 +10,10 @@ var ( errNoSuchInterface = errors.New("no such interface") errNoIP = errors.New("could not find the system's IP address") errMustSpecifyListenAddr = errors.New("must specify a listening address because the address to advertise is not recognized as a system address, and a system's IP address to use could not be uniquely identified") + errBadNetworkIdentifier = errors.New("must specify a valid IP address or interface name") errBadListenAddr = errors.New("listen address must be an IP address or network interface (with optional port number)") errBadAdvertiseAddr = errors.New("advertise address must be a non-zero IP address or network interface (with optional port number)") + errBadDataPathAddr = errors.New("data path address must be a non-zero IP address or network interface (without a port number)") errBadDefaultAdvertiseAddr = errors.New("default advertise address must be a non-zero IP address or network interface (without a port number)") ) @@ -20,23 +22,17 @@ func resolveListenAddr(specifiedAddr string) (string, string, error) { if err != nil { return "", "", fmt.Errorf("could not parse listen address %s", specifiedAddr) } - // Does the host component match any of the interface names on the // system? If so, use the address from that interface. - interfaceAddr, err := resolveInterfaceAddr(specifiedHost) - if err == nil { - return interfaceAddr.String(), specifiedPort, nil - } - if err != errNoSuchInterface { + specifiedIP, err := resolveInputIPAddr(specifiedHost, true) + if err != nil { + if err == errBadNetworkIdentifier { + err = errBadListenAddr + } return "", "", err } - // If it's not an interface, it must be an IP (for now) - if net.ParseIP(specifiedHost) == nil { - return "", "", errBadListenAddr - } - - return specifiedHost, specifiedPort, nil + return specifiedIP.String(), specifiedPort, nil } func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (string, string, error) { @@ -57,43 +53,32 @@ func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (st advertiseHost = advertiseAddr advertisePort = listenAddrPort } - // Does the host component match any of the interface names on the // system? If so, use the address from that interface. - interfaceAddr, err := resolveInterfaceAddr(advertiseHost) - if err == nil { - return interfaceAddr.String(), advertisePort, nil - } - if err != errNoSuchInterface { + advertiseIP, err := resolveInputIPAddr(advertiseHost, false) + if err != nil { + if err == errBadNetworkIdentifier { + err = errBadAdvertiseAddr + } return "", "", err } - // If it's not an interface, it must be an IP (for now) - if ip := net.ParseIP(advertiseHost); ip == nil || ip.IsUnspecified() { - return "", "", errBadAdvertiseAddr - } - - return advertiseHost, advertisePort, nil + return advertiseIP.String(), advertisePort, nil } if c.config.DefaultAdvertiseAddr != "" { // Does the default advertise address component match any of the // interface names on the system? If so, use the address from // that interface. - interfaceAddr, err := resolveInterfaceAddr(c.config.DefaultAdvertiseAddr) - if err == nil { - return interfaceAddr.String(), listenAddrPort, nil - } - if err != errNoSuchInterface { + defaultAdvertiseIP, err := resolveInputIPAddr(c.config.DefaultAdvertiseAddr, false) + if err != nil { + if err == errBadNetworkIdentifier { + err = errBadDefaultAdvertiseAddr + } return "", "", err } - // If it's not an interface, it must be an IP (for now) - if ip := net.ParseIP(c.config.DefaultAdvertiseAddr); ip == nil || ip.IsUnspecified() { - return "", "", errBadDefaultAdvertiseAddr - } - - return c.config.DefaultAdvertiseAddr, listenAddrPort, nil + return defaultAdvertiseIP.String(), listenAddrPort, nil } systemAddr, err := c.resolveSystemAddr() @@ -103,6 +88,22 @@ func (c *Cluster) resolveAdvertiseAddr(advertiseAddr, listenAddrPort string) (st return systemAddr.String(), listenAddrPort, nil } +func resolveDataPathAddr(dataPathAddr string) (string, error) { + if dataPathAddr == "" { + // dataPathAddr is not defined + return "", nil + } + // If a data path flag is specified try to resolve the IP address. + dataPathIP, err := resolveInputIPAddr(dataPathAddr, false) + if err != nil { + if err == errBadNetworkIdentifier { + err = errBadDataPathAddr + } + return "", err + } + return dataPathIP.String(), nil +} + func resolveInterfaceAddr(specifiedInterface string) (net.IP, error) { // Use a specific interface's IP address. intf, err := net.InterfaceByName(specifiedInterface) @@ -149,6 +150,30 @@ func resolveInterfaceAddr(specifiedInterface string) (net.IP, error) { return interfaceAddr6, nil } +// resolveInputIPAddr tries to resolve the IP address from the string passed as input +// - tries to match the string as an interface name, if so returns the IP address associated with it +// - on failure of previous step tries to parse the string as an IP address itself +// if succeeds returns the IP address +func resolveInputIPAddr(input string, isUnspecifiedValid bool) (net.IP, error) { + // Try to see if it is an interface name + interfaceAddr, err := resolveInterfaceAddr(input) + if err == nil { + return interfaceAddr, nil + } + // String matched interface but there is a potential ambiguity to be resolved + if err != errNoSuchInterface { + return nil, err + } + + // String is not an interface check if it is a valid IP + if ip := net.ParseIP(input); ip != nil && (isUnspecifiedValid || !ip.IsUnspecified()) { + return ip, nil + } + + // Not valid IP found + return nil, errBadNetworkIdentifier +} + func (c *Cluster) resolveSystemAddrViaSubnetCheck() (net.IP, error) { // Use the system's only IP address, or fail if there are // multiple addresses to choose from. Skip interfaces which diff --git a/daemon/cluster/noderunner.go b/daemon/cluster/noderunner.go index 49fef1fcc8..6c8ca70376 100644 --- a/daemon/cluster/noderunner.go +++ b/daemon/cluster/noderunner.go @@ -46,7 +46,10 @@ type nodeStartConfig struct { ListenAddr string // AdvertiseAddr is the address other nodes should connect to, // including a port. - AdvertiseAddr string + AdvertiseAddr string + // DataPathAddr is the address that has to be used for the data path + DataPathAddr string + joinAddr string forceNewCluster bool joinToken string diff --git a/daemon/cluster/swarm.go b/daemon/cluster/swarm.go index 6b6a54303a..8db4f3621f 100644 --- a/daemon/cluster/swarm.go +++ b/daemon/cluster/swarm.go @@ -54,6 +54,11 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) { return "", err } + dataPathAddr, err := resolveDataPathAddr(req.DataPathAddr) + if err != nil { + return "", err + } + localAddr := listenHost // If the local address is undetermined, the advertise address @@ -93,6 +98,7 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) { LocalAddr: localAddr, ListenAddr: net.JoinHostPort(listenHost, listenPort), AdvertiseAddr: net.JoinHostPort(advertiseHost, advertisePort), + DataPathAddr: dataPathAddr, availability: req.Availability, }) if err != nil { @@ -155,12 +161,18 @@ func (c *Cluster) Join(req types.JoinRequest) error { } } + dataPathAddr, err := resolveDataPathAddr(req.DataPathAddr) + if err != nil { + return err + } + clearPersistentState(c.root) nr, err := c.newNodeRunner(nodeStartConfig{ RemoteAddr: req.RemoteAddrs[0], ListenAddr: net.JoinHostPort(listenHost, listenPort), AdvertiseAddr: advertiseAddr, + DataPathAddr: dataPathAddr, joinAddr: req.RemoteAddrs[0], joinToken: req.JoinToken, availability: req.Availability, diff --git a/docs/reference/commandline/swarm_init.md b/docs/reference/commandline/swarm_init.md index f4c6348e86..0daace038f 100644 --- a/docs/reference/commandline/swarm_init.md +++ b/docs/reference/commandline/swarm_init.md @@ -25,6 +25,7 @@ Options: --autolock Enable manager autolocking (requiring an unlock key to start a stopped manager) --availability string Availability of the node ("active"|"pause"|"drain") (default "active") --cert-expiry duration Validity period for node certificates (ns|us|ms|s|m|h) (default 2160h0m0s) + --data-path-addr string Address or interface to use for data path traffic (format: ) --dispatcher-heartbeat duration Dispatcher heartbeat period (ns|us|ms|s|m|h) (default 5s) --external-ca external-ca Specifications of one or more certificate signing endpoints --force-new-cluster Force create a new cluster from current state @@ -118,6 +119,15 @@ for example `--advertise-addr eth0:2377`. Specifying a port is optional. If the value is a bare IP address or interface name, the default port 2377 will be used. +### `--data-path-addr` + +This flag specifies the address that global scope network drivers will publish towards +other nodes in order to reach the containers running on this node. +Using this parameter it is then possible to separate the container's data traffic from the +management traffic of the cluster. +If unspecified, Docker will use the same IP address or interface that is used for the +advertise address. + ### `--task-history-limit` This flag sets up task history retention limit. diff --git a/docs/reference/commandline/swarm_join.md b/docs/reference/commandline/swarm_join.md index 4ee11c188b..346fb3532e 100644 --- a/docs/reference/commandline/swarm_join.md +++ b/docs/reference/commandline/swarm_join.md @@ -23,6 +23,7 @@ Join a swarm as a node and/or manager Options: --advertise-addr string Advertised address (format: [:port]) --availability string Availability of the node ("active"|"pause"|"drain") (default "active") + --data-path-addr string Address or interface to use for data path traffic (format: ) --help Print usage --listen-addr node-addr Listen address (format: [:port]) (default 0.0.0.0:2377) --token string Token for entry into the swarm @@ -95,6 +96,15 @@ name, the default port 2377 will be used. This flag is generally not necessary when joining an existing swarm. +### `--data-path-addr` + +This flag specifies the address that global scope network drivers will publish towards +other nodes in order to reach the containers running on this node. +Using this parameter it is then possible to separate the container's data traffic from the +management traffic of the cluster. +If unspecified, Docker will use the same IP address or interface that is used for the +advertise address. + ### `--token string` Secret value required for nodes to join the swarm diff --git a/integration-cli/docker_cli_swarm_test.go b/integration-cli/docker_cli_swarm_test.go index 5d79ee9176..5c1bd2f9ce 100644 --- a/integration-cli/docker_cli_swarm_test.go +++ b/integration-cli/docker_cli_swarm_test.go @@ -1932,3 +1932,15 @@ func (s *DockerSwarmSuite) TestSwarmServiceLsFilterMode(c *check.C) { c.Assert(out, checker.Contains, "top1") c.Assert(out, checker.Not(checker.Contains), "top2") } + +func (s *DockerSwarmSuite) TestSwarmInitUnspecifiedDataPathAddr(c *check.C) { + d := s.AddDaemon(c, false, false) + + out, err := d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0") + c.Assert(err, checker.NotNil) + c.Assert(out, checker.Contains, "data path address must be a non-zero IP") + + out, err = d.Cmd("swarm", "init", "--data-path-addr", "0.0.0.0:2000") + c.Assert(err, checker.NotNil) + c.Assert(out, checker.Contains, "data path address must be a non-zero IP") +}