From 32180ac0c77760d9023c34c73ea9f0c41a41c7cd Mon Sep 17 00:00:00 2001 From: selansen Date: Tue, 20 Nov 2018 16:44:40 -0500 Subject: [PATCH] VXLAN UDP Port configuration support This commit contains changes to configure DataPathPort option. By default we use 4789 port number. But this commit will allow user to configure port number during swarm init. DataPathPort can't be modified after swarm init. Signed-off-by: selansen --- api/server/router/swarm/cluster_routes.go | 5 ++ api/swagger.yaml | 17 +++++++ api/types/swarm/swarm.go | 2 + daemon/cluster/convert/swarm.go | 1 + daemon/cluster/listen_addr.go | 19 ++++++++ daemon/cluster/noderunner.go | 3 ++ daemon/cluster/swarm.go | 7 +++ docs/api/version-history.md | 3 ++ integration/network/service_test.go | 59 +++++++++++++++++++++++ internal/test/daemon/daemon.go | 1 + internal/test/daemon/ops.go | 7 +++ internal/test/daemon/swarm.go | 3 ++ 12 files changed, 127 insertions(+) diff --git a/api/server/router/swarm/cluster_routes.go b/api/server/router/swarm/cluster_routes.go index b29cfb0e6c..1e0e6100f7 100644 --- a/api/server/router/swarm/cluster_routes.go +++ b/api/server/router/swarm/cluster_routes.go @@ -28,11 +28,16 @@ func (sr *swarmRouter) initCluster(ctx context.Context, w http.ResponseWriter, r return errdefs.InvalidParameter(err) } version := httputils.VersionFromContext(ctx) + // DefaultAddrPool and SubnetSize were added in API 1.39. Ignore on older API versions. if versions.LessThan(version, "1.39") { req.DefaultAddrPool = nil req.SubnetSize = 0 } + // DataPathPort was added in API 1.40. Ignore this option on older API versions. + if versions.LessThan(version, "1.40") { + req.DataPathPort = 0 + } nodeID, err := sr.backend.Init(req) if err != nil { logrus.Errorf("Error initializing swarm: %v", err) diff --git a/api/swagger.yaml b/api/swagger.yaml index 5bc7173dc7..49d3e0a83b 100644 --- a/api/swagger.yaml +++ b/api/swagger.yaml @@ -2465,6 +2465,15 @@ definitions: description: "Whether there is currently a root CA rotation in progress for the swarm" type: "boolean" example: false + DataPathPort: + description: | + DataPathPort specifies the data path port number for data traffic. + Acceptable port range is 1024 to 49151. + If no port is set or is set to 0, the default port (4789) is used. + type: "integer" + format: "uint32" + default: 4789 + example: 4789 DefaultAddrPool: description: | Default Address Pool specifies default subnet pools for global scope networks. @@ -8877,6 +8886,13 @@ paths: nodes in order to reach the containers running on this node. Using this parameter it is possible to separate the container data traffic from the management traffic of the cluster. type: "string" + DataPathPort: + description: | + DataPathPort specifies the data path port number for data traffic. + Acceptable port range is 1024 to 49151. + if no port is set or is set to 0, default port 4789 will be used. + type: "integer" + format: "uint32" DefaultAddrPool: description: | Default Address Pool specifies default subnet pools for global scope networks. @@ -8897,6 +8913,7 @@ paths: example: ListenAddr: "0.0.0.0:2377" AdvertiseAddr: "192.168.1.1:2377" + DataPathPort: 4789 DefaultAddrPool: ["10.10.0.0/8", "20.20.0.0/8"] SubnetSize: 24 ForceNewCluster: false diff --git a/api/types/swarm/swarm.go b/api/types/swarm/swarm.go index b742cf1bfb..484cd0be7f 100644 --- a/api/types/swarm/swarm.go +++ b/api/types/swarm/swarm.go @@ -14,6 +14,7 @@ type ClusterInfo struct { RootRotationInProgress bool DefaultAddrPool []string SubnetSize uint32 + DataPathPort uint32 } // Swarm represents a swarm. @@ -153,6 +154,7 @@ type InitRequest struct { ListenAddr string AdvertiseAddr string DataPathAddr string + DataPathPort uint32 ForceNewCluster bool Spec Spec AutoLockManagers bool diff --git a/daemon/cluster/convert/swarm.go b/daemon/cluster/convert/swarm.go index 643505badf..bdad2a5ea7 100644 --- a/daemon/cluster/convert/swarm.go +++ b/daemon/cluster/convert/swarm.go @@ -42,6 +42,7 @@ func SwarmFromGRPC(c swarmapi.Cluster) types.Swarm { RootRotationInProgress: c.RootCA.RootRotation != nil, DefaultAddrPool: c.DefaultAddressPool, SubnetSize: c.SubnetSize, + DataPathPort: c.VXLANUDPPort, }, JoinTokens: types.JoinTokens{ Worker: c.RootCA.JoinTokens.Worker, diff --git a/daemon/cluster/listen_addr.go b/daemon/cluster/listen_addr.go index 44ea5fce42..9e455d317f 100644 --- a/daemon/cluster/listen_addr.go +++ b/daemon/cluster/listen_addr.go @@ -123,6 +123,25 @@ func validateDefaultAddrPool(defaultAddrPool []string, size uint32) error { return nil } +// getDataPathPort validates vxlan udp port (data path port) number. +// if no port is set, the default (4789) is returned +// valid port numbers are between 1024 and 49151 +func getDataPathPort(portNum uint32) (uint32, error) { + // if the value comes as 0 by any reason we set it to default value 4789 + if portNum == 0 { + portNum = 4789 + return portNum, nil + } + // IANA procedures for each range in detail + // The Well Known Ports, aka the System Ports, from 0-1023 + // The Registered Ports, aka the User Ports, from 1024-49151 + // The Dynamic Ports, aka the Private Ports, from 49152-65535 + // So we can allow range between 1024 to 49151 + if portNum < 1024 || portNum > 49151 { + return 0, fmt.Errorf("Datapath port number is not in valid range (1024-49151) : %d", portNum) + } + return portNum, nil +} func resolveDataPathAddr(dataPathAddr string) (string, error) { if dataPathAddr == "" { // dataPathAddr is not defined diff --git a/daemon/cluster/noderunner.go b/daemon/cluster/noderunner.go index aa905d6780..ef46021e4b 100644 --- a/daemon/cluster/noderunner.go +++ b/daemon/cluster/noderunner.go @@ -57,6 +57,8 @@ type nodeStartConfig struct { DefaultAddressPool []string // SubnetSize contains subnet size of DefaultAddressPool SubnetSize uint32 + // DataPathPort contains Data path port (VXLAN UDP port) number that is used for data traffic. + DataPathPort uint32 // JoinInProgress is set to true if a join operation has started, but // not completed yet. JoinInProgress bool @@ -125,6 +127,7 @@ func (n *nodeRunner) start(conf nodeStartConfig) error { NetworkConfig: &swarmallocator.NetworkConfig{ DefaultAddrPool: conf.DefaultAddressPool, SubnetSize: conf.SubnetSize, + VXLANUDPPort: conf.DataPathPort, }, JoinAddr: joinAddr, StateDir: n.cluster.root, diff --git a/daemon/cluster/swarm.go b/daemon/cluster/swarm.go index 65dfe9eb45..8cc172e9ce 100644 --- a/daemon/cluster/swarm.go +++ b/daemon/cluster/swarm.go @@ -96,6 +96,12 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) { if err := validateDefaultAddrPool(req.DefaultAddrPool, req.SubnetSize); err != nil { return "", err } + + port, err := getDataPathPort(req.DataPathPort) + if err != nil { + return "", err + } + nr, err := c.newNodeRunner(nodeStartConfig{ forceNewCluster: req.ForceNewCluster, autolock: req.AutoLockManagers, @@ -106,6 +112,7 @@ func (c *Cluster) Init(req types.InitRequest) (string, error) { DefaultAddressPool: req.DefaultAddrPool, SubnetSize: req.SubnetSize, availability: req.Availability, + DataPathPort: port, }) if err != nil { return "", err diff --git a/docs/api/version-history.md b/docs/api/version-history.md index 611ac38fae..a59a90322a 100644 --- a/docs/api/version-history.md +++ b/docs/api/version-history.md @@ -29,6 +29,9 @@ keywords: "API, Docker, rcli, REST, documentation" to return those without the specified labels. * `POST /containers/create`, `GET /containers/{id}/json`, and `GET /containers/json` now supports `BindOptions.NonRecursive`. +* `POST /swarm/init` now accepts a `DataPathPort` property to set data path port number. +* `GET /info` now returns information about `DataPathPort` that is currently used in swarm +* `GET /swarm` endpoint now returns DataPathPort info ## V1.39 API changes diff --git a/integration/network/service_test.go b/integration/network/service_test.go index 4762b577ac..b3a3223523 100644 --- a/integration/network/service_test.go +++ b/integration/network/service_test.go @@ -321,6 +321,65 @@ func noServices(client client.ServiceAPIClient) func(log poll.LogT) poll.Result } } +func TestServiceWithDataPathPortInit(t *testing.T) { + skip.If(t, testEnv.OSType == "windows") + skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.40"), "DataPathPort was added in API v1.40") + defer setupTest(t)() + var ops = []func(*daemon.Daemon){} + var datapathPort uint32 = 7777 + ops = append(ops, daemon.WithSwarmDataPathPort(datapathPort)) + d := swarm.NewSwarm(t, testEnv, ops...) + + cli := d.NewClientT(t) + defer cli.Close() + + // Create a overlay network + name := "saanvisthira" + t.Name() + network.CreateNoError(t, context.Background(), cli, name, + network.WithDriver("overlay")) + + var instances uint64 = 1 + serviceID := swarm.CreateService(t, d, + swarm.ServiceWithReplicas(instances), + swarm.ServiceWithNetwork(name), + ) + + poll.WaitOn(t, serviceRunningCount(cli, serviceID, instances), swarm.ServicePoll) + + info := d.Info(t) + assert.Equal(t, info.Swarm.Cluster.DataPathPort, datapathPort) + err := cli.ServiceRemove(context.Background(), serviceID) + assert.NilError(t, err) + d.SwarmLeave(true) + d.Stop(t) + + // Clean up , set it back to original one to make sure other tests don't fail + // call without datapath port option. + ops = []func(*daemon.Daemon){} + d = swarm.NewSwarm(t, testEnv, ops...) + cli = d.NewClientT(t) + + // Create a overlay network + name = "saanvisthira" + t.Name() + network.CreateNoError(t, context.Background(), cli, name, + network.WithDriver("overlay")) + + serviceID = swarm.CreateService(t, d, + swarm.ServiceWithReplicas(instances), + swarm.ServiceWithNetwork(name), + ) + + poll.WaitOn(t, serviceRunningCount(cli, serviceID, instances), swarm.ServicePoll) + + info = d.Info(t) + var defaultDataPathPort uint32 = 4789 + assert.Equal(t, info.Swarm.Cluster.DataPathPort, defaultDataPathPort) + err = cli.ServiceRemove(context.Background(), serviceID) + assert.NilError(t, err) + d.SwarmLeave(true) + defer d.Stop(t) +} + func TestServiceWithDefaultAddressPoolInit(t *testing.T) { skip.If(t, testEnv.OSType == "windows") defer setupTest(t)() diff --git a/internal/test/daemon/daemon.go b/internal/test/daemon/daemon.go index 7471d1093b..fb970e095c 100644 --- a/internal/test/daemon/daemon.go +++ b/internal/test/daemon/daemon.go @@ -76,6 +76,7 @@ type Daemon struct { SwarmPort int // FIXME(vdemeester) should probably not be exported DefaultAddrPool []string SubnetSize uint32 + DataPathPort uint32 // cached information CachedInfo types.Info } diff --git a/internal/test/daemon/ops.go b/internal/test/daemon/ops.go index 4154a77977..ff8cd88946 100644 --- a/internal/test/daemon/ops.go +++ b/internal/test/daemon/ops.go @@ -48,6 +48,13 @@ func WithSwarmDefaultAddrPoolSubnetSize(subnetSize uint32) func(*Daemon) { } } +// WithSwarmDataPathPort sets the swarm datapath port to use for swarm mode +func WithSwarmDataPathPort(datapathPort uint32) func(*Daemon) { + return func(d *Daemon) { + d.DataPathPort = datapathPort + } +} + // WithEnvironment sets options from internal/test/environment.Execution struct func WithEnvironment(e environment.Execution) func(*Daemon) { return func(d *Daemon) { diff --git a/internal/test/daemon/swarm.go b/internal/test/daemon/swarm.go index 92ef856640..e500fe0fdc 100644 --- a/internal/test/daemon/swarm.go +++ b/internal/test/daemon/swarm.go @@ -85,6 +85,9 @@ func (d *Daemon) SwarmInit(t assert.TestingT, req swarm.InitRequest) { req.DefaultAddrPool = d.DefaultAddrPool req.SubnetSize = d.SubnetSize } + if d.DataPathPort > 0 { + req.DataPathPort = d.DataPathPort + } cli := d.NewClientT(t) defer cli.Close() _, err := cli.SwarmInit(context.Background(), req)