Added API to set ephemeral port allocator range.

Also reduce the allowed port range as the total number of containers
per host is typically less than 1K.

This change helps in scenarios where there are other services on
the same host that uses ephemeral ports in iptables manipulation.

The workflow requires changes in docker engine (
https://github.com/moby/moby/pull/40055) and this change. It
works as follows:

1. user can now specified to docker engine an option
   --published-port-range="50000-60000" as cmdline argument or
   in daemon.json.
2. docker engine read and pass this info to libnetwork via
   config.go:OptionDynamicPortRange.
3. libnetwork uses this range to allocate dynamic port henceforth.
4. --published-port-range can be set either via SIGHUP or
   restart docker engine
5. if --published-port-range is not set by user, a OS specific
   default range is used for dynamic port allocation.
   Linux: 49153-60999, Windows: 60000-65000
6 if --published-port-range is invalid, that is, the range
  given is outside of allowed default range, no change takes place.
  libnetwork will continue to use old/existing port range for
  dynamic port allocation.

Signed-off-by: Su Wang <su.wang@docker.com>

libnetwork/config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/BurntSushi/toml"
@@ -13,6 +14,7 @@ import (
 	"github.com/docker/libnetwork/ipamutils"
 	"github.com/docker/libnetwork/netlabel"
 	"github.com/docker/libnetwork/osl"
+	"github.com/docker/libnetwork/portallocator"
 	"github.com/sirupsen/logrus"
 )
 
@@ -238,6 +240,23 @@ func OptionExperimental(exp bool) Option {
 	}
 }
 
+// OptionDynamicPortRange function returns an option setter for service port allocation range
+func OptionDynamicPortRange(in string) Option {
+	return func(c *Config) {
+		start, end := 0, 0
+		if len(in) > 0 {
+			n, err := fmt.Sscanf(in, "%d-%d", &start, &end)
+			if n != 2 || err != nil {
+				logrus.Errorf("Failed to parse range string with err %v", err)
+				return
+			}
+		}
+		if err := portallocator.Get().SetPortRange(start, end); err != nil {
+			logrus.Errorf("Failed to set port range with err %v", err)
+		}
+	}
+}
+
 // OptionNetworkControlPlaneMTU function returns an option setter for control plane MTU
 func OptionNetworkControlPlaneMTU(exp int) Option {
 	return func(c *Config) {

libnetwork/portallocator/portallocator.go

@@ -3,17 +3,36 @@ package portallocator
 import (
 	"errors"
 	"fmt"
+	"github.com/sirupsen/logrus"
 	"net"
 	"sync"
 )
 
-const (
+var (
-	// DefaultPortRangeStart indicates the first port in port range
+	// defaultPortRangeStart indicates the first port in port range
-	DefaultPortRangeStart = 49153
+	defaultPortRangeStart = 49153
-	// DefaultPortRangeEnd indicates the last port in port range
+	// defaultPortRangeEnd indicates the last port in port range
-	DefaultPortRangeEnd = 65535
+	// consistent with default /proc/sys/net/ipv4/ip_local_port_range
+	// upper bound on linux
+	defaultPortRangeEnd = 60999
 )
 
+func sanitizePortRange(start int, end int) (newStart, newEnd int, err error) {
+	if start > defaultPortRangeEnd || end < defaultPortRangeStart || start > end {
+		return 0, 0, fmt.Errorf("Request out allowed range [%v, %v]",
+			defaultPortRangeStart, defaultPortRangeEnd)
+	}
+	err = nil
+	newStart, newEnd = start, end
+	if start < defaultPortRangeStart {
+		newStart = defaultPortRangeStart
+	}
+	if end > defaultPortRangeEnd {
+		newEnd = defaultPortRangeEnd
+	}
+	return
+}
+
 type ipMapping map[string]protoMap
 
 var (
@@ -92,11 +111,19 @@ func Get() *PortAllocator {
 	return instance
 }
 
-func newInstance() *PortAllocator {
+func getDefaultPortRange() (int, int) {
 	start, end, err := getDynamicPortRange()
-	if err != nil {
+	if err == nil {
-		start, end = DefaultPortRangeStart, DefaultPortRangeEnd
+		start, end, err = sanitizePortRange(start, end)
 	}
+	if err != nil {
+		start, end = defaultPortRangeStart, defaultPortRangeEnd
+	}
+	return start, end
+}
+
+func newInstance() *PortAllocator {
+	start, end := getDefaultPortRange()
 	return &PortAllocator{
 		ipMap: ipMapping{},
 		Begin: start,
@@ -170,6 +197,35 @@ func (p *PortAllocator) ReleasePort(ip net.IP, proto string, port int) error {
 	return nil
 }
 
+// SetPortRange sets dynamic port allocation range.
+// if both portBegin and portEnd are 0, the port range reverts to default
+// value. Otherwise they are sanitized against the default values to
+// ensure their validity.
+func (p *PortAllocator) SetPortRange(portBegin, portEnd int) error {
+	// if begin and end is zero, revert to default values
+	var begin, end int
+	var err error
+	if portBegin == 0 && portEnd == 0 {
+		begin, end = getDefaultPortRange()
+
+	} else {
+		begin, end, err = sanitizePortRange(portBegin, portEnd)
+		if err != nil {
+			return err
+		}
+	}
+	logrus.Debugf("Setting up port allocator to range %v-%v, current %v-%v",
+		begin, end, p.Begin, p.End)
+	p.mutex.Lock()
+	defer p.mutex.Unlock()
+	if p.Begin == begin && p.End == end {
+		return nil
+	}
+	p.ipMap = ipMapping{}
+	p.Begin, p.End = begin, end
+	return nil
+}
+
 func (p *PortAllocator) newPortMap() *portMap {
 	defaultKey := getRangeKey(p.Begin, p.End)
 	pm := &portMap{

libnetwork/portallocator/portallocator_freebsd.go

@@ -8,7 +8,7 @@ import (
 
 func getDynamicPortRange() (start int, end int, err error) {
 	portRangeKernelSysctl := []string{"net.inet.ip.portrange.hifirst", "net.ip.portrange.hilast"}
-	portRangeFallback := fmt.Sprintf("using fallback port range %d-%d", DefaultPortRangeStart, DefaultPortRangeEnd)
+	portRangeFallback := fmt.Sprintf("using fallback port range %d-%d", defaultPortRangeStart, defaultPortRangeEnd)
 	portRangeLowCmd := exec.Command("/sbin/sysctl", portRangeKernelSysctl[0])
 	var portRangeLowOut bytes.Buffer
 	portRangeLowCmd.Stdout = &portRangeLowOut

libnetwork/portallocator/portallocator_linux.go

@@ -8,7 +8,7 @@ import (
 
 func getDynamicPortRange() (start int, end int, err error) {
 	const portRangeKernelParam = "/proc/sys/net/ipv4/ip_local_port_range"
-	portRangeFallback := fmt.Sprintf("using fallback port range %d-%d", DefaultPortRangeStart, DefaultPortRangeEnd)
+	portRangeFallback := fmt.Sprintf("using fallback port range %d-%d", defaultPortRangeStart, defaultPortRangeEnd)
 	file, err := os.Open(portRangeKernelParam)
 	if err != nil {
 		return 0, 0, fmt.Errorf("port allocator - %s due to error: %v", portRangeFallback, err)

libnetwork/portallocator/portallocator_test.go

@@ -1,6 +1,7 @@
 package portallocator
 
 import (
+	"fmt"
 	"net"
 	"testing"
 
@@ -321,3 +322,47 @@ func TestNoDuplicateBPR(t *testing.T) {
 		t.Fatalf("Acquire(0) allocated the same port twice: %d", port)
 	}
 }
+
+func TestChangePortRange(t *testing.T) {
+	var tests = []struct {
+		begin  int
+		end    int
+		setErr error
+		reqRlt int
+	}{
+		{defaultPortRangeEnd + 1, defaultPortRangeEnd + 10, fmt.Errorf("begin out of range"), 0},
+		{defaultPortRangeStart - 10, defaultPortRangeStart - 1, fmt.Errorf("end out of range"), 0},
+		{defaultPortRangeEnd, defaultPortRangeStart, fmt.Errorf("out of order"), 0},
+		{defaultPortRangeStart + 100, defaultPortRangeEnd + 10, nil, defaultPortRangeStart + 100},
+		{0, 0, nil, defaultPortRangeStart}, // revert to default if no value given
+		{defaultPortRangeStart - 100, defaultPortRangeEnd, nil, defaultPortRangeStart + 1},
+	}
+	p := Get()
+	port := 0
+	for _, c := range tests {
+		t.Logf("test: port allocate range %v-%v, setErr=%v, reqPort=%v",
+			c.begin, c.end, c.setErr, c.reqRlt)
+		err := p.SetPortRange(c.begin, c.end)
+		if (c.setErr == nil && c.setErr != err) ||
+			(c.setErr != nil && err == nil) {
+			t.Fatalf("Unexpected set range result, expected=%v, actual=%v", c.setErr, err)
+		}
+		if err != nil {
+			continue
+		}
+		if port > 0 {
+			err := p.ReleasePort(defaultIP, "tcp", port)
+			if err != nil {
+				t.Fatalf("Releasing port %v failed, err=%v", port, err)
+			}
+		}
+
+		port, err = p.RequestPort(defaultIP, "tcp", 0)
+		if err != nil {
+			t.Fatalf("Request failed, err %v", err)
+		}
+		if port != c.reqRlt {
+			t.Fatalf("Incorrect port returned, expected=%v, actual=%v", c.reqRlt, port)
+		}
+	}
+}

libnetwork/portallocator/portallocator_windows.go

@@ -1,10 +1,10 @@
 package portallocator
 
-const (
+func init() {
-	StartPortRange = 60000
+	defaultPortRangeStart = 60000
-	EndPortRange   = 65000
+	defaultPortRangeEnd = 65000
-)
+}
 
 func getDynamicPortRange() (start int, end int, err error) {
-	return StartPortRange, EndPortRange, nil
+	return defaultPortRangeStart, defaultPortRangeEnd, nil
 }