From 8d73de97224bf94e5d2dceca2efbacbc8aa5959b Mon Sep 17 00:00:00 2001 From: Don Kjer Date: Fri, 12 Jun 2015 01:19:42 +0000 Subject: [PATCH] Adding libnetwork support to publish on custom host port ranges. See https://github.com/docker/docker/pull/12927 for docker portion. Signed-off-by: Don Kjer --- libnetwork/drivers/bridge/port_mapping.go | 9 +- libnetwork/libnetwork_test.go | 10 +- libnetwork/portallocator/portallocator.go | 100 ++++++++++++++---- .../portallocator/portallocator_test.go | 66 ++++++++++++ libnetwork/portmapper/mapper.go | 9 +- libnetwork/types/types.go | 25 +++-- 6 files changed, 179 insertions(+), 40 deletions(-) diff --git a/libnetwork/drivers/bridge/port_mapping.go b/libnetwork/drivers/bridge/port_mapping.go index b102132190..4dab8a0c89 100644 --- a/libnetwork/drivers/bridge/port_mapping.go +++ b/libnetwork/drivers/bridge/port_mapping.go @@ -57,6 +57,11 @@ func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHos bnd.HostIP = defHostIP } + // Adjust HostPortEnd if this is not a range. + if bnd.HostPortEnd == 0 { + bnd.HostPortEnd = bnd.HostPort + } + // Construct the container side transport address container, err := bnd.ContainerAddr() if err != nil { @@ -65,12 +70,12 @@ func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHos // Try up to maxAllocatePortAttempts times to get a port that's not already allocated. for i := 0; i < maxAllocatePortAttempts; i++ { - if host, err = n.portMapper.Map(container, bnd.HostIP, int(bnd.HostPort), ulPxyEnabled); err == nil { + if host, err = n.portMapper.MapRange(container, bnd.HostIP, int(bnd.HostPort), int(bnd.HostPortEnd), ulPxyEnabled); err == nil { break } // There is no point in immediately retrying to map an explicitly chosen port. if bnd.HostPort != 0 { - logrus.Warnf("Failed to allocate and map port %d: %s", bnd.HostPort, err) + logrus.Warnf("Failed to allocate and map port %d-%d: %s", bnd.HostPort, bnd.HostPortEnd, err) break } logrus.Warnf("Failed to allocate and map port: %s, retry: %d", err, i+1) diff --git a/libnetwork/libnetwork_test.go b/libnetwork/libnetwork_test.go index 3d4a8b0eb3..f2f9407cf9 100644 --- a/libnetwork/libnetwork_test.go +++ b/libnetwork/libnetwork_test.go @@ -89,9 +89,11 @@ func getEmptyGenericOption() map[string]interface{} { func getPortMapping() []types.PortBinding { return []types.PortBinding{ - types.PortBinding{Proto: types.TCP, Port: uint16(230), HostPort: uint16(23000)}, - types.PortBinding{Proto: types.UDP, Port: uint16(200), HostPort: uint16(22000)}, - types.PortBinding{Proto: types.TCP, Port: uint16(120), HostPort: uint16(12000)}, + {Proto: types.TCP, Port: uint16(230), HostPort: uint16(23000)}, + {Proto: types.UDP, Port: uint16(200), HostPort: uint16(22000)}, + {Proto: types.TCP, Port: uint16(120), HostPort: uint16(12000)}, + {Proto: types.TCP, Port: uint16(320), HostPort: uint16(32000), HostPortEnd: uint16(32999)}, + {Proto: types.UDP, Port: uint16(420), HostPort: uint16(42000), HostPortEnd: uint16(42001)}, } } @@ -279,7 +281,7 @@ func TestBridge(t *testing.T) { if !ok { t.Fatalf("Unexpected format for port mapping in endpoint operational data") } - if len(pm) != 3 { + if len(pm) != 5 { t.Fatalf("Incomplete data for port mapping in endpoint operational data: %d", len(pm)) } diff --git a/libnetwork/portallocator/portallocator.go b/libnetwork/portallocator/portallocator.go index 37d9c769b4..240e94fc82 100644 --- a/libnetwork/portallocator/portallocator.go +++ b/libnetwork/portallocator/portallocator.go @@ -70,10 +70,15 @@ type ( Begin int End int } + portRange struct { + begin int + end int + last int + } portMap struct { - p map[int]struct{} - begin, end int - last int + p map[int]struct{} + defaultRange string + portRanges map[string]*portRange } protoMap map[string]*portMap ) @@ -123,8 +128,17 @@ func getDynamicPortRange() (start int, end int, err error) { // RequestPort requests new port from global ports pool for specified ip and proto. // If port is 0 it returns first free port. Otherwise it checks port availability -// in pool and return that port or error if port is already busy. +// in proto's pool and returns that port or error if port is already busy. func (p *PortAllocator) RequestPort(ip net.IP, proto string, port int) (int, error) { + return p.RequestPortInRange(ip, proto, port, port) +} + +// RequestPortInRange requests new port from global ports pool for specified ip and proto. +// If portStart and portEnd are 0 it returns the first free port in the default ephemeral range. +// If portStart != portEnd it returns the first free port in the requested range. +// Otherwise (portStart == portEnd) it checks port availability in the requested proto's port-pool +// and returns that port or error if port is already busy. +func (p *PortAllocator) RequestPortInRange(ip net.IP, proto string, portStart, portEnd int) (int, error) { p.mutex.Lock() defer p.mutex.Unlock() @@ -146,15 +160,15 @@ func (p *PortAllocator) RequestPort(ip net.IP, proto string, port int) (int, err p.ipMap[ipstr] = protomap } mapping := protomap[proto] - if port > 0 { - if _, ok := mapping.p[port]; !ok { - mapping.p[port] = struct{}{} - return port, nil + if portStart > 0 && portStart == portEnd { + if _, ok := mapping.p[portStart]; !ok { + mapping.p[portStart] = struct{}{} + return portStart, nil } - return 0, newErrPortAlreadyAllocated(ipstr, port) + return 0, newErrPortAlreadyAllocated(ipstr, portStart) } - port, err := mapping.findPort() + port, err := mapping.findPort(portStart, portEnd) if err != nil { return 0, err } @@ -178,12 +192,15 @@ func (p *PortAllocator) ReleasePort(ip net.IP, proto string, port int) error { } func (p *PortAllocator) newPortMap() *portMap { - return &portMap{ - p: map[int]struct{}{}, - begin: p.Begin, - end: p.End, - last: p.End, + defaultKey := getRangeKey(p.Begin, p.End) + pm := &portMap{ + p: map[int]struct{}{}, + defaultRange: defaultKey, + portRanges: map[string]*portRange{ + defaultKey: newPortRange(p.Begin, p.End), + }, } + return pm } // ReleaseAll releases all ports for all ips. @@ -194,17 +211,58 @@ func (p *PortAllocator) ReleaseAll() error { return nil } -func (pm *portMap) findPort() (int, error) { - port := pm.last - for i := 0; i <= pm.end-pm.begin; i++ { +func getRangeKey(portStart, portEnd int) string { + return fmt.Sprintf("%d-%d", portStart, portEnd) +} + +func newPortRange(portStart, portEnd int) *portRange { + return &portRange{ + begin: portStart, + end: portEnd, + last: portEnd, + } +} + +func (pm *portMap) getPortRange(portStart, portEnd int) (*portRange, error) { + var key string + if portStart == 0 && portEnd == 0 { + key = pm.defaultRange + } else { + key = getRangeKey(portStart, portEnd) + if portStart == portEnd || + portStart == 0 || portEnd == 0 || + portEnd < portStart { + return nil, fmt.Errorf("invalid port range: %s", key) + } + } + + // Return existing port range, if already known. + if pr, exists := pm.portRanges[key]; exists { + return pr, nil + } + + // Otherwise create a new port range. + pr := newPortRange(portStart, portEnd) + pm.portRanges[key] = pr + return pr, nil +} + +func (pm *portMap) findPort(portStart, portEnd int) (int, error) { + pr, err := pm.getPortRange(portStart, portEnd) + if err != nil { + return 0, err + } + port := pr.last + + for i := 0; i <= pr.end-pr.begin; i++ { port++ - if port > pm.end { - port = pm.begin + if port > pr.end { + port = pr.begin } if _, ok := pm.p[port]; !ok { pm.p[port] = struct{}{} - pm.last = port + pr.last = port return port, nil } } diff --git a/libnetwork/portallocator/portallocator_test.go b/libnetwork/portallocator/portallocator_test.go index 20756494af..ffabb778ac 100644 --- a/libnetwork/portallocator/portallocator_test.go +++ b/libnetwork/portallocator/portallocator_test.go @@ -236,6 +236,72 @@ func TestPortAllocation(t *testing.T) { } } +func TestPortAllocationWithCustomRange(t *testing.T) { + p := Get() + defer resetPortAllocator() + + start, end := 8081, 8082 + specificPort := 8000 + + //get an ephemeral port. + port1, err := p.RequestPortInRange(defaultIP, "tcp", 0, 0) + if err != nil { + t.Fatal(err) + } + + //request invalid ranges + if _, err := p.RequestPortInRange(defaultIP, "tcp", 0, end); err == nil { + t.Fatalf("Expected error for invalid range %d-%d", 0, end) + } + if _, err := p.RequestPortInRange(defaultIP, "tcp", start, 0); err == nil { + t.Fatalf("Expected error for invalid range %d-%d", 0, end) + } + if _, err := p.RequestPortInRange(defaultIP, "tcp", 8081, 8080); err == nil { + t.Fatalf("Expected error for invalid range %d-%d", 0, end) + } + + //request a single port + port, err := p.RequestPortInRange(defaultIP, "tcp", specificPort, specificPort) + if err != nil { + t.Fatal(err) + } + if port != specificPort { + t.Fatalf("Expected port %d, got %d", specificPort, port) + } + + //get a port from the range + port2, err := p.RequestPortInRange(defaultIP, "tcp", start, end) + if err != nil { + t.Fatal(err) + } + if port2 < start || port2 > end { + t.Fatalf("Expected a port between %d and %d, got %d", start, end, port2) + } + //get another ephemeral port (should be > port1) + port3, err := p.RequestPortInRange(defaultIP, "tcp", 0, 0) + if err != nil { + t.Fatal(err) + } + if port3 < port1 { + t.Fatalf("Expected new port > %d in the ephemeral range, got %d", port1, port3) + } + //get another (and in this case the only other) port from the range + port4, err := p.RequestPortInRange(defaultIP, "tcp", start, end) + if err != nil { + t.Fatal(err) + } + if port4 < start || port4 > end { + t.Fatalf("Expected a port between %d and %d, got %d", start, end, port4) + } + if port4 == port2 { + t.Fatal("Allocated the same port from a custom range") + } + //request 3rd port from the range of 2 + if _, err := p.RequestPortInRange(defaultIP, "tcp", start, end); err != ErrAllPortsAllocated { + t.Fatalf("Expected error %s got %s", ErrAllPortsAllocated, err) + } +} + func TestNoDuplicateBPR(t *testing.T) { p := Get() defer resetPortAllocator() diff --git a/libnetwork/portmapper/mapper.go b/libnetwork/portmapper/mapper.go index 48a19053ef..bbdedaa3ca 100644 --- a/libnetwork/portmapper/mapper.go +++ b/libnetwork/portmapper/mapper.go @@ -62,6 +62,11 @@ func (pm *PortMapper) SetIptablesChain(c *iptables.ChainInfo, bridgeName string) // Map maps the specified container transport address to the host's network address and transport port func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, useProxy bool) (host net.Addr, err error) { + return pm.MapRange(container, hostIP, hostPort, hostPort, useProxy) +} + +// MapRange maps the specified container transport address to the host's network address and transport port range +func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, hostPortEnd int, useProxy bool) (host net.Addr, err error) { pm.lock.Lock() defer pm.lock.Unlock() @@ -74,7 +79,7 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, usePr switch container.(type) { case *net.TCPAddr: proto = "tcp" - if allocatedHostPort, err = pm.Allocator.RequestPort(hostIP, proto, hostPort); err != nil { + if allocatedHostPort, err = pm.Allocator.RequestPortInRange(hostIP, proto, hostPortStart, hostPortEnd); err != nil { return nil, err } @@ -91,7 +96,7 @@ func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, usePr } case *net.UDPAddr: proto = "udp" - if allocatedHostPort, err = pm.Allocator.RequestPort(hostIP, proto, hostPort); err != nil { + if allocatedHostPort, err = pm.Allocator.RequestPortInRange(hostIP, proto, hostPortStart, hostPortEnd); err != nil { return nil, err } diff --git a/libnetwork/types/types.go b/libnetwork/types/types.go index fa13bdeacc..3f394baa19 100644 --- a/libnetwork/types/types.go +++ b/libnetwork/types/types.go @@ -24,11 +24,12 @@ func (t *TransportPort) GetCopy() TransportPort { // PortBinding represent a port binding between the container and the host type PortBinding struct { - Proto Protocol - IP net.IP - Port uint16 - HostIP net.IP - HostPort uint16 + Proto Protocol + IP net.IP + Port uint16 + HostIP net.IP + HostPort uint16 + HostPortEnd uint16 } // HostAddr returns the host side transport address @@ -58,11 +59,12 @@ func (p PortBinding) ContainerAddr() (net.Addr, error) { // GetCopy returns a copy of this PortBinding structure instance func (p *PortBinding) GetCopy() PortBinding { return PortBinding{ - Proto: p.Proto, - IP: GetIPCopy(p.IP), - Port: p.Port, - HostIP: GetIPCopy(p.HostIP), - HostPort: p.HostPort, + Proto: p.Proto, + IP: GetIPCopy(p.IP), + Port: p.Port, + HostIP: GetIPCopy(p.HostIP), + HostPort: p.HostPort, + HostPortEnd: p.HostPortEnd, } } @@ -76,7 +78,8 @@ func (p *PortBinding) Equal(o *PortBinding) bool { return false } - if p.Proto != o.Proto || p.Port != o.Port || p.HostPort != o.HostPort { + if p.Proto != o.Proto || p.Port != o.Port || + p.HostPort != o.HostPort || p.HostPortEnd != o.HostPortEnd { return false }