From 799ffa176399877e610ff2049dbf84610037a2be Mon Sep 17 00:00:00 2001 From: Andrea Luzzardi Date: Thu, 28 Feb 2013 11:50:02 -0800 Subject: [PATCH] Network: Port mapping support. Implemented a port allocator and a port mapper that is able to forward TCP ports from the host to the container. --- network.go | 290 ++++++++++++++++++++++++++++++++++++++---------- network_test.go | 18 +-- 2 files changed, 244 insertions(+), 64 deletions(-) diff --git a/network.go b/network.go index db4048feb5..1b6395b0b1 100644 --- a/network.go +++ b/network.go @@ -5,20 +5,20 @@ import ( "encoding/binary" "errors" "fmt" + "log" "net" + "os/exec" + "strconv" + "strings" ) const ( networkBridgeIface = "lxcbr0" + portRangeStart = 49153 + portRangeEnd = 65535 ) -type NetworkInterface struct { - IPNet net.IPNet - Gateway net.IP -} - -// IP utils - +// Calculates the first and last IP addresses in an IPNet func networkRange(network *net.IPNet) (net.IP, net.IP) { netIP := network.IP.To4() firstIP := netIP.Mask(network.Mask) @@ -29,6 +29,7 @@ func networkRange(network *net.IPNet) (net.IP, net.IP) { return firstIP, lastIP } +// Converts a 4 bytes IP into a 32 bit integer func ipToInt(ip net.IP) (int32, error) { buf := bytes.NewBuffer(ip.To4()) var n int32 @@ -38,6 +39,7 @@ func ipToInt(ip net.IP) (int32, error) { return n, nil } +// Converts 32 bit integer into a 4 bytes IP address func intToIp(n int32) (net.IP, error) { var buf bytes.Buffer if err := binary.Write(&buf, binary.BigEndian, &n); err != nil { @@ -50,6 +52,7 @@ func intToIp(n int32) (net.IP, error) { return ip, nil } +// Given a netmask, calculates the number of available hosts func networkSize(mask net.IPMask) (int32, error) { m := net.IPv4Mask(0, 0, 0, 0) for i := 0; i < net.IPv4len; i++ { @@ -63,6 +66,15 @@ func networkSize(mask net.IPMask) (int32, error) { return n + 1, nil } +// Wrapper around the iptables command +func iptables(args ...string) error { + if err := exec.Command("/sbin/iptables", args...).Run(); err != nil { + return fmt.Errorf("iptables failed: iptables %v", strings.Join(args, " ")) + } + return nil +} + +// Return the IPv4 address of a network interface func getIfaceAddr(name string) (net.Addr, error) { iface, err := net.InterfaceByName(name) if err != nil { @@ -81,60 +93,122 @@ func getIfaceAddr(name string) (net.Addr, error) { } switch { case len(addrs4) == 0: - return nil, fmt.Errorf("Bridge %v has no IP addresses", name) + return nil, fmt.Errorf("Interface %v has no IP addresses", name) case len(addrs4) > 1: - return nil, fmt.Errorf("Bridge %v has more than 1 IPv4 address", name) + return nil, fmt.Errorf("Interface %v has more than 1 IPv4 address", name) } return addrs4[0], nil } -// Network allocator -func newNetworkAllocator(iface string) (*NetworkAllocator, error) { - addr, err := getIfaceAddr(iface) - if err != nil { - return nil, err - } - network := addr.(*net.IPNet) - - alloc := &NetworkAllocator{ - iface: iface, - net: network, - } - if err := alloc.populateFromNetwork(network); err != nil { - return nil, err - } - return alloc, nil +// Port mapper takes care of mapping external ports to containers by setting +// up iptables rules. +// It keeps track of all mappings and is able to unmap at will +type PortMapper struct { + mapping map[int]net.TCPAddr } -type NetworkAllocator struct { - iface string - net *net.IPNet - queue chan (net.IP) +func (mapper *PortMapper) cleanup() error { + // Ignore errors - This could mean the chains were never set up + iptables("-t", "nat", "-D", "PREROUTING", "-j", "DOCKER") + iptables("-t", "nat", "-F", "DOCKER") + iptables("-t", "nat", "-X", "DOCKER") + mapper.mapping = make(map[int]net.TCPAddr) + return nil } -func (alloc *NetworkAllocator) acquireIP() (net.IP, error) { - select { - case ip := <-alloc.queue: - return ip, nil - default: - return net.IP{}, errors.New("No more IP addresses available") +func (mapper *PortMapper) setup() error { + if err := iptables("-t", "nat", "-N", "DOCKER"); err != nil { + return errors.New("Unable to setup port networking: Failed to create DOCKER chain") } - return net.IP{}, nil -} - -func (alloc *NetworkAllocator) releaseIP(ip net.IP) error { - select { - case alloc.queue <- ip: - return nil - default: - return errors.New("Too many IP addresses have been released") + if err := iptables("-t", "nat", "-A", "PREROUTING", "-j", "DOCKER"); err != nil { + return errors.New("Unable to setup port networking: Failed to inject docker in PREROUTING chain") } return nil } -func (alloc *NetworkAllocator) populateFromNetwork(network *net.IPNet) error { - firstIP, _ := networkRange(network) - size, err := networkSize(network.Mask) +func (mapper *PortMapper) iptablesForward(rule string, port int, dest net.TCPAddr) error { + return iptables("-t", "nat", rule, "DOCKER", "-p", "tcp", "--dport", strconv.Itoa(port), + "-j", "DNAT", "--to-destination", net.JoinHostPort(dest.IP.String(), strconv.Itoa(dest.Port))) +} + +func (mapper *PortMapper) Map(port int, dest net.TCPAddr) error { + if err := mapper.iptablesForward("-A", port, dest); err != nil { + return err + } + mapper.mapping[port] = dest + return nil +} + +func (mapper *PortMapper) Unmap(port int) error { + dest, ok := mapper.mapping[port] + if !ok { + return errors.New("Port is not mapped") + } + if err := mapper.iptablesForward("-D", port, dest); err != nil { + return err + } + delete(mapper.mapping, port) + return nil +} + +func newPortMapper() (*PortMapper, error) { + mapper := &PortMapper{} + if err := mapper.cleanup(); err != nil { + return nil, err + } + if err := mapper.setup(); err != nil { + return nil, err + } + return mapper, nil +} + +// Port allocator: Atomatically allocate and release networking ports +type PortAllocator struct { + ports chan (int) +} + +func (alloc *PortAllocator) populate(start, end int) { + alloc.ports = make(chan int, end-start) + for port := start; port < end; port++ { + alloc.ports <- port + } +} + +func (alloc *PortAllocator) Acquire() (int, error) { + select { + case port := <-alloc.ports: + return port, nil + default: + return -1, errors.New("No more ports available") + } + return -1, nil +} + +func (alloc *PortAllocator) Release(port int) error { + select { + case alloc.ports <- port: + return nil + default: + return errors.New("Too many ports have been released") + } + return nil +} + +func newPortAllocator(start, end int) (*PortAllocator, error) { + allocator := &PortAllocator{} + allocator.populate(start, end) + return allocator, nil +} + +// IP allocator: Atomatically allocate and release networking ports +type IPAllocator struct { + network *net.IPNet + queue chan (net.IP) +} + +func (alloc *IPAllocator) populate() error { + firstIP, _ := networkRange(alloc.network) + size, err := networkSize(alloc.network.Mask) if err != nil { return err } @@ -152,27 +226,131 @@ func (alloc *NetworkAllocator) populateFromNetwork(network *net.IPNet) error { return err } // Discard the network IP (that's the host IP address) - if ip.Equal(network.IP) { + if ip.Equal(alloc.network.IP) { continue } - alloc.releaseIP(ip) + alloc.queue <- ip } return nil } -func (alloc *NetworkAllocator) Allocate() (*NetworkInterface, error) { - // ipPrefixLen, _ := alloc.net.Mask.Size() - ip, err := alloc.acquireIP() +func (alloc *IPAllocator) Acquire() (net.IP, error) { + select { + case ip := <-alloc.queue: + return ip, nil + default: + return net.IP{}, errors.New("No more IP addresses available") + } + return net.IP{}, nil +} + +func (alloc *IPAllocator) Release(ip net.IP) error { + select { + case alloc.queue <- ip: + return nil + default: + return errors.New("Too many IP addresses have been released") + } + return nil +} + +func newIPAllocator(network *net.IPNet) (*IPAllocator, error) { + alloc := &IPAllocator{ + network: network, + } + if err := alloc.populate(); err != nil { + return nil, err + } + return alloc, nil +} + +// Network interface represents the networking stack of a container +type NetworkInterface struct { + IPNet net.IPNet + Gateway net.IP + + manager *NetworkManager + extPorts []int +} + +// Allocate an external TCP port and map it to the interface +func (iface *NetworkInterface) AllocatePort(port int) (int, error) { + extPort, err := iface.manager.portAllocator.Acquire() + if err != nil { + return -1, err + } + if err := iface.manager.portMapper.Map(extPort, net.TCPAddr{iface.IPNet.IP, port}); err != nil { + iface.manager.portAllocator.Release(extPort) + return -1, err + } + iface.extPorts = append(iface.extPorts, extPort) + return extPort, nil +} + +// Release: Network cleanup - release all resources +func (iface *NetworkInterface) Release() error { + for _, port := range iface.extPorts { + if err := iface.manager.portMapper.Unmap(port); err != nil { + log.Printf("Unable to unmap port %v: %v", port, err) + } + if err := iface.manager.portAllocator.Release(port); err != nil { + log.Printf("Unable to release port %v: %v", port, err) + } + + } + return iface.manager.ipAllocator.Release(iface.IPNet.IP) +} + +// Network Manager manages a set of network interfaces +// Only *one* manager per host machine should be used +type NetworkManager struct { + bridgeIface string + bridgeNetwork *net.IPNet + + ipAllocator *IPAllocator + portAllocator *PortAllocator + portMapper *PortMapper +} + +// Allocate a network interface +func (manager *NetworkManager) Allocate() (*NetworkInterface, error) { + ip, err := manager.ipAllocator.Acquire() if err != nil { return nil, err } iface := &NetworkInterface{ - IPNet: net.IPNet{ip, alloc.net.Mask}, - Gateway: alloc.net.IP, + IPNet: net.IPNet{ip, manager.bridgeNetwork.Mask}, + Gateway: manager.bridgeNetwork.IP, + manager: manager, } return iface, nil } -func (alloc *NetworkAllocator) Release(iface *NetworkInterface) error { - return alloc.releaseIP(iface.IPNet.IP) +func newNetworkManager(bridgeIface string) (*NetworkManager, error) { + addr, err := getIfaceAddr(bridgeIface) + if err != nil { + return nil, err + } + network := addr.(*net.IPNet) + + ipAllocator, err := newIPAllocator(network) + if err != nil { + return nil, err + } + + portAllocator, err := newPortAllocator(portRangeStart, portRangeEnd) + if err != nil { + return nil, err + } + + portMapper, err := newPortMapper() + + manager := &NetworkManager{ + bridgeIface: bridgeIface, + bridgeNetwork: network, + ipAllocator: ipAllocator, + portAllocator: portAllocator, + portMapper: portMapper, + } + return manager, nil } diff --git a/network_test.go b/network_test.go index d66e7e5393..c456b54838 100644 --- a/network_test.go +++ b/network_test.go @@ -100,25 +100,27 @@ func TestConversion(t *testing.T) { } } -func TestNetworkAllocator(t *testing.T) { - alloc := NetworkAllocator{} - _, n, _ := net.ParseCIDR("127.0.0.1/29") - alloc.populateFromNetwork(n) +func TestIPAllocator(t *testing.T) { + gwIP, n, _ := net.ParseCIDR("127.0.0.1/29") + alloc, err := newIPAllocator(&net.IPNet{gwIP, n.Mask}) + if err != nil { + t.Fatal(err) + } var lastIP net.IP for i := 0; i < 5; i++ { - ip, err := alloc.acquireIP() + ip, err := alloc.Acquire() if err != nil { t.Fatal(err) } lastIP = ip } - ip, err := alloc.acquireIP() + ip, err := alloc.Acquire() if err == nil { t.Fatal("There shouldn't be any IP addresses at this point") } // Release 1 IP - alloc.releaseIP(lastIP) - ip, err = alloc.acquireIP() + alloc.Release(lastIP) + ip, err = alloc.Acquire() if err != nil { t.Fatal(err) }