From 68d223a0e014c85f2078560238c94968dd45b64a Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Tue, 24 Mar 2015 13:56:52 -0700 Subject: [PATCH] Issue #18: IP Allocator rework - Move ipallocator package into libnetwork - Also ported network utility functions and their tests in libnetwork: docker/daemon/networkdriver/utilg.go => libnetwork/utils.go docker/daemon/networkdriver/network_test.go => libnetwork/utils_test.go - Changed drivers/setup_device.go and setup_ipv4.go to reuse functions in utils.go, instead of redefining internally. - Modified utils to use vishvananda/netlink instead of libcontainer/netlink Signed-off-by: Alessandro Boch --- libnetwork/drivers/bridge/setup_device.go | 15 +- .../drivers/bridge/setup_device_test.go | 4 +- .../drivers/bridge/setup_fixedcidrv4.go | 2 +- .../drivers/bridge/setup_fixedcidrv4_test.go | 2 +- .../drivers/bridge/setup_fixedcidrv6.go | 2 +- .../drivers/bridge/setup_fixedcidrv6_test.go | 2 +- libnetwork/drivers/bridge/setup_ipv4.go | 62 +- libnetwork/ipallocator/allocator.go | 167 +++++ libnetwork/ipallocator/allocator_test.go | 681 ++++++++++++++++++ libnetwork/utils.go | 127 ++++ libnetwork/utils_test.go | 176 +++++ 11 files changed, 1163 insertions(+), 77 deletions(-) create mode 100644 libnetwork/ipallocator/allocator.go create mode 100644 libnetwork/ipallocator/allocator_test.go create mode 100644 libnetwork/utils.go create mode 100644 libnetwork/utils_test.go diff --git a/libnetwork/drivers/bridge/setup_device.go b/libnetwork/drivers/bridge/setup_device.go index a2f9e49aae..bb947f660f 100644 --- a/libnetwork/drivers/bridge/setup_device.go +++ b/libnetwork/drivers/bridge/setup_device.go @@ -2,11 +2,10 @@ package bridge import ( "fmt" - "math/rand" - "net" log "github.com/Sirupsen/logrus" "github.com/docker/docker/pkg/parsers/kernel" + "github.com/docker/libnetwork" "github.com/vishvananda/netlink" ) @@ -29,7 +28,7 @@ func setupDevice(i *bridgeInterface) error { // was not supported before that. kv, err := kernel.GetKernelVersion() if err == nil && (kv.Kernel >= 3 && kv.Major >= 3) { - i.Link.Attrs().HardwareAddr = generateRandomMAC() + i.Link.Attrs().HardwareAddr = libnetwork.GenerateRandomMAC() log.Debugf("Setting bridge mac address to %s", i.Link.Attrs().HardwareAddr) } @@ -51,13 +50,3 @@ func setupDeviceUp(i *bridgeInterface) error { } return nil } - -func generateRandomMAC() net.HardwareAddr { - hw := make(net.HardwareAddr, 6) - for i := 0; i < 6; i++ { - hw[i] = byte(rand.Intn(255)) - } - hw[0] &^= 0x1 // clear multicast bit - hw[0] |= 0x2 // set local assignment bit (IEEE802) - return hw -} diff --git a/libnetwork/drivers/bridge/setup_device_test.go b/libnetwork/drivers/bridge/setup_device_test.go index 4768a32d26..04fd27cb30 100644 --- a/libnetwork/drivers/bridge/setup_device_test.go +++ b/libnetwork/drivers/bridge/setup_device_test.go @@ -69,8 +69,8 @@ func TestSetupDeviceUp(t *testing.T) { func TestGenerateRandomMAC(t *testing.T) { defer libnetwork.SetupTestNetNS(t)() - mac1 := generateRandomMAC() - mac2 := generateRandomMAC() + mac1 := libnetwork.GenerateRandomMAC() + mac2 := libnetwork.GenerateRandomMAC() if bytes.Compare(mac1, mac2) == 0 { t.Fatalf("Generated twice the same MAC address %v", mac1) } diff --git a/libnetwork/drivers/bridge/setup_fixedcidrv4.go b/libnetwork/drivers/bridge/setup_fixedcidrv4.go index e320ac9b23..ee94d2dedc 100644 --- a/libnetwork/drivers/bridge/setup_fixedcidrv4.go +++ b/libnetwork/drivers/bridge/setup_fixedcidrv4.go @@ -4,7 +4,7 @@ import ( "fmt" log "github.com/Sirupsen/logrus" - "github.com/docker/docker/daemon/networkdriver/ipallocator" + "github.com/docker/libnetwork/ipallocator" ) func setupFixedCIDRv4(i *bridgeInterface) error { diff --git a/libnetwork/drivers/bridge/setup_fixedcidrv4_test.go b/libnetwork/drivers/bridge/setup_fixedcidrv4_test.go index 36872c7770..964f339861 100644 --- a/libnetwork/drivers/bridge/setup_fixedcidrv4_test.go +++ b/libnetwork/drivers/bridge/setup_fixedcidrv4_test.go @@ -4,8 +4,8 @@ import ( "net" "testing" - "github.com/docker/docker/daemon/networkdriver/ipallocator" "github.com/docker/libnetwork" + "github.com/docker/libnetwork/ipallocator" ) func TestSetupFixedCIDRv4(t *testing.T) { diff --git a/libnetwork/drivers/bridge/setup_fixedcidrv6.go b/libnetwork/drivers/bridge/setup_fixedcidrv6.go index 0c29ec291c..d272deda95 100644 --- a/libnetwork/drivers/bridge/setup_fixedcidrv6.go +++ b/libnetwork/drivers/bridge/setup_fixedcidrv6.go @@ -4,7 +4,7 @@ import ( "fmt" log "github.com/Sirupsen/logrus" - "github.com/docker/docker/daemon/networkdriver/ipallocator" + "github.com/docker/libnetwork/ipallocator" ) func setupFixedCIDRv6(i *bridgeInterface) error { diff --git a/libnetwork/drivers/bridge/setup_fixedcidrv6_test.go b/libnetwork/drivers/bridge/setup_fixedcidrv6_test.go index 1f3031af88..2cc3534cda 100644 --- a/libnetwork/drivers/bridge/setup_fixedcidrv6_test.go +++ b/libnetwork/drivers/bridge/setup_fixedcidrv6_test.go @@ -4,8 +4,8 @@ import ( "net" "testing" - "github.com/docker/docker/daemon/networkdriver/ipallocator" "github.com/docker/libnetwork" + "github.com/docker/libnetwork/ipallocator" ) func TestSetupFixedCIDRv6(t *testing.T) { diff --git a/libnetwork/drivers/bridge/setup_ipv4.go b/libnetwork/drivers/bridge/setup_ipv4.go index e0a13362f7..bfeb809b22 100644 --- a/libnetwork/drivers/bridge/setup_ipv4.go +++ b/libnetwork/drivers/bridge/setup_ipv4.go @@ -5,6 +5,7 @@ import ( "net" log "github.com/Sirupsen/logrus" + "github.com/docker/libnetwork" "github.com/vishvananda/netlink" ) @@ -70,67 +71,12 @@ func electBridgeIPv4(config *Configuration) (*net.IPNet, error) { // Try to automatically elect appropriate brige IPv4 settings. for _, n := range bridgeNetworks { - if err := checkNameserverOverlaps(nameservers, n); err == nil { - if err := checkRouteOverlaps(n); err == nil { + if err := libnetwork.CheckNameserverOverlaps(nameservers, n); err == nil { + if err := libnetwork.CheckRouteOverlaps(n); err == nil { return n, nil } } } - return nil, fmt.Errorf("Couldn't find an address range for interface %q", config.BridgeName) -} - -func checkNameserverOverlaps(nameservers []string, toCheck *net.IPNet) error { - for _, ns := range nameservers { - _, nsNetwork, err := net.ParseCIDR(ns) - if err != nil { - return err - } - if networkOverlaps(toCheck, nsNetwork) { - return fmt.Errorf("Requested network %s overlaps with name server", toCheck.String()) - } - } - return nil -} - -func checkRouteOverlaps(toCheck *net.IPNet) error { - networks, err := netlink.RouteList(nil, netlink.FAMILY_V4) - if err != nil { - return err - } - - for _, network := range networks { - // TODO Is that right? - if network.Dst != nil && networkOverlaps(toCheck, network.Dst) { - return fmt.Errorf("Requested network %s overlaps with an existing network", toCheck.String()) - } - } - return nil -} - -func networkOverlaps(netX *net.IPNet, netY *net.IPNet) bool { - if firstIP, _ := networkRange(netX); netY.Contains(firstIP) { - return true - } - if firstIP, _ := networkRange(netY); netX.Contains(firstIP) { - return true - } - return false -} - -func networkRange(network *net.IPNet) (net.IP, net.IP) { - var netIP net.IP - if network.IP.To4() != nil { - netIP = network.IP.To4() - } else if network.IP.To16() != nil { - netIP = network.IP.To16() - } else { - return nil, nil - } - - lastIP := make([]byte, len(netIP), len(netIP)) - for i := 0; i < len(netIP); i++ { - lastIP[i] = netIP[i] | ^network.Mask[i] - } - return netIP.Mask(network.Mask), net.IP(lastIP) + return nil, fmt.Errorf("'t find an address range for interface %q", config.BridgeName) } diff --git a/libnetwork/ipallocator/allocator.go b/libnetwork/ipallocator/allocator.go new file mode 100644 index 0000000000..7f0c566a85 --- /dev/null +++ b/libnetwork/ipallocator/allocator.go @@ -0,0 +1,167 @@ +// Package ipallocator defines the default IP allocator. It will move out of libnetwork as an external IPAM plugin. +// This has been imported unchanged from Docker, besides additon of registration logic +package ipallocator + +import ( + "errors" + "math/big" + "net" + "sync" + + log "github.com/Sirupsen/logrus" + "github.com/docker/libnetwork" +) + +// allocatedMap is thread-unsafe set of allocated IP +type allocatedMap struct { + p map[string]struct{} + last *big.Int + begin *big.Int + end *big.Int +} + +func newAllocatedMap(network *net.IPNet) *allocatedMap { + firstIP, lastIP := libnetwork.NetworkRange(network) + begin := big.NewInt(0).Add(ipToBigInt(firstIP), big.NewInt(1)) + end := big.NewInt(0).Sub(ipToBigInt(lastIP), big.NewInt(1)) + + return &allocatedMap{ + p: make(map[string]struct{}), + begin: begin, + end: end, + last: big.NewInt(0).Sub(begin, big.NewInt(1)), // so first allocated will be begin + } +} + +type networkSet map[string]*allocatedMap + +var ( + // ErrNoAvailableIPs preformatted error + ErrNoAvailableIPs = errors.New("no available ip addresses on network") + // ErrIPAlreadyAllocated preformatted error + ErrIPAlreadyAllocated = errors.New("ip already allocated") + // ErrIPOutOfRange preformatted error + ErrIPOutOfRange = errors.New("requested ip is out of range") + // ErrNetworkAlreadyRegistered preformatted error + ErrNetworkAlreadyRegistered = errors.New("network already registered") + // ErrBadSubnet preformatted error + ErrBadSubnet = errors.New("network does not contain specified subnet") +) + +var ( + lock = sync.Mutex{} + allocatedIPs = networkSet{} +) + +// RegisterSubnet registers network in global allocator with bounds +// defined by subnet. If you want to use network range you must call +// this method before first RequestIP, otherwise full network range will be used +func RegisterSubnet(network *net.IPNet, subnet *net.IPNet) error { + lock.Lock() + defer lock.Unlock() + key := network.String() + if _, ok := allocatedIPs[key]; ok { + return ErrNetworkAlreadyRegistered + } + n := newAllocatedMap(network) + beginIP, endIP := libnetwork.NetworkRange(subnet) + begin := big.NewInt(0).Add(ipToBigInt(beginIP), big.NewInt(1)) + end := big.NewInt(0).Sub(ipToBigInt(endIP), big.NewInt(1)) + + // Check that subnet is within network + if !(begin.Cmp(n.begin) >= 0 && end.Cmp(n.end) <= 0 && begin.Cmp(end) == -1) { + return ErrBadSubnet + } + n.begin.Set(begin) + n.end.Set(end) + n.last.Sub(begin, big.NewInt(1)) + allocatedIPs[key] = n + return nil +} + +// RequestIP requests an available ip from the given network. It +// will return the next available ip if the ip provided is nil. If the +// ip provided is not nil it will validate that the provided ip is available +// for use or return an error +func RequestIP(network *net.IPNet, ip net.IP) (net.IP, error) { + lock.Lock() + defer lock.Unlock() + key := network.String() + allocated, ok := allocatedIPs[key] + if !ok { + allocated = newAllocatedMap(network) + allocatedIPs[key] = allocated + } + + if ip == nil { + return allocated.getNextIP() + } + return allocated.checkIP(ip) +} + +// ReleaseIP adds the provided ip back into the pool of +// available ips to be returned for use. +func ReleaseIP(network *net.IPNet, ip net.IP) error { + lock.Lock() + defer lock.Unlock() + if allocated, exists := allocatedIPs[network.String()]; exists { + delete(allocated.p, ip.String()) + } + return nil +} + +func (allocated *allocatedMap) checkIP(ip net.IP) (net.IP, error) { + if _, ok := allocated.p[ip.String()]; ok { + return nil, ErrIPAlreadyAllocated + } + + pos := ipToBigInt(ip) + // Verify that the IP address is within our network range. + if pos.Cmp(allocated.begin) == -1 || pos.Cmp(allocated.end) == 1 { + return nil, ErrIPOutOfRange + } + + // Register the IP. + allocated.p[ip.String()] = struct{}{} + + return ip, nil +} + +// return an available ip if one is currently available. If not, +// return the next available ip for the nextwork +func (allocated *allocatedMap) getNextIP() (net.IP, error) { + pos := big.NewInt(0).Set(allocated.last) + allRange := big.NewInt(0).Sub(allocated.end, allocated.begin) + for i := big.NewInt(0); i.Cmp(allRange) <= 0; i.Add(i, big.NewInt(1)) { + pos.Add(pos, big.NewInt(1)) + if pos.Cmp(allocated.end) == 1 { + pos.Set(allocated.begin) + } + if _, ok := allocated.p[bigIntToIP(pos).String()]; ok { + continue + } + allocated.p[bigIntToIP(pos).String()] = struct{}{} + allocated.last.Set(pos) + return bigIntToIP(pos), nil + } + return nil, ErrNoAvailableIPs +} + +// Converts a 4 bytes IP into a 128 bit integer +func ipToBigInt(ip net.IP) *big.Int { + x := big.NewInt(0) + if ip4 := ip.To4(); ip4 != nil { + return x.SetBytes(ip4) + } + if ip6 := ip.To16(); ip6 != nil { + return x.SetBytes(ip6) + } + + log.Errorf("ipToBigInt: Wrong IP length! %s", ip) + return nil +} + +// Converts 128 bit integer into a 4 bytes IP address +func bigIntToIP(v *big.Int) net.IP { + return net.IP(v.Bytes()) +} diff --git a/libnetwork/ipallocator/allocator_test.go b/libnetwork/ipallocator/allocator_test.go new file mode 100644 index 0000000000..8e0d8fdca6 --- /dev/null +++ b/libnetwork/ipallocator/allocator_test.go @@ -0,0 +1,681 @@ +package ipallocator + +import ( + "fmt" + "math/big" + "net" + "testing" +) + +func reset() { + allocatedIPs = networkSet{} +} + +func TestConversion(t *testing.T) { + ip := net.ParseIP("127.0.0.1") + i := ipToBigInt(ip) + if i.Cmp(big.NewInt(0x7f000001)) != 0 { + t.Fatal("incorrect conversion") + } + conv := bigIntToIP(i) + if !ip.Equal(conv) { + t.Error(conv.String()) + } +} + +func TestConversionIPv6(t *testing.T) { + ip := net.ParseIP("2a00:1450::1") + ip2 := net.ParseIP("2a00:1450::2") + ip3 := net.ParseIP("2a00:1450::1:1") + i := ipToBigInt(ip) + val, success := big.NewInt(0).SetString("2a001450000000000000000000000001", 16) + if !success { + t.Fatal("Hex-String to BigInt conversion failed.") + } + if i.Cmp(val) != 0 { + t.Fatal("incorrent conversion") + } + + conv := bigIntToIP(i) + conv2 := bigIntToIP(big.NewInt(0).Add(i, big.NewInt(1))) + conv3 := bigIntToIP(big.NewInt(0).Add(i, big.NewInt(0x10000))) + + if !ip.Equal(conv) { + t.Error("2a00:1450::1 should be equal to " + conv.String()) + } + if !ip2.Equal(conv2) { + t.Error("2a00:1450::2 should be equal to " + conv2.String()) + } + if !ip3.Equal(conv3) { + t.Error("2a00:1450::1:1 should be equal to " + conv3.String()) + } +} + +func TestRequestNewIps(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + + var ip net.IP + var err error + + for i := 1; i < 10; i++ { + ip, err = RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + if expected := fmt.Sprintf("192.168.0.%d", i); ip.String() != expected { + t.Fatalf("Expected ip %s got %s", expected, ip.String()) + } + } + value := bigIntToIP(big.NewInt(0).Add(ipToBigInt(ip), big.NewInt(1))).String() + if err := ReleaseIP(network, ip); err != nil { + t.Fatal(err) + } + ip, err = RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + if ip.String() != value { + t.Fatalf("Expected to receive the next ip %s got %s", value, ip.String()) + } +} + +func TestRequestNewIpV6(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{0x2a, 0x00, 0x14, 0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + Mask: []byte{255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0}, // /64 netmask + } + + var ip net.IP + var err error + for i := 1; i < 10; i++ { + ip, err = RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + if expected := fmt.Sprintf("2a00:1450::%d", i); ip.String() != expected { + t.Fatalf("Expected ip %s got %s", expected, ip.String()) + } + } + value := bigIntToIP(big.NewInt(0).Add(ipToBigInt(ip), big.NewInt(1))).String() + if err := ReleaseIP(network, ip); err != nil { + t.Fatal(err) + } + ip, err = RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + if ip.String() != value { + t.Fatalf("Expected to receive the next ip %s got %s", value, ip.String()) + } +} + +func TestReleaseIp(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + + ip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + if err := ReleaseIP(network, ip); err != nil { + t.Fatal(err) + } +} + +func TestReleaseIpV6(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{0x2a, 0x00, 0x14, 0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + Mask: []byte{255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0}, // /64 netmask + } + + ip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + if err := ReleaseIP(network, ip); err != nil { + t.Fatal(err) + } +} + +func TestGetReleasedIp(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + + ip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + value := ip.String() + if err := ReleaseIP(network, ip); err != nil { + t.Fatal(err) + } + + for i := 0; i < 253; i++ { + _, err = RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + err = ReleaseIP(network, ip) + if err != nil { + t.Fatal(err) + } + } + + ip, err = RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + if ip.String() != value { + t.Fatalf("Expected to receive same ip %s got %s", value, ip.String()) + } +} + +func TestGetReleasedIpV6(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{0x2a, 0x00, 0x14, 0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + Mask: []byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 0}, + } + + ip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + value := ip.String() + if err := ReleaseIP(network, ip); err != nil { + t.Fatal(err) + } + + for i := 0; i < 253; i++ { + _, err = RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + err = ReleaseIP(network, ip) + if err != nil { + t.Fatal(err) + } + } + + ip, err = RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + if ip.String() != value { + t.Fatalf("Expected to receive same ip %s got %s", value, ip.String()) + } +} + +func TestRequestSpecificIp(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 224}, + } + + ip := net.ParseIP("192.168.0.5") + + // Request a "good" IP. + if _, err := RequestIP(network, ip); err != nil { + t.Fatal(err) + } + + // Request the same IP again. + if _, err := RequestIP(network, ip); err != ErrIPAlreadyAllocated { + t.Fatalf("Got the same IP twice: %#v", err) + } + + // Request an out of range IP. + if _, err := RequestIP(network, net.ParseIP("192.168.0.42")); err != ErrIPOutOfRange { + t.Fatalf("Got an out of range IP: %#v", err) + } +} + +func TestRequestSpecificIpV6(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{0x2a, 0x00, 0x14, 0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + Mask: []byte{255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0}, // /64 netmask + } + + ip := net.ParseIP("2a00:1450::5") + + // Request a "good" IP. + if _, err := RequestIP(network, ip); err != nil { + t.Fatal(err) + } + + // Request the same IP again. + if _, err := RequestIP(network, ip); err != ErrIPAlreadyAllocated { + t.Fatalf("Got the same IP twice: %#v", err) + } + + // Request an out of range IP. + if _, err := RequestIP(network, net.ParseIP("2a00:1500::1")); err != ErrIPOutOfRange { + t.Fatalf("Got an out of range IP: %#v", err) + } +} + +func TestIPAllocator(t *testing.T) { + expectedIPs := []net.IP{ + 0: net.IPv4(127, 0, 0, 1), + 1: net.IPv4(127, 0, 0, 2), + 2: net.IPv4(127, 0, 0, 3), + 3: net.IPv4(127, 0, 0, 4), + 4: net.IPv4(127, 0, 0, 5), + 5: net.IPv4(127, 0, 0, 6), + } + + gwIP, n, _ := net.ParseCIDR("127.0.0.1/29") + + network := &net.IPNet{IP: gwIP, Mask: n.Mask} + // Pool after initialisation (f = free, u = used) + // 1(f) - 2(f) - 3(f) - 4(f) - 5(f) - 6(f) + // ↑ + + // Check that we get 6 IPs, from 127.0.0.1–127.0.0.6, in that + // order. + for i := 0; i < 6; i++ { + ip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + assertIPEquals(t, expectedIPs[i], ip) + } + // Before loop begin + // 1(f) - 2(f) - 3(f) - 4(f) - 5(f) - 6(f) + // ↑ + + // After i = 0 + // 1(u) - 2(f) - 3(f) - 4(f) - 5(f) - 6(f) + // ↑ + + // After i = 1 + // 1(u) - 2(u) - 3(f) - 4(f) - 5(f) - 6(f) + // ↑ + + // After i = 2 + // 1(u) - 2(u) - 3(u) - 4(f) - 5(f) - 6(f) + // ↑ + + // After i = 3 + // 1(u) - 2(u) - 3(u) - 4(u) - 5(f) - 6(f) + // ↑ + + // After i = 4 + // 1(u) - 2(u) - 3(u) - 4(u) - 5(u) - 6(f) + // ↑ + + // After i = 5 + // 1(u) - 2(u) - 3(u) - 4(u) - 5(u) - 6(u) + // ↑ + + // Check that there are no more IPs + ip, err := RequestIP(network, nil) + if err == nil { + t.Fatalf("There shouldn't be any IP addresses at this point, got %s\n", ip) + } + + // Release some IPs in non-sequential order + if err := ReleaseIP(network, expectedIPs[3]); err != nil { + t.Fatal(err) + } + // 1(u) - 2(u) - 3(u) - 4(f) - 5(u) - 6(u) + // ↑ + + if err := ReleaseIP(network, expectedIPs[2]); err != nil { + t.Fatal(err) + } + // 1(u) - 2(u) - 3(f) - 4(f) - 5(u) - 6(u) + // ↑ + + if err := ReleaseIP(network, expectedIPs[4]); err != nil { + t.Fatal(err) + } + // 1(u) - 2(u) - 3(f) - 4(f) - 5(f) - 6(u) + // ↑ + + // Make sure that IPs are reused in sequential order, starting + // with the first released IP + newIPs := make([]net.IP, 3) + for i := 0; i < 3; i++ { + ip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + newIPs[i] = ip + } + assertIPEquals(t, expectedIPs[2], newIPs[0]) + assertIPEquals(t, expectedIPs[3], newIPs[1]) + assertIPEquals(t, expectedIPs[4], newIPs[2]) + + _, err = RequestIP(network, nil) + if err == nil { + t.Fatal("There shouldn't be any IP addresses at this point") + } +} + +func TestAllocateFirstIP(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 0}, + Mask: []byte{255, 255, 255, 0}, + } + + firstIP := network.IP.To4().Mask(network.Mask) + first := big.NewInt(0).Add(ipToBigInt(firstIP), big.NewInt(1)) + + ip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + allocated := ipToBigInt(ip) + + if allocated == first { + t.Fatalf("allocated ip should not equal first ip: %d == %d", first, allocated) + } +} + +func TestAllocateAllIps(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + + var ( + current, first net.IP + err error + isFirst = true + ) + + for err == nil { + current, err = RequestIP(network, nil) + if isFirst { + first = current + isFirst = false + } + } + + if err != ErrNoAvailableIPs { + t.Fatal(err) + } + + if _, err := RequestIP(network, nil); err != ErrNoAvailableIPs { + t.Fatal(err) + } + + if err := ReleaseIP(network, first); err != nil { + t.Fatal(err) + } + + again, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + assertIPEquals(t, first, again) + + // ensure that alloc.last == alloc.begin won't result in dead loop + if _, err := RequestIP(network, nil); err != ErrNoAvailableIPs { + t.Fatal(err) + } + + // Test by making alloc.last the only free ip and ensure we get it back + // #1. first of the range, (alloc.last == ipToInt(first) already) + if err := ReleaseIP(network, first); err != nil { + t.Fatal(err) + } + + ret, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + assertIPEquals(t, first, ret) + + // #2. last of the range, note that current is the last one + last := net.IPv4(192, 168, 0, 254) + setLastTo(t, network, last) + + ret, err = RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + assertIPEquals(t, last, ret) + + // #3. middle of the range + mid := net.IPv4(192, 168, 0, 7) + setLastTo(t, network, mid) + + ret, err = RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + assertIPEquals(t, mid, ret) +} + +// make sure the pool is full when calling setLastTo. +// we don't cheat here +func setLastTo(t *testing.T, network *net.IPNet, ip net.IP) { + if err := ReleaseIP(network, ip); err != nil { + t.Fatal(err) + } + + ret, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + + assertIPEquals(t, ip, ret) + + if err := ReleaseIP(network, ip); err != nil { + t.Fatal(err) + } +} + +func TestAllocateDifferentSubnets(t *testing.T) { + defer reset() + network1 := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + network2 := &net.IPNet{ + IP: []byte{127, 0, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + network3 := &net.IPNet{ + IP: []byte{0x2a, 0x00, 0x14, 0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + Mask: []byte{255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0}, // /64 netmask + } + network4 := &net.IPNet{ + IP: []byte{0x2a, 0x00, 0x16, 0x32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}, + Mask: []byte{255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0}, // /64 netmask + } + expectedIPs := []net.IP{ + 0: net.IPv4(192, 168, 0, 1), + 1: net.IPv4(192, 168, 0, 2), + 2: net.IPv4(127, 0, 0, 1), + 3: net.IPv4(127, 0, 0, 2), + 4: net.ParseIP("2a00:1450::1"), + 5: net.ParseIP("2a00:1450::2"), + 6: net.ParseIP("2a00:1450::3"), + 7: net.ParseIP("2a00:1632::1"), + 8: net.ParseIP("2a00:1632::2"), + } + + ip11, err := RequestIP(network1, nil) + if err != nil { + t.Fatal(err) + } + ip12, err := RequestIP(network1, nil) + if err != nil { + t.Fatal(err) + } + ip21, err := RequestIP(network2, nil) + if err != nil { + t.Fatal(err) + } + ip22, err := RequestIP(network2, nil) + if err != nil { + t.Fatal(err) + } + ip31, err := RequestIP(network3, nil) + if err != nil { + t.Fatal(err) + } + ip32, err := RequestIP(network3, nil) + if err != nil { + t.Fatal(err) + } + ip33, err := RequestIP(network3, nil) + if err != nil { + t.Fatal(err) + } + ip41, err := RequestIP(network4, nil) + if err != nil { + t.Fatal(err) + } + ip42, err := RequestIP(network4, nil) + if err != nil { + t.Fatal(err) + } + assertIPEquals(t, expectedIPs[0], ip11) + assertIPEquals(t, expectedIPs[1], ip12) + assertIPEquals(t, expectedIPs[2], ip21) + assertIPEquals(t, expectedIPs[3], ip22) + assertIPEquals(t, expectedIPs[4], ip31) + assertIPEquals(t, expectedIPs[5], ip32) + assertIPEquals(t, expectedIPs[6], ip33) + assertIPEquals(t, expectedIPs[7], ip41) + assertIPEquals(t, expectedIPs[8], ip42) +} + +func TestRegisterBadTwice(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 1, 1}, + Mask: []byte{255, 255, 255, 0}, + } + subnet := &net.IPNet{ + IP: []byte{192, 168, 1, 8}, + Mask: []byte{255, 255, 255, 248}, + } + + if err := RegisterSubnet(network, subnet); err != nil { + t.Fatal(err) + } + subnet = &net.IPNet{ + IP: []byte{192, 168, 1, 16}, + Mask: []byte{255, 255, 255, 248}, + } + if err := RegisterSubnet(network, subnet); err != ErrNetworkAlreadyRegistered { + t.Fatalf("Expecteded ErrNetworkAlreadyRegistered error, got %v", err) + } +} + +func TestRegisterBadRange(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 1, 1}, + Mask: []byte{255, 255, 255, 0}, + } + subnet := &net.IPNet{ + IP: []byte{192, 168, 1, 1}, + Mask: []byte{255, 255, 0, 0}, + } + if err := RegisterSubnet(network, subnet); err != ErrBadSubnet { + t.Fatalf("Expected ErrBadSubnet error, got %v", err) + } +} + +func TestAllocateFromRange(t *testing.T) { + defer reset() + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + // 192.168.1.9 - 192.168.1.14 + subnet := &net.IPNet{ + IP: []byte{192, 168, 0, 8}, + Mask: []byte{255, 255, 255, 248}, + } + + if err := RegisterSubnet(network, subnet); err != nil { + t.Fatal(err) + } + expectedIPs := []net.IP{ + 0: net.IPv4(192, 168, 0, 9), + 1: net.IPv4(192, 168, 0, 10), + 2: net.IPv4(192, 168, 0, 11), + 3: net.IPv4(192, 168, 0, 12), + 4: net.IPv4(192, 168, 0, 13), + 5: net.IPv4(192, 168, 0, 14), + } + for _, ip := range expectedIPs { + rip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + assertIPEquals(t, ip, rip) + } + + if _, err := RequestIP(network, nil); err != ErrNoAvailableIPs { + t.Fatalf("Expected ErrNoAvailableIPs error, got %v", err) + } + for _, ip := range expectedIPs { + ReleaseIP(network, ip) + rip, err := RequestIP(network, nil) + if err != nil { + t.Fatal(err) + } + assertIPEquals(t, ip, rip) + } +} + +func assertIPEquals(t *testing.T, ip1, ip2 net.IP) { + if !ip1.Equal(ip2) { + t.Fatalf("Expected IP %s, got %s", ip1, ip2) + } +} + +func BenchmarkRequestIP(b *testing.B) { + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 0; j < 253; j++ { + _, err := RequestIP(network, nil) + if err != nil { + b.Fatal(err) + } + } + reset() + } +} diff --git a/libnetwork/utils.go b/libnetwork/utils.go new file mode 100644 index 0000000000..64e6d7e852 --- /dev/null +++ b/libnetwork/utils.go @@ -0,0 +1,127 @@ +// Network utility functions. +// Imported unchanged from Docker + +package libnetwork + +import ( + "errors" + "fmt" + "math/rand" + "net" + + "github.com/vishvananda/netlink" +) + +var ( + // ErrNetworkOverlapsWithNameservers preformatted error + ErrNetworkOverlapsWithNameservers = errors.New("requested network overlaps with nameserver") + // ErrNetworkOverlaps preformatted error + ErrNetworkOverlaps = errors.New("requested network overlaps with existing network") + // ErrNoDefaultRoute preformatted error + ErrNoDefaultRoute = errors.New("no default route") + + networkGetRoutesFct = netlink.RouteList +) + +// CheckNameserverOverlaps checks whether the passed network overlaps with any of the nameservers +func CheckNameserverOverlaps(nameservers []string, toCheck *net.IPNet) error { + if len(nameservers) > 0 { + for _, ns := range nameservers { + _, nsNetwork, err := net.ParseCIDR(ns) + if err != nil { + return err + } + if NetworkOverlaps(toCheck, nsNetwork) { + return ErrNetworkOverlapsWithNameservers + } + } + } + return nil +} + +// CheckRouteOverlaps checks whether the passed network overlaps with any existing routes +func CheckRouteOverlaps(toCheck *net.IPNet) error { + networks, err := networkGetRoutesFct(nil, netlink.FAMILY_V4) + if err != nil { + return err + } + + for _, network := range networks { + if network.Dst != nil && NetworkOverlaps(toCheck, network.Dst) { + return ErrNetworkOverlaps + } + } + return nil +} + +// NetworkOverlaps detects overlap between one IPNet and another +func NetworkOverlaps(netX *net.IPNet, netY *net.IPNet) bool { + if len(netX.IP) == len(netY.IP) { + if firstIP, _ := NetworkRange(netX); netY.Contains(firstIP) { + return true + } + if firstIP, _ := NetworkRange(netY); netX.Contains(firstIP) { + return true + } + } + return false +} + +// NetworkRange calculates the first and last IP addresses in an IPNet +func NetworkRange(network *net.IPNet) (net.IP, net.IP) { + var netIP net.IP + if network.IP.To4() != nil { + netIP = network.IP.To4() + } else if network.IP.To16() != nil { + netIP = network.IP.To16() + } else { + return nil, nil + } + + lastIP := make([]byte, len(netIP), len(netIP)) + for i := 0; i < len(netIP); i++ { + lastIP[i] = netIP[i] | ^network.Mask[i] + } + return netIP.Mask(network.Mask), net.IP(lastIP) +} + +// GetIfaceAddr returns the first IPv4 address and slice of IPv6 addresses for the specified network interface +func GetIfaceAddr(name string) (net.Addr, []net.Addr, error) { + iface, err := net.InterfaceByName(name) + if err != nil { + return nil, nil, err + } + addrs, err := iface.Addrs() + if err != nil { + return nil, nil, err + } + var addrs4 []net.Addr + var addrs6 []net.Addr + for _, addr := range addrs { + ip := (addr.(*net.IPNet)).IP + if ip4 := ip.To4(); ip4 != nil { + addrs4 = append(addrs4, addr) + } else if ip6 := ip.To16(); len(ip6) == net.IPv6len { + addrs6 = append(addrs6, addr) + } + } + switch { + case len(addrs4) == 0: + return nil, nil, fmt.Errorf("Interface %v has no IPv4 addresses", name) + case len(addrs4) > 1: + fmt.Printf("Interface %v has more than 1 IPv4 address. Defaulting to using %v\n", + name, (addrs4[0].(*net.IPNet)).IP) + } + return addrs4[0], addrs6, nil +} + +// GenerateRandomMAC returns a random MAC address +func GenerateRandomMAC() net.HardwareAddr { + hw := make(net.HardwareAddr, 6) + for i := 0; i < 6; i++ { + hw[i] = byte(rand.Intn(255)) + } + hw[0] &^= 0x1 // clear multicast bit + hw[0] |= 0x2 // set local assignment bit (IEEE802) + return hw +} diff --git a/libnetwork/utils_test.go b/libnetwork/utils_test.go new file mode 100644 index 0000000000..f6f0eb29c6 --- /dev/null +++ b/libnetwork/utils_test.go @@ -0,0 +1,176 @@ +package libnetwork + +import ( + "net" + "testing" + + "github.com/vishvananda/netlink" +) + +func TestNonOverlapingNameservers(t *testing.T) { + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + nameservers := []string{ + "127.0.0.1/32", + } + + if err := CheckNameserverOverlaps(nameservers, network); err != nil { + t.Fatal(err) + } +} + +func TestOverlapingNameservers(t *testing.T) { + network := &net.IPNet{ + IP: []byte{192, 168, 0, 1}, + Mask: []byte{255, 255, 255, 0}, + } + nameservers := []string{ + "192.168.0.1/32", + } + + if err := CheckNameserverOverlaps(nameservers, network); err == nil { + t.Fatalf("Expected error %s got %s", ErrNetworkOverlapsWithNameservers, err) + } +} + +func TestCheckRouteOverlaps(t *testing.T) { + orig := networkGetRoutesFct + defer func() { + networkGetRoutesFct = orig + }() + networkGetRoutesFct = func(netlink.Link, int) ([]netlink.Route, error) { + routesData := []string{"10.0.2.0/32", "10.0.3.0/24", "10.0.42.0/24", "172.16.42.0/24", "192.168.142.0/24"} + + routes := []netlink.Route{} + for _, addr := range routesData { + _, netX, _ := net.ParseCIDR(addr) + routes = append(routes, netlink.Route{Dst: netX}) + } + return routes, nil + } + + _, netX, _ := net.ParseCIDR("172.16.0.1/24") + if err := CheckRouteOverlaps(netX); err != nil { + t.Fatal(err) + } + + _, netX, _ = net.ParseCIDR("10.0.2.0/24") + if err := CheckRouteOverlaps(netX); err == nil { + t.Fatalf("10.0.2.0/24 and 10.0.2.0 should overlap but it doesn't") + } +} + +func TestCheckNameserverOverlaps(t *testing.T) { + nameservers := []string{"10.0.2.3/32", "192.168.102.1/32"} + + _, netX, _ := net.ParseCIDR("10.0.2.3/32") + + if err := CheckNameserverOverlaps(nameservers, netX); err == nil { + t.Fatalf("%s should overlap 10.0.2.3/32 but doesn't", netX) + } + + _, netX, _ = net.ParseCIDR("192.168.102.2/32") + + if err := CheckNameserverOverlaps(nameservers, netX); err != nil { + t.Fatalf("%s should not overlap %v but it does", netX, nameservers) + } +} + +func AssertOverlap(CIDRx string, CIDRy string, t *testing.T) { + _, netX, _ := net.ParseCIDR(CIDRx) + _, netY, _ := net.ParseCIDR(CIDRy) + if !NetworkOverlaps(netX, netY) { + t.Errorf("%v and %v should overlap", netX, netY) + } +} + +func AssertNoOverlap(CIDRx string, CIDRy string, t *testing.T) { + _, netX, _ := net.ParseCIDR(CIDRx) + _, netY, _ := net.ParseCIDR(CIDRy) + if NetworkOverlaps(netX, netY) { + t.Errorf("%v and %v should not overlap", netX, netY) + } +} + +func TestNetworkOverlaps(t *testing.T) { + //netY starts at same IP and ends within netX + AssertOverlap("172.16.0.1/24", "172.16.0.1/25", t) + //netY starts within netX and ends at same IP + AssertOverlap("172.16.0.1/24", "172.16.0.128/25", t) + //netY starts and ends within netX + AssertOverlap("172.16.0.1/24", "172.16.0.64/25", t) + //netY starts at same IP and ends outside of netX + AssertOverlap("172.16.0.1/24", "172.16.0.1/23", t) + //netY starts before and ends at same IP of netX + AssertOverlap("172.16.1.1/24", "172.16.0.1/23", t) + //netY starts before and ends outside of netX + AssertOverlap("172.16.1.1/24", "172.16.0.1/22", t) + //netY starts and ends before netX + AssertNoOverlap("172.16.1.1/25", "172.16.0.1/24", t) + //netX starts and ends before netY + AssertNoOverlap("172.16.1.1/25", "172.16.2.1/24", t) +} + +func TestNetworkRange(t *testing.T) { + // Simple class C test + _, network, _ := net.ParseCIDR("192.168.0.1/24") + first, last := NetworkRange(network) + if !first.Equal(net.ParseIP("192.168.0.0")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("192.168.0.255")) { + t.Error(last.String()) + } + + // Class A test + _, network, _ = net.ParseCIDR("10.0.0.1/8") + first, last = NetworkRange(network) + if !first.Equal(net.ParseIP("10.0.0.0")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.255.255.255")) { + t.Error(last.String()) + } + + // Class A, random IP address + _, network, _ = net.ParseCIDR("10.1.2.3/8") + first, last = NetworkRange(network) + if !first.Equal(net.ParseIP("10.0.0.0")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.255.255.255")) { + t.Error(last.String()) + } + + // 32bit mask + _, network, _ = net.ParseCIDR("10.1.2.3/32") + first, last = NetworkRange(network) + if !first.Equal(net.ParseIP("10.1.2.3")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.1.2.3")) { + t.Error(last.String()) + } + + // 31bit mask + _, network, _ = net.ParseCIDR("10.1.2.3/31") + first, last = NetworkRange(network) + if !first.Equal(net.ParseIP("10.1.2.2")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.1.2.3")) { + t.Error(last.String()) + } + + // 26bit mask + _, network, _ = net.ParseCIDR("10.1.2.3/26") + first, last = NetworkRange(network) + if !first.Equal(net.ParseIP("10.1.2.0")) { + t.Error(first.String()) + } + if !last.Equal(net.ParseIP("10.1.2.63")) { + t.Error(last.String()) + } +}