From c0162f53a611f004ec0b9048f6f7a99d43924aee Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Fri, 26 Feb 2016 11:54:35 -0800 Subject: [PATCH] Add overlay manager driver Because overlay is a builtin driver and global allocation of overlay resources is probably going to happen in a different node (a single node) and the actual plumbing of the network is probably going to happen in all nodes, it makes sense to split the functionality of allocation into two different packages. The central component(this package) only implements the NetworkAllocate/Free apis while the distributed component(the existing overlay driver) implements the rest of the driver api. This way we can reduce the memory footprint overall. Signed-off-by: Jana Radhakrishnan --- libnetwork/driverapi/driverapi.go | 4 +- libnetwork/drivers/bridge/bridge.go | 2 +- libnetwork/drivers/host/host.go | 2 +- libnetwork/drivers/ipvlan/ipvlan.go | 2 +- libnetwork/drivers/macvlan/macvlan.go | 2 +- libnetwork/drivers/null/null.go | 2 +- libnetwork/drivers/overlay/ov_network.go | 2 +- .../drivers/overlay/ovmanager/ovmanager.go | 240 ++++++++++++++++++ .../overlay/ovmanager/ovmanager_test.go | 88 +++++++ libnetwork/drivers/remote/driver.go | 2 +- libnetwork/drivers/windows/windows.go | 2 +- libnetwork/drvregistry/drvregistry_test.go | 2 +- libnetwork/libnetwork_internal_test.go | 2 +- libnetwork/netlabel/labels.go | 3 + 14 files changed, 343 insertions(+), 12 deletions(-) create mode 100644 libnetwork/drivers/overlay/ovmanager/ovmanager.go create mode 100644 libnetwork/drivers/overlay/ovmanager/ovmanager_test.go diff --git a/libnetwork/driverapi/driverapi.go b/libnetwork/driverapi/driverapi.go index 33e8a16993..87f71355c0 100644 --- a/libnetwork/driverapi/driverapi.go +++ b/libnetwork/driverapi/driverapi.go @@ -16,8 +16,8 @@ type Driver interface { // NetworkAllocate invokes the driver method to allocate network // specific resources passing network id and network specific config. // It returns a key,value pair of network specific driver allocations - // to the caller and an error. - NetworkAllocate(nid string, options map[string]interface{}, ipV4Data, ipV6Data []IPAMData) (map[string]string, error) + // to the caller. + NetworkAllocate(nid string, options map[string]string, ipV4Data, ipV6Data []IPAMData) (map[string]string, error) // NetworkFree invokes the driver method to free network specific resources // associated with a given network id. diff --git a/libnetwork/drivers/bridge/bridge.go b/libnetwork/drivers/bridge/bridge.go index 795a3360f9..51d589fc4c 100644 --- a/libnetwork/drivers/bridge/bridge.go +++ b/libnetwork/drivers/bridge/bridge.go @@ -535,7 +535,7 @@ func (d *driver) getNetworks() []*bridgeNetwork { return ls } -func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { +func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { return nil, types.NotImplementedErrorf("not implemented") } diff --git a/libnetwork/drivers/host/host.go b/libnetwork/drivers/host/host.go index 4ba68d71f6..7c7a254b8f 100644 --- a/libnetwork/drivers/host/host.go +++ b/libnetwork/drivers/host/host.go @@ -24,7 +24,7 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { return dc.RegisterDriver(networkType, &driver{}, c) } -func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { +func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { return nil, types.NotImplementedErrorf("not implemented") } diff --git a/libnetwork/drivers/ipvlan/ipvlan.go b/libnetwork/drivers/ipvlan/ipvlan.go index cb95a658c1..cb8296a985 100644 --- a/libnetwork/drivers/ipvlan/ipvlan.go +++ b/libnetwork/drivers/ipvlan/ipvlan.go @@ -65,7 +65,7 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { return dc.RegisterDriver(ipvlanType, d, c) } -func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { +func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { return nil, types.NotImplementedErrorf("not implemented") } diff --git a/libnetwork/drivers/macvlan/macvlan.go b/libnetwork/drivers/macvlan/macvlan.go index 31b3d76181..826e1dec16 100644 --- a/libnetwork/drivers/macvlan/macvlan.go +++ b/libnetwork/drivers/macvlan/macvlan.go @@ -67,7 +67,7 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { return dc.RegisterDriver(macvlanType, d, c) } -func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { +func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { return nil, types.NotImplementedErrorf("not implemented") } diff --git a/libnetwork/drivers/null/null.go b/libnetwork/drivers/null/null.go index 048c43e62e..a689708823 100644 --- a/libnetwork/drivers/null/null.go +++ b/libnetwork/drivers/null/null.go @@ -24,7 +24,7 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { return dc.RegisterDriver(networkType, &driver{}, c) } -func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { +func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { return nil, types.NotImplementedErrorf("not implemented") } diff --git a/libnetwork/drivers/overlay/ov_network.go b/libnetwork/drivers/overlay/ov_network.go index f0cb5a65af..dae3d642fc 100644 --- a/libnetwork/drivers/overlay/ov_network.go +++ b/libnetwork/drivers/overlay/ov_network.go @@ -59,7 +59,7 @@ type network struct { sync.Mutex } -func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { +func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { return nil, types.NotImplementedErrorf("not implemented") } diff --git a/libnetwork/drivers/overlay/ovmanager/ovmanager.go b/libnetwork/drivers/overlay/ovmanager/ovmanager.go new file mode 100644 index 0000000000..5eb8bebf9b --- /dev/null +++ b/libnetwork/drivers/overlay/ovmanager/ovmanager.go @@ -0,0 +1,240 @@ +package ovmanager + +import ( + "fmt" + "log" + "net" + "strconv" + "strings" + "sync" + + "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/discoverapi" + "github.com/docker/libnetwork/driverapi" + "github.com/docker/libnetwork/idm" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/types" +) + +const ( + networkType = "overlay" + vxlanIDStart = 256 + vxlanIDEnd = 1000 +) + +type networkTable map[string]*network + +type driver struct { + config map[string]interface{} + networks networkTable + store datastore.DataStore + vxlanIdm *idm.Idm + sync.Mutex +} + +type subnet struct { + subnetIP *net.IPNet + gwIP *net.IPNet + vni uint32 +} + +type network struct { + id string + driver *driver + subnets []*subnet + sync.Mutex +} + +// Init registers a new instance of overlay driver +func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { + var err error + c := driverapi.Capability{ + DataScope: datastore.GlobalScope, + } + + d := &driver{ + networks: networkTable{}, + config: config, + } + + d.vxlanIdm, err = idm.New(nil, "vxlan-id", vxlanIDStart, vxlanIDEnd) + if err != nil { + return fmt.Errorf("failed to initialize vxlan id manager: %v", err) + } + + return dc.RegisterDriver(networkType, d, c) +} + +func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { + if id == "" { + return nil, fmt.Errorf("invalid network id for overlay network") + } + + if ipV4Data == nil { + return nil, fmt.Errorf("empty ipv4 data passed during overlay network creation") + } + + n := &network{ + id: id, + driver: d, + subnets: []*subnet{}, + } + + vxlanIDList := make([]uint32, 0, len(ipV4Data)) + if val, ok := option[netlabel.OverlayVxlanIDList]; ok { + log.Println("overlay network option: ", val) + valStrList := strings.Split(val, ",") + for _, idStr := range valStrList { + vni, err := strconv.Atoi(idStr) + if err != nil { + return nil, fmt.Errorf("invalid vxlan id value %q passed", idStr) + } + + vxlanIDList = append(vxlanIDList, uint32(vni)) + } + } + + for i, ipd := range ipV4Data { + s := &subnet{ + subnetIP: ipd.Pool, + gwIP: ipd.Gateway, + } + + if len(vxlanIDList) > i { + s.vni = vxlanIDList[i] + } + + if err := n.obtainVxlanID(s); err != nil { + log.Printf("Could not obtain vxlan id for pool %s: %v", s.subnetIP, err) + } + + n.subnets = append(n.subnets, s) + } + + opts := make(map[string]string) + val := fmt.Sprintf("%d", n.subnets[0].vni) + for _, s := range n.subnets[1:] { + val = val + fmt.Sprintf(",%d", s.vni) + } + opts[netlabel.OverlayVxlanIDList] = val + + d.Lock() + d.networks[id] = n + d.Unlock() + + return opts, nil +} + +func (d *driver) NetworkFree(id string) error { + if id == "" { + return fmt.Errorf("invalid network id passed while freeing overlay network") + } + + d.Lock() + n, ok := d.networks[id] + d.Unlock() + + if !ok { + return fmt.Errorf("overlay network with id %s not found", id) + } + + // Release all vxlan IDs in one shot. + n.releaseVxlanID() + + d.Lock() + delete(d.networks, id) + d.Unlock() + + return nil +} + +func (n *network) obtainVxlanID(s *subnet) error { + var ( + err error + vni uint64 + ) + + n.Lock() + vni = uint64(s.vni) + n.Unlock() + + if vni == 0 { + vni, err = n.driver.vxlanIdm.GetID() + if err != nil { + return err + } + + n.Lock() + s.vni = uint32(vni) + n.Unlock() + return nil + } + + return n.driver.vxlanIdm.GetSpecificID(vni) +} + +func (n *network) releaseVxlanID() { + n.Lock() + vnis := make([]uint32, 0, len(n.subnets)) + for _, s := range n.subnets { + vnis = append(vnis, s.vni) + s.vni = 0 + } + n.Unlock() + + for _, vni := range vnis { + n.driver.vxlanIdm.Release(uint64(vni)) + } +} + +func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) error { + return types.NotImplementedErrorf("not implemented") +} + +func (d *driver) DeleteNetwork(nid string) error { + return types.NotImplementedErrorf("not implemented") +} + +func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, epOptions map[string]interface{}) error { + return types.NotImplementedErrorf("not implemented") +} + +func (d *driver) DeleteEndpoint(nid, eid string) error { + return types.NotImplementedErrorf("not implemented") +} + +func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, error) { + return nil, types.NotImplementedErrorf("not implemented") +} + +// Join method is invoked when a Sandbox is attached to an endpoint. +func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error { + return types.NotImplementedErrorf("not implemented") +} + +// Leave method is invoked when a Sandbox detaches from an endpoint. +func (d *driver) Leave(nid, eid string) error { + return types.NotImplementedErrorf("not implemented") +} + +func (d *driver) Type() string { + return networkType +} + +// DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster +func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { + return types.NotImplementedErrorf("not implemented") +} + +// DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster +func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { + return types.NotImplementedErrorf("not implemented") +} + +func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error { + return types.NotImplementedErrorf("not implemented") +} + +func (d *driver) RevokeExternalConnectivity(nid, eid string) error { + return types.NotImplementedErrorf("not implemented") +} diff --git a/libnetwork/drivers/overlay/ovmanager/ovmanager_test.go b/libnetwork/drivers/overlay/ovmanager/ovmanager_test.go new file mode 100644 index 0000000000..f6d5921de5 --- /dev/null +++ b/libnetwork/drivers/overlay/ovmanager/ovmanager_test.go @@ -0,0 +1,88 @@ +package ovmanager + +import ( + "fmt" + "net" + "strings" + "testing" + + "github.com/docker/libnetwork/driverapi" + "github.com/docker/libnetwork/idm" + "github.com/docker/libnetwork/netlabel" + _ "github.com/docker/libnetwork/testutils" + "github.com/docker/libnetwork/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newDriver(t *testing.T) *driver { + d := &driver{ + networks: networkTable{}, + } + + vxlanIdm, err := idm.New(nil, "vxlan-id", vxlanIDStart, vxlanIDEnd) + require.NoError(t, err) + + d.vxlanIdm = vxlanIdm + return d +} + +func parseCIDR(t *testing.T, ipnet string) *net.IPNet { + subnet, err := types.ParseCIDR(ipnet) + require.NoError(t, err) + return subnet +} + +func TestNetworkAllocateFree(t *testing.T) { + d := newDriver(t) + + ipamData := []driverapi.IPAMData{ + { + Pool: parseCIDR(t, "10.1.1.0/24"), + }, + { + Pool: parseCIDR(t, "10.1.2.0/24"), + }, + } + + vals, err := d.NetworkAllocate("testnetwork", nil, ipamData, nil) + require.NoError(t, err) + + vxlanIDs, ok := vals[netlabel.OverlayVxlanIDList] + assert.Equal(t, true, ok) + assert.Equal(t, 2, len(strings.Split(vxlanIDs, ","))) + + err = d.NetworkFree("testnetwork") + require.NoError(t, err) +} + +func TestNetworkAllocateUserDefinedVNIs(t *testing.T) { + d := newDriver(t) + + ipamData := []driverapi.IPAMData{ + { + Pool: parseCIDR(t, "10.1.1.0/24"), + }, + { + Pool: parseCIDR(t, "10.1.2.0/24"), + }, + } + + options := make(map[string]string) + // Intentionally add mode vnis than subnets + options[netlabel.OverlayVxlanIDList] = fmt.Sprintf("%d,%d,%d", 256, 257, 258) + + vals, err := d.NetworkAllocate("testnetwork", options, ipamData, nil) + require.NoError(t, err) + + vxlanIDs, ok := vals[netlabel.OverlayVxlanIDList] + assert.Equal(t, true, ok) + + // We should only get exactly the same number of vnis as + // subnets. No more, no less, even if we passed more vnis. + assert.Equal(t, 2, len(strings.Split(vxlanIDs, ","))) + assert.Equal(t, fmt.Sprintf("%d,%d", 256, 257), vxlanIDs) + + err = d.NetworkFree("testnetwork") + require.NoError(t, err) +} diff --git a/libnetwork/drivers/remote/driver.go b/libnetwork/drivers/remote/driver.go index 5400e69649..78f36587e5 100644 --- a/libnetwork/drivers/remote/driver.go +++ b/libnetwork/drivers/remote/driver.go @@ -83,7 +83,7 @@ func (d *driver) call(methodName string, arg interface{}, retVal maybeError) err return nil } -func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { +func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { return nil, types.NotImplementedErrorf("not implemented") } diff --git a/libnetwork/drivers/windows/windows.go b/libnetwork/drivers/windows/windows.go index c455484b87..851f545850 100644 --- a/libnetwork/drivers/windows/windows.go +++ b/libnetwork/drivers/windows/windows.go @@ -560,7 +560,7 @@ func (d *driver) RevokeExternalConnectivity(nid, eid string) error { return nil } -func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { +func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { return nil, types.NotImplementedErrorf("not implemented") } diff --git a/libnetwork/drvregistry/drvregistry_test.go b/libnetwork/drvregistry/drvregistry_test.go index 92a4108580..bcadad26be 100644 --- a/libnetwork/drvregistry/drvregistry_test.go +++ b/libnetwork/drvregistry/drvregistry_test.go @@ -73,7 +73,7 @@ func (m *mockDriver) RevokeExternalConnectivity(nid, eid string) error { return nil } -func (m *mockDriver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { +func (m *mockDriver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { return nil, nil } diff --git a/libnetwork/libnetwork_internal_test.go b/libnetwork/libnetwork_internal_test.go index ea2950ff19..deb5b42b3e 100644 --- a/libnetwork/libnetwork_internal_test.go +++ b/libnetwork/libnetwork_internal_test.go @@ -433,7 +433,7 @@ func (b *badDriver) RevokeExternalConnectivity(nid, eid string) error { return nil } -func (b *badDriver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { +func (b *badDriver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) { return nil, types.NotImplementedErrorf("not implemented") } diff --git a/libnetwork/netlabel/labels.go b/libnetwork/netlabel/labels.go index d44015f159..7d5c35579b 100644 --- a/libnetwork/netlabel/labels.go +++ b/libnetwork/netlabel/labels.go @@ -39,6 +39,9 @@ const ( // OverlayNeighborIP constant represents overlay driver neighbor IP OverlayNeighborIP = DriverPrefix + ".overlay.neighbor_ip" + // OverlayVxlanIDList constant represents a list of VXLAN Ids as csv + OverlayVxlanIDList = DriverPrefix + ".overlay.vxlanid_list" + // Gateway represents the gateway for the network Gateway = Prefix + ".gateway"