Add overlay manager driver

Because overlay is a builtin driver and global allocation of overlay
resources is probably going to happen in a different node (a single
node) and the actual plumbing of the network is probably going to happen
in all nodes, it makes sense to split the functionality of allocation
into two different packages. The central component(this package) only
implements the NetworkAllocate/Free apis while the distributed
component(the existing overlay driver) implements the rest of the driver
api. This way we can reduce the memory footprint overall.

Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>

libnetwork/driverapi/driverapi.go

@@ -16,8 +16,8 @@ type Driver interface {
 	// NetworkAllocate invokes the driver method to allocate network
 	// specific resources passing network id and network specific config.
 	// It returns a key,value pair of network specific driver allocations
-	// to the caller and an error.
-	NetworkAllocate(nid string, options map[string]interface{}, ipV4Data, ipV6Data []IPAMData) (map[string]string, error)
+	// to the caller.
+	NetworkAllocate(nid string, options map[string]string, ipV4Data, ipV6Data []IPAMData) (map[string]string, error)
 
 	// NetworkFree invokes the driver method to free network specific resources
 	// associated with a given network id.

libnetwork/drivers/bridge/bridge.go

@@ -535,7 +535,7 @@ func (d *driver) getNetworks() []*bridgeNetwork {
 	return ls
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
+func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
 	return nil, types.NotImplementedErrorf("not implemented")
 }
 

libnetwork/drivers/host/host.go

@@ -24,7 +24,7 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	return dc.RegisterDriver(networkType, &driver{}, c)
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
+func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
 	return nil, types.NotImplementedErrorf("not implemented")
 }
 

libnetwork/drivers/ipvlan/ipvlan.go

@@ -65,7 +65,7 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	return dc.RegisterDriver(ipvlanType, d, c)
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
+func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
 	return nil, types.NotImplementedErrorf("not implemented")
 }
 

libnetwork/drivers/macvlan/macvlan.go

@@ -67,7 +67,7 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	return dc.RegisterDriver(macvlanType, d, c)
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
+func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
 	return nil, types.NotImplementedErrorf("not implemented")
 }
 

libnetwork/drivers/null/null.go

@@ -24,7 +24,7 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	return dc.RegisterDriver(networkType, &driver{}, c)
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
+func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
 	return nil, types.NotImplementedErrorf("not implemented")
 }
 

libnetwork/drivers/overlay/ov_network.go

@@ -59,7 +59,7 @@ type network struct {
 	sync.Mutex
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
+func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
 	return nil, types.NotImplementedErrorf("not implemented")
 }
 

libnetwork/drivers/overlay/ovmanager/ovmanager.go

@@ -0,0 +1,240 @@
+package ovmanager
+
+import (
+	"fmt"
+	"log"
+	"net"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/docker/libnetwork/datastore"
+	"github.com/docker/libnetwork/discoverapi"
+	"github.com/docker/libnetwork/driverapi"
+	"github.com/docker/libnetwork/idm"
+	"github.com/docker/libnetwork/netlabel"
+	"github.com/docker/libnetwork/types"
+)
+
+const (
+	networkType  = "overlay"
+	vxlanIDStart = 256
+	vxlanIDEnd   = 1000
+)
+
+type networkTable map[string]*network
+
+type driver struct {
+	config   map[string]interface{}
+	networks networkTable
+	store    datastore.DataStore
+	vxlanIdm *idm.Idm
+	sync.Mutex
+}
+
+type subnet struct {
+	subnetIP *net.IPNet
+	gwIP     *net.IPNet
+	vni      uint32
+}
+
+type network struct {
+	id      string
+	driver  *driver
+	subnets []*subnet
+	sync.Mutex
+}
+
+// Init registers a new instance of overlay driver
+func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
+	var err error
+	c := driverapi.Capability{
+		DataScope: datastore.GlobalScope,
+	}
+
+	d := &driver{
+		networks: networkTable{},
+		config:   config,
+	}
+
+	d.vxlanIdm, err = idm.New(nil, "vxlan-id", vxlanIDStart, vxlanIDEnd)
+	if err != nil {
+		return fmt.Errorf("failed to initialize vxlan id manager: %v", err)
+	}
+
+	return dc.RegisterDriver(networkType, d, c)
+}
+
+func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
+	if id == "" {
+		return nil, fmt.Errorf("invalid network id for overlay network")
+	}
+
+	if ipV4Data == nil {
+		return nil, fmt.Errorf("empty ipv4 data passed during overlay network creation")
+	}
+
+	n := &network{
+		id:      id,
+		driver:  d,
+		subnets: []*subnet{},
+	}
+
+	vxlanIDList := make([]uint32, 0, len(ipV4Data))
+	if val, ok := option[netlabel.OverlayVxlanIDList]; ok {
+		log.Println("overlay network option: ", val)
+		valStrList := strings.Split(val, ",")
+		for _, idStr := range valStrList {
+			vni, err := strconv.Atoi(idStr)
+			if err != nil {
+				return nil, fmt.Errorf("invalid vxlan id value %q passed", idStr)
+			}
+
+			vxlanIDList = append(vxlanIDList, uint32(vni))
+		}
+	}
+
+	for i, ipd := range ipV4Data {
+		s := &subnet{
+			subnetIP: ipd.Pool,
+			gwIP:     ipd.Gateway,
+		}
+
+		if len(vxlanIDList) > i {
+			s.vni = vxlanIDList[i]
+		}
+
+		if err := n.obtainVxlanID(s); err != nil {
+			log.Printf("Could not obtain vxlan id for pool %s: %v", s.subnetIP, err)
+		}
+
+		n.subnets = append(n.subnets, s)
+	}
+
+	opts := make(map[string]string)
+	val := fmt.Sprintf("%d", n.subnets[0].vni)
+	for _, s := range n.subnets[1:] {
+		val = val + fmt.Sprintf(",%d", s.vni)
+	}
+	opts[netlabel.OverlayVxlanIDList] = val
+
+	d.Lock()
+	d.networks[id] = n
+	d.Unlock()
+
+	return opts, nil
+}
+
+func (d *driver) NetworkFree(id string) error {
+	if id == "" {
+		return fmt.Errorf("invalid network id passed while freeing overlay network")
+	}
+
+	d.Lock()
+	n, ok := d.networks[id]
+	d.Unlock()
+
+	if !ok {
+		return fmt.Errorf("overlay network with id %s not found", id)
+	}
+
+	// Release all vxlan IDs in one shot.
+	n.releaseVxlanID()
+
+	d.Lock()
+	delete(d.networks, id)
+	d.Unlock()
+
+	return nil
+}
+
+func (n *network) obtainVxlanID(s *subnet) error {
+	var (
+		err error
+		vni uint64
+	)
+
+	n.Lock()
+	vni = uint64(s.vni)
+	n.Unlock()
+
+	if vni == 0 {
+		vni, err = n.driver.vxlanIdm.GetID()
+		if err != nil {
+			return err
+		}
+
+		n.Lock()
+		s.vni = uint32(vni)
+		n.Unlock()
+		return nil
+	}
+
+	return n.driver.vxlanIdm.GetSpecificID(vni)
+}
+
+func (n *network) releaseVxlanID() {
+	n.Lock()
+	vnis := make([]uint32, 0, len(n.subnets))
+	for _, s := range n.subnets {
+		vnis = append(vnis, s.vni)
+		s.vni = 0
+	}
+	n.Unlock()
+
+	for _, vni := range vnis {
+		n.driver.vxlanIdm.Release(uint64(vni))
+	}
+}
+
+func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) error {
+	return types.NotImplementedErrorf("not implemented")
+}
+
+func (d *driver) DeleteNetwork(nid string) error {
+	return types.NotImplementedErrorf("not implemented")
+}
+
+func (d *driver) CreateEndpoint(nid, eid string, ifInfo driverapi.InterfaceInfo, epOptions map[string]interface{}) error {
+	return types.NotImplementedErrorf("not implemented")
+}
+
+func (d *driver) DeleteEndpoint(nid, eid string) error {
+	return types.NotImplementedErrorf("not implemented")
+}
+
+func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, error) {
+	return nil, types.NotImplementedErrorf("not implemented")
+}
+
+// Join method is invoked when a Sandbox is attached to an endpoint.
+func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error {
+	return types.NotImplementedErrorf("not implemented")
+}
+
+// Leave method is invoked when a Sandbox detaches from an endpoint.
+func (d *driver) Leave(nid, eid string) error {
+	return types.NotImplementedErrorf("not implemented")
+}
+
+func (d *driver) Type() string {
+	return networkType
+}
+
+// DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
+func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
+	return types.NotImplementedErrorf("not implemented")
+}
+
+// DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster
+func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error {
+	return types.NotImplementedErrorf("not implemented")
+}
+
+func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error {
+	return types.NotImplementedErrorf("not implemented")
+}
+
+func (d *driver) RevokeExternalConnectivity(nid, eid string) error {
+	return types.NotImplementedErrorf("not implemented")
+}

libnetwork/drivers/overlay/ovmanager/ovmanager_test.go

@@ -0,0 +1,88 @@
+package ovmanager
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+
+	"github.com/docker/libnetwork/driverapi"
+	"github.com/docker/libnetwork/idm"
+	"github.com/docker/libnetwork/netlabel"
+	_ "github.com/docker/libnetwork/testutils"
+	"github.com/docker/libnetwork/types"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newDriver(t *testing.T) *driver {
+	d := &driver{
+		networks: networkTable{},
+	}
+
+	vxlanIdm, err := idm.New(nil, "vxlan-id", vxlanIDStart, vxlanIDEnd)
+	require.NoError(t, err)
+
+	d.vxlanIdm = vxlanIdm
+	return d
+}
+
+func parseCIDR(t *testing.T, ipnet string) *net.IPNet {
+	subnet, err := types.ParseCIDR(ipnet)
+	require.NoError(t, err)
+	return subnet
+}
+
+func TestNetworkAllocateFree(t *testing.T) {
+	d := newDriver(t)
+
+	ipamData := []driverapi.IPAMData{
+		{
+			Pool: parseCIDR(t, "10.1.1.0/24"),
+		},
+		{
+			Pool: parseCIDR(t, "10.1.2.0/24"),
+		},
+	}
+
+	vals, err := d.NetworkAllocate("testnetwork", nil, ipamData, nil)
+	require.NoError(t, err)
+
+	vxlanIDs, ok := vals[netlabel.OverlayVxlanIDList]
+	assert.Equal(t, true, ok)
+	assert.Equal(t, 2, len(strings.Split(vxlanIDs, ",")))
+
+	err = d.NetworkFree("testnetwork")
+	require.NoError(t, err)
+}
+
+func TestNetworkAllocateUserDefinedVNIs(t *testing.T) {
+	d := newDriver(t)
+
+	ipamData := []driverapi.IPAMData{
+		{
+			Pool: parseCIDR(t, "10.1.1.0/24"),
+		},
+		{
+			Pool: parseCIDR(t, "10.1.2.0/24"),
+		},
+	}
+
+	options := make(map[string]string)
+	// Intentionally add mode vnis than subnets
+	options[netlabel.OverlayVxlanIDList] = fmt.Sprintf("%d,%d,%d", 256, 257, 258)
+
+	vals, err := d.NetworkAllocate("testnetwork", options, ipamData, nil)
+	require.NoError(t, err)
+
+	vxlanIDs, ok := vals[netlabel.OverlayVxlanIDList]
+	assert.Equal(t, true, ok)
+
+	// We should only get exactly the same number of vnis as
+	// subnets. No more, no less, even if we passed more vnis.
+	assert.Equal(t, 2, len(strings.Split(vxlanIDs, ",")))
+	assert.Equal(t, fmt.Sprintf("%d,%d", 256, 257), vxlanIDs)
+
+	err = d.NetworkFree("testnetwork")
+	require.NoError(t, err)
+}

libnetwork/drivers/remote/driver.go

@@ -83,7 +83,7 @@ func (d *driver) call(methodName string, arg interface{}, retVal maybeError) err
 	return nil
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
+func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
 	return nil, types.NotImplementedErrorf("not implemented")
 }
 

libnetwork/drivers/windows/windows.go

@@ -560,7 +560,7 @@ func (d *driver) RevokeExternalConnectivity(nid, eid string) error {
 	return nil
 }
 
-func (d *driver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
+func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
 	return nil, types.NotImplementedErrorf("not implemented")
 }
 

libnetwork/drvregistry/drvregistry_test.go

@@ -73,7 +73,7 @@ func (m *mockDriver) RevokeExternalConnectivity(nid, eid string) error {
 	return nil
 }
 
-func (m *mockDriver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
+func (m *mockDriver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
 	return nil, nil
 }
 

libnetwork/libnetwork_internal_test.go

@@ -433,7 +433,7 @@ func (b *badDriver) RevokeExternalConnectivity(nid, eid string) error {
 	return nil
 }
 
-func (b *badDriver) NetworkAllocate(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
+func (b *badDriver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
 	return nil, types.NotImplementedErrorf("not implemented")
 }
 

libnetwork/netlabel/labels.go

@@ -39,6 +39,9 @@ const (
 	// OverlayNeighborIP constant represents overlay driver neighbor IP
 	OverlayNeighborIP = DriverPrefix + ".overlay.neighbor_ip"
 
+	// OverlayVxlanIDList constant represents a list of VXLAN Ids as csv
+	OverlayVxlanIDList = DriverPrefix + ".overlay.vxlanid_list"
+
 	// Gateway represents the gateway for the network
 	Gateway = Prefix + ".gateway"