From b1d422b6b5b1b1e2d0255ed4638cd5571858b847 Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Thu, 28 Apr 2016 16:54:47 -0700 Subject: [PATCH] Make overlay driver work without a kv store Currently overlay driver requires a k/v store to allocate a vxlan id and add an entry in k/v store for network->vxlanIDs binding. But the overlay driver should be able to work without a k/v store provided libnetwork can pass along the vxlanIDs needed for the network, rather than the driver managing it themselves. Modified the driver to work with vxlanIDs passed down by libnetwork. Also made changes in the driver to make use of the gossip layer available in libnetwork if available. Signed-off-by: Jana Radhakrishnan --- libnetwork/drivers/overlay/joinleave.go | 45 +++++++++++ libnetwork/drivers/overlay/ov_network.go | 88 +++++++++++++++++----- libnetwork/drivers/overlay/overlay.go | 2 +- libnetwork/drivers/overlay/overlay_test.go | 26 ------- libnetwork/drivers/overlay/peerdb.go | 2 + 5 files changed, 116 insertions(+), 47 deletions(-) diff --git a/libnetwork/drivers/overlay/joinleave.go b/libnetwork/drivers/overlay/joinleave.go index f9567d7dad..46efd3f051 100644 --- a/libnetwork/drivers/overlay/joinleave.go +++ b/libnetwork/drivers/overlay/joinleave.go @@ -3,6 +3,7 @@ package overlay import ( "fmt" "net" + "strings" log "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/driverapi" @@ -104,11 +105,55 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, d.peerDbAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.bindAddress), true) + + if err := jinfo.AddTableEntry(ovPeerTable, eid, []byte(fmt.Sprintf("%s,%s,%s", ep.addr, ep.mac, d.bindAddress))); err != nil { + log.Errorf("overlay: Failed adding table entry to joininfo: %v", err) + } + d.pushLocalEndpointEvent("join", nid, eid) return nil } +func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key string, value []byte) { + if tableName != ovPeerTable { + log.Errorf("Unexpected table notification for table %s received", tableName) + return + } + + eid := key + values := strings.Split(string(value), ",") + if len(values) < 3 { + log.Errorf("Invalid value %s received through event notify", string(value)) + return + } + + addr, err := types.ParseCIDR(values[0]) + if err != nil { + log.Errorf("Invalid peer IP %s received in event notify", values[0]) + return + } + + mac, err := net.ParseMAC(values[1]) + if err != nil { + log.Errorf("Invalid mac %s received in event notify", values[1]) + return + } + + vtep := net.ParseIP(values[2]) + if vtep == nil { + log.Errorf("Invalid VTEP %s received in event notify", values[2]) + return + } + + if etype == driverapi.Delete { + d.peerDelete(nid, eid, addr.IP, addr.Mask, mac, vtep, true) + return + } + + d.peerAdd(nid, eid, addr.IP, addr.Mask, mac, vtep, true) +} + // Leave method is invoked when a Sandbox detaches from an endpoint. func (d *driver) Leave(nid, eid string) error { if err := validateID(nid, eid); err != nil { diff --git a/libnetwork/drivers/overlay/ov_network.go b/libnetwork/drivers/overlay/ov_network.go index bf50cb7f60..893f8da314 100644 --- a/libnetwork/drivers/overlay/ov_network.go +++ b/libnetwork/drivers/overlay/ov_network.go @@ -6,6 +6,7 @@ import ( "net" "os" "path/filepath" + "strconv" "strings" "sync" "syscall" @@ -13,6 +14,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/driverapi" + "github.com/docker/libnetwork/netlabel" "github.com/docker/libnetwork/netutils" "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/resolvconf" @@ -67,9 +69,6 @@ func (d *driver) NetworkFree(id string) error { return types.NotImplementedErrorf("not implemented") } -func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key string, value []byte) { -} - func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error { if id == "" { return fmt.Errorf("invalid network id") @@ -92,12 +91,40 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d subnets: []*subnet{}, } - for _, ipd := range ipV4Data { + vnis := make([]uint32, 0, len(ipV4Data)) + if gval, ok := option[netlabel.GenericData]; ok { + optMap := gval.(map[string]string) + if val, ok := optMap[netlabel.OverlayVxlanIDList]; ok { + logrus.Debugf("overlay: Received vxlan IDs: %s", val) + vniStrings := strings.Split(val, ",") + for _, vniStr := range vniStrings { + vni, err := strconv.Atoi(vniStr) + if err != nil { + return fmt.Errorf("invalid vxlan id value %q passed", vniStr) + } + + vnis = append(vnis, uint32(vni)) + } + } + } + + // If we are getting vnis from libnetwork, either we get for + // all subnets or none. + if len(vnis) != 0 && len(vnis) < len(ipV4Data) { + return fmt.Errorf("insufficient vnis(%d) passed to overlay", len(vnis)) + } + + for i, ipd := range ipV4Data { s := &subnet{ subnetIP: ipd.Pool, gwIP: ipd.Gateway, once: &sync.Once{}, } + + if len(vnis) != 0 { + s.vni = vnis[i] + } + n.subnets = append(n.subnets, s) } @@ -105,8 +132,13 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo d return fmt.Errorf("failed to update data store for network %v: %v", n.id, err) } - d.addNetwork(n) + if nInfo != nil { + if err := nInfo.TableEventRegister(ovPeerTable); err != nil { + return err + } + } + d.addNetwork(n) return nil } @@ -255,11 +287,21 @@ func setHostMode() { } func (n *network) generateVxlanName(s *subnet) string { - return "vx-" + fmt.Sprintf("%06x", n.vxlanID(s)) + "-" + n.id[:5] + id := n.id + if len(n.id) > 5 { + id = n.id[:5] + } + + return "vx-" + fmt.Sprintf("%06x", n.vxlanID(s)) + "-" + id } func (n *network) generateBridgeName(s *subnet) string { - return "ov-" + fmt.Sprintf("%06x", n.vxlanID(s)) + "-" + n.id[:5] + id := n.id + if len(n.id) > 5 { + id = n.id[:5] + } + + return "ov-" + fmt.Sprintf("%06x", n.vxlanID(s)) + "-" + id } func isOverlap(nw *net.IPNet) bool { @@ -587,32 +629,38 @@ func (n *network) DataScope() string { } func (n *network) writeToStore() error { + if n.driver.store == nil { + return nil + } + return n.driver.store.PutObjectAtomic(n) } func (n *network) releaseVxlanID() error { - if n.driver.store == nil { - return fmt.Errorf("no datastore configured. cannot release vxlan id") - } - if len(n.subnets) == 0 { return nil } - if err := n.driver.store.DeleteObjectAtomic(n); err != nil { - if err == datastore.ErrKeyModified || err == datastore.ErrKeyNotFound { - // In both the above cases we can safely assume that the key has been removed by some other - // instance and so simply get out of here - return nil - } + if n.driver.store != nil { + if err := n.driver.store.DeleteObjectAtomic(n); err != nil { + if err == datastore.ErrKeyModified || err == datastore.ErrKeyNotFound { + // In both the above cases we can safely assume that the key has been removed by some other + // instance and so simply get out of here + return nil + } - return fmt.Errorf("failed to delete network to vxlan id map: %v", err) + return fmt.Errorf("failed to delete network to vxlan id map: %v", err) + } } for _, s := range n.subnets { - n.driver.vxlanIdm.Release(uint64(n.vxlanID(s))) + if n.driver.vxlanIdm != nil { + n.driver.vxlanIdm.Release(uint64(n.vxlanID(s))) + } + n.setVxlanID(s, 0) } + return nil } @@ -623,7 +671,7 @@ func (n *network) obtainVxlanID(s *subnet) error { } if n.driver.store == nil { - return fmt.Errorf("no datastore configured. cannot obtain vxlan id") + return fmt.Errorf("no valid vxlan id and no datastore configured, cannot obtain vxlan id") } for { diff --git a/libnetwork/drivers/overlay/overlay.go b/libnetwork/drivers/overlay/overlay.go index 80fc19b7e4..f6666bf2d4 100644 --- a/libnetwork/drivers/overlay/overlay.go +++ b/libnetwork/drivers/overlay/overlay.go @@ -88,7 +88,7 @@ func Fini(drv driverapi.Driver) { func (d *driver) configure() error { if d.store == nil { - return types.NoServiceErrorf("datastore is not available") + return nil } if d.vxlanIdm == nil { diff --git a/libnetwork/drivers/overlay/overlay_test.go b/libnetwork/drivers/overlay/overlay_test.go index a606677a87..e29530ba42 100644 --- a/libnetwork/drivers/overlay/overlay_test.go +++ b/libnetwork/drivers/overlay/overlay_test.go @@ -8,7 +8,6 @@ import ( "github.com/docker/libnetwork/discoverapi" "github.com/docker/libnetwork/driverapi" _ "github.com/docker/libnetwork/testutils" - "github.com/docker/libnetwork/types" ) type driverTester struct { @@ -24,14 +23,6 @@ func setupDriver(t *testing.T) *driverTester { t.Fatal(err) } - err := dt.d.configure() - if err == nil { - t.Fatalf("Failed to detect nil store") - } - if _, ok := err.(types.NoServiceError); !ok { - t.Fatalf("Unexpected error type: %v", err) - } - iface, err := net.InterfaceByName("eth0") if err != nil { t.Fatal(err) @@ -93,23 +84,6 @@ func TestOverlayFiniWithoutConfig(t *testing.T) { cleanupDriver(t, dt) } -func TestOverlayNilConfig(t *testing.T) { - dt := &driverTester{t: t} - if err := Init(dt, nil); err != nil { - t.Fatal(err) - } - - err := dt.d.configure() - if err == nil { - t.Fatalf("Failed to detect nil store") - } - if _, ok := err.(types.NoServiceError); !ok { - t.Fatalf("Unexpected error type: %v", err) - } - - cleanupDriver(t, dt) -} - func TestOverlayConfig(t *testing.T) { dt := setupDriver(t) diff --git a/libnetwork/drivers/overlay/peerdb.go b/libnetwork/drivers/overlay/peerdb.go index c820da9f05..3676136434 100644 --- a/libnetwork/drivers/overlay/peerdb.go +++ b/libnetwork/drivers/overlay/peerdb.go @@ -7,6 +7,8 @@ import ( "syscall" ) +const ovPeerTable = "overlay_peer_table" + type peerKey struct { peerIP net.IP peerMac net.HardwareAddr