Merge pull request #613 from mrjana/bridge

Make bridge driver networks persistent

libnetwork/cmd/dnet/dnet.go

@@ -185,17 +185,22 @@ func createDefaultNetwork(c libnetwork.NetworkController) {
 		// Bridge driver is special due to legacy reasons
 		if d == "bridge" {
 			genericOption[netlabel.GenericData] = map[string]interface{}{
-				"BridgeName": nw,
+				"BridgeName":    "docker0",
+				"DefaultBridge": "true",
 			}
 			createOptions = append(createOptions,
 				libnetwork.NetworkOptionGeneric(genericOption),
 				ipamOption(nw))
 		}
 
-		if _, err := c.NetworkByName(nw); err == nil {
-			logrus.Debugf("Default network %s already present", nw)
-			return
+		if n, err := c.NetworkByName(nw); err == nil {
+			logrus.Debugf("Default network %s already present. Deleting it", nw)
+			if err = n.Delete(); err != nil {
+				logrus.Debugf("Network could not be deleted: %v", err)
+				return
+			}
 		}
+
 		_, err := c.NewNetwork(d, nw, createOptions...)
 		if err != nil {
 			logrus.Errorf("Error creating default network : %s : %v", nw, err)

libnetwork/datastore/datastore.go

@@ -92,8 +92,7 @@ type KVConstructor interface {
 
 // ScopeCfg represents Datastore configuration.
 type ScopeCfg struct {
-	Embedded bool
-	Client   ScopeClientCfg
+	Client ScopeClientCfg
 }
 
 // ScopeClientCfg represents Datastore Client-only mode configuration
@@ -125,7 +124,6 @@ var (
 func makeDefaultScopes() map[string]*ScopeCfg {
 	def := make(map[string]*ScopeCfg)
 	def[LocalScope] = &ScopeCfg{
-		Embedded: true,
 		Client: ScopeClientCfg{
 			Provider: "boltdb",
 			Address:  defaultPrefix + "/boltdb.db",

libnetwork/datastore/datastore_test.go

@@ -38,7 +38,6 @@ func TestParseKey(t *testing.T) {
 
 func TestInvalidDataStore(t *testing.T) {
 	config := &ScopeCfg{}
-	config.Embedded = false
 	config.Client.Provider = "invalid"
 	config.Client.Address = "localhost:8500"
 	_, err := NewDataStore(GlobalScope, config)

libnetwork/drivers/bridge/bridge.go

@@ -55,6 +55,7 @@ type configuration struct {
 
 // networkConfiguration for network specific configuration
 type networkConfiguration struct {
+	ID                 string
 	BridgeName         string
 	EnableIPv6         bool
 	EnableIPMasquerade bool
@@ -67,6 +68,8 @@ type networkConfiguration struct {
 	AddressIPv6        *net.IPNet
 	DefaultGatewayIPv4 net.IP
 	DefaultGatewayIPv6 net.IP
+	dbIndex            uint64
+	dbExists           bool
 }
 
 // endpointConfiguration represents the user specified configuration for the sandbox endpoint
@@ -109,6 +112,7 @@ type driver struct {
 	natChain    *iptables.ChainInfo
 	filterChain *iptables.ChainInfo
 	networks    map[string]*bridgeNetwork
+	store       datastore.DataStore
 	sync.Mutex
 }
 
@@ -376,6 +380,11 @@ func (d *driver) configure(option map[string]interface{}) error {
 	var config *configuration
 	var err error
 
+	err = d.initStore(option)
+	if err != nil {
+		return err
+	}
+
 	d.Lock()
 	defer d.Unlock()
 
@@ -511,6 +520,8 @@ func parseNetworkOptions(id string, option options.Generic) (*networkConfigurati
 	if config.BridgeName == "" && config.DefaultBridge == false {
 		config.BridgeName = "br-" + id[:12]
 	}
+
+	config.ID = id
 	return config, nil
 }
 
@@ -540,10 +551,6 @@ func (d *driver) getNetworks() []*bridgeNetwork {
 
 // Create a new network using bridge plugin
 func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) error {
-	var err error
-
-	defer osl.InitOSContext()()
-
 	// Sanity checks
 	d.Lock()
 	if _, ok := d.networks[id]; ok {
@@ -563,6 +570,18 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat
 		return err
 	}
 
+	if err = d.createNetwork(config); err != nil {
+		return err
+	}
+
+	return d.storeUpdate(config)
+}
+
+func (d *driver) createNetwork(config *networkConfiguration) error {
+	var err error
+
+	defer osl.InitOSContext()()
+
 	networkList := d.getNetworks()
 	for _, nw := range networkList {
 		nw.Lock()
@@ -570,13 +589,13 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat
 		nw.Unlock()
 		if err := nwConfig.Conflicts(config); err != nil {
 			return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s",
-				nwConfig.BridgeName, id, nw.id, nw.config.BridgeName, err.Error())
+				nwConfig.BridgeName, config.ID, nw.id, nw.config.BridgeName, err.Error())
 		}
 	}
 
 	// Create and set network handler in driver
 	network := &bridgeNetwork{
-		id:         id,
+		id:         config.ID,
 		endpoints:  make(map[string]*bridgeEndpoint),
 		config:     config,
 		portMapper: portmapper.New(),
@@ -584,14 +603,14 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat
 	}
 
 	d.Lock()
-	d.networks[id] = network
+	d.networks[config.ID] = network
 	d.Unlock()
 
 	// On failure make sure to reset driver network handler to nil
 	defer func() {
 		if err != nil {
 			d.Lock()
-			delete(d.networks, id)
+			delete(d.networks, config.ID)
 			d.Unlock()
 		}
 	}()
@@ -604,7 +623,7 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat
 	// networks. This step is needed now because driver might have now set the bridge
 	// name on this config struct. And because we need to check for possible address
 	// conflicts, so we need to check against operationa lnetworks.
-	if err = config.conflictsWithNetworks(id, networkList); err != nil {
+	if err = config.conflictsWithNetworks(config.ID, networkList); err != nil {
 		return err
 	}
 
@@ -705,10 +724,6 @@ func (d *driver) DeleteNetwork(nid string) error {
 	config := n.config
 	n.Unlock()
 
-	if config.BridgeName == DefaultBridgeName {
-		return types.ForbiddenErrorf("default network of type \"%s\" cannot be deleted", networkType)
-	}
-
 	d.Lock()
 	delete(d.networks, nid)
 	d.Unlock()
@@ -753,8 +768,12 @@ func (d *driver) DeleteNetwork(nid string) error {
 		return err
 	}
 
-	// Programming
-	return netlink.LinkDel(n.bridge.Link)
+	// We only delete the bridge when it's not the default bridge. This is keep the backward compatible behavior.
+	if !config.DefaultBridge {
+		err = netlink.LinkDel(n.bridge.Link)
+	}
+
+	return d.storeDelete(config)
 }
 
 func addToBridge(ifaceName, bridgeName string) error {

libnetwork/drivers/bridge/bridge_store.go

@@ -0,0 +1,210 @@
+package bridge
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/libkv/store"
+	"github.com/docker/libkv/store/boltdb"
+	"github.com/docker/libnetwork/datastore"
+	"github.com/docker/libnetwork/netlabel"
+	"github.com/docker/libnetwork/types"
+)
+
+const bridgePrefix = "bridge"
+
+func (d *driver) initStore(option map[string]interface{}) error {
+	var err error
+
+	provider, provOk := option[netlabel.LocalKVProvider]
+	provURL, urlOk := option[netlabel.LocalKVProviderURL]
+
+	if provOk && urlOk {
+		cfg := &datastore.ScopeCfg{
+			Client: datastore.ScopeClientCfg{
+				Provider: provider.(string),
+				Address:  provURL.(string),
+			},
+		}
+
+		provConfig, confOk := option[netlabel.LocalKVProviderConfig]
+		if confOk {
+			cfg.Client.Config = provConfig.(*store.Config)
+		}
+
+		d.store, err = datastore.NewDataStore(datastore.LocalScope, cfg)
+		if err != nil {
+			return fmt.Errorf("bridge driver failed to initialize data store: %v", err)
+		}
+
+		return d.populateNetworks()
+	}
+
+	return nil
+}
+
+func (d *driver) populateNetworks() error {
+	kvol, err := d.store.List(datastore.Key(bridgePrefix), &networkConfiguration{})
+	if err != nil && err != datastore.ErrKeyNotFound && err != boltdb.ErrBoltBucketNotFound {
+		return fmt.Errorf("failed to get bridge network configurations from store: %v", err)
+	}
+
+	// It's normal for network configuration state to be empty. Just return.
+	if err == datastore.ErrKeyNotFound {
+		return nil
+	}
+
+	for _, kvo := range kvol {
+		ncfg := kvo.(*networkConfiguration)
+		if err = d.createNetwork(ncfg); err != nil {
+			logrus.Warnf("could not create bridge network for id %s bridge name %s while booting up from persistent state", ncfg.ID, ncfg.BridgeName)
+		}
+	}
+
+	return nil
+}
+
+func (d *driver) storeUpdate(kvObject datastore.KVObject) error {
+	if d.store == nil {
+		logrus.Warnf("bridge store not initialized. kv object %s is not added to the store", datastore.Key(kvObject.Key()...))
+		return nil
+	}
+
+	if err := d.store.PutObjectAtomic(kvObject); err != nil {
+		return fmt.Errorf("failed to update bridge store for object type %T: %v", kvObject, err)
+	}
+
+	return nil
+}
+
+func (d *driver) storeDelete(kvObject datastore.KVObject) error {
+	if d.store == nil {
+		logrus.Debugf("bridge store not initialized. kv object %s is not deleted from store", datastore.Key(kvObject.Key()...))
+		return nil
+	}
+
+retry:
+	if err := d.store.DeleteObjectAtomic(kvObject); err != nil {
+		if err == datastore.ErrKeyModified {
+			if err := d.store.GetObject(datastore.Key(kvObject.Key()...), kvObject); err != nil {
+				return fmt.Errorf("could not update the kvobject to latest when trying to delete: %v", err)
+			}
+			goto retry
+		}
+		return err
+	}
+
+	return nil
+}
+
+func (ncfg *networkConfiguration) MarshalJSON() ([]byte, error) {
+	nMap := make(map[string]interface{})
+	nMap["ID"] = ncfg.ID
+	nMap["BridgeName"] = ncfg.BridgeName
+	nMap["EnableIPv6"] = ncfg.EnableIPv6
+	nMap["EnableIPMasquerade"] = ncfg.EnableIPMasquerade
+	nMap["EnableICC"] = ncfg.EnableICC
+	nMap["Mtu"] = ncfg.Mtu
+	nMap["DefaultBridge"] = ncfg.DefaultBridge
+	nMap["DefaultBindingIP"] = ncfg.DefaultBindingIP.String()
+	nMap["DefaultGatewayIPv4"] = ncfg.DefaultGatewayIPv4.String()
+	nMap["DefaultGatewayIPv6"] = ncfg.DefaultGatewayIPv6.String()
+
+	if ncfg.AddressIPv4 != nil {
+		nMap["AddressIPv4"] = ncfg.AddressIPv4.String()
+	}
+
+	if ncfg.AddressIPv6 != nil {
+		nMap["AddressIPv6"] = ncfg.AddressIPv6.String()
+	}
+
+	return json.Marshal(nMap)
+}
+
+func (ncfg *networkConfiguration) UnmarshalJSON(b []byte) error {
+	var (
+		err  error
+		nMap map[string]interface{}
+	)
+
+	if err = json.Unmarshal(b, &nMap); err != nil {
+		return err
+	}
+
+	if v, ok := nMap["AddressIPv4"]; ok {
+		if ncfg.AddressIPv4, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge network address IPv4 after json unmarshal: %s", v.(string))
+		}
+	}
+
+	if v, ok := nMap["AddressIPv6"]; ok {
+		if ncfg.AddressIPv6, err = types.ParseCIDR(v.(string)); err != nil {
+			return types.InternalErrorf("failed to decode bridge network address IPv6 after json unmarshal: %s", v.(string))
+		}
+	}
+
+	ncfg.DefaultBindingIP = net.ParseIP(nMap["DefaultBindingIP"].(string))
+	ncfg.DefaultGatewayIPv4 = net.ParseIP(nMap["DefaultGatewayIPv4"].(string))
+	ncfg.DefaultGatewayIPv6 = net.ParseIP(nMap["DefaultGatewayIPv6"].(string))
+	ncfg.ID = nMap["ID"].(string)
+	ncfg.BridgeName = nMap["BridgeName"].(string)
+	ncfg.EnableIPv6 = nMap["EnableIPv6"].(bool)
+	ncfg.EnableIPMasquerade = nMap["EnableIPMasquerade"].(bool)
+	ncfg.EnableICC = nMap["EnableICC"].(bool)
+	ncfg.Mtu = int(nMap["Mtu"].(float64))
+
+	return nil
+}
+
+func (ncfg *networkConfiguration) Key() []string {
+	return []string{bridgePrefix, ncfg.ID}
+}
+
+func (ncfg *networkConfiguration) KeyPrefix() []string {
+	return []string{bridgePrefix}
+}
+
+func (ncfg *networkConfiguration) Value() []byte {
+	b, err := json.Marshal(ncfg)
+	if err != nil {
+		return nil
+	}
+	return b
+}
+
+func (ncfg *networkConfiguration) SetValue(value []byte) error {
+	return json.Unmarshal(value, ncfg)
+}
+
+func (ncfg *networkConfiguration) Index() uint64 {
+	return ncfg.dbIndex
+}
+
+func (ncfg *networkConfiguration) SetIndex(index uint64) {
+	ncfg.dbIndex = index
+	ncfg.dbExists = true
+}
+
+func (ncfg *networkConfiguration) Exists() bool {
+	return ncfg.dbExists
+}
+
+func (ncfg *networkConfiguration) Skip() bool {
+	return false
+}
+
+func (ncfg *networkConfiguration) New() datastore.KVObject {
+	return &networkConfiguration{}
+}
+
+func (ncfg *networkConfiguration) CopyTo(o datastore.KVObject) error {
+	dstNcfg := o.(*networkConfiguration)
+	*dstNcfg = *ncfg
+	return nil
+}
+
+func (ncfg *networkConfiguration) DataScope() string {
+	return datastore.LocalScope
+}

libnetwork/drivers/bridge/bridge_test.go

@@ -120,14 +120,6 @@ func TestCreate(t *testing.T) {
 	if _, ok := err.(types.ForbiddenError); !ok {
 		t.Fatalf("Creation of second network with default name failed with unexpected error type")
 	}
-
-	err = d.DeleteNetwork("dummy")
-	if err == nil {
-		t.Fatalf("deletion of network with default name should fail on this driver")
-	}
-	if _, ok := err.(types.ForbiddenError); !ok {
-		t.Fatalf("deletion of network with default name failed with unexpected error type")
-	}
 }
 
 func TestCreateFail(t *testing.T) {

libnetwork/ipam/allocator_test.go

@@ -32,7 +32,6 @@ func randomLocalStore() (datastore.DataStore, error) {
 		return nil, fmt.Errorf("Error closing temp file: %v", err)
 	}
 	return datastore.NewDataStore(datastore.LocalScope, &datastore.ScopeCfg{
-		Embedded: true,
 		Client: datastore.ScopeClientCfg{
 			Provider: "boltdb",
 			Address:  defaultPrefix + tmp.Name(),

libnetwork/network.go

@@ -557,17 +557,6 @@ func (n *network) deleteNetwork() error {
 		return fmt.Errorf("failed deleting network: %v", err)
 	}
 
-	// If it is bridge network type make sure we call the driver about the network
-	// because the network may have been created in some past life of libnetwork.
-	if n.Type() == "bridge" {
-		n.drvOnce.Do(func() {
-			err = n.getController().addNetwork(n)
-		})
-		if err != nil {
-			return err
-		}
-	}
-
 	if err := d.DeleteNetwork(n.ID()); err != nil {
 		// Forbidden Errors should be honored
 		if _, ok := err.(types.ForbiddenError); ok {
@@ -585,17 +574,6 @@ func (n *network) addEndpoint(ep *endpoint) error {
 		return fmt.Errorf("failed to add endpoint: %v", err)
 	}
 
-	// If it is bridge network type make sure we call the driver about the network
-	// because the network may have been created in some past life of libnetwork.
-	if n.Type() == "bridge" {
-		n.drvOnce.Do(func() {
-			err = n.getController().addNetwork(n)
-		})
-		if err != nil {
-			return err
-		}
-	}
-
 	err = d.CreateEndpoint(n.id, ep.id, ep.Interface(), ep.generic)
 	if err != nil {
 		return types.InternalErrorf("failed to create endpoint %s on network %s: %v",

libnetwork/test/integration/dnet/bridge.bats

@@ -49,6 +49,21 @@ function test_single_network_connectivity() {
     test_single_network_connectivity bridge 3
 }
 
+@test "Test default network dnet restart" {
+    skip_for_circleci
+
+    echo $(docker ps)
+
+    for iter in `seq 1 2`;
+    do
+	test_single_network_connectivity bridge 3
+	if [ "$iter" -eq 1 ]; then
+	    docker restart dnet-1-bridge
+	    sleep 5
+	fi
+    done
+}
+
 @test "Test bridge network" {
     skip_for_circleci
 
@@ -67,8 +82,10 @@ function test_single_network_connectivity() {
     for iter in `seq 1 2`;
     do
 	test_single_network_connectivity singlehost 3
-	docker restart dnet-1-bridge
-	sleep 2
+	if [ "$iter" -eq 1 ]; then
+	    docker restart dnet-1-bridge
+	    sleep 5
+	fi
     done
 
     dnet_cmd $(inst_id2port 1) network rm singlehost
@@ -84,12 +101,11 @@ function test_single_network_connectivity() {
     do
 	if [ "$iter" -eq 1 ]; then
 	    test_single_network_connectivity singlehost 3 skip
+	    docker restart dnet-1-bridge
+	    sleep 5
 	else
 	    test_single_network_connectivity singlehost 3
 	fi
-
-	docker restart dnet-1-bridge
-	sleep 5
     done
 
     dnet_cmd $(inst_id2port 1) network rm singlehost

libnetwork/test/integration/dnet/helpers.bash

@@ -110,7 +110,7 @@ EOF
 	   -v /usr/local/bin/runc:/usr/local/bin/runc \
 	   -w /go/src/github.com/docker/libnetwork \
 	   golang:1.4 ./cmd/dnet/dnet -d -D ${hopt} -c ${tomlfile}
-    sleep 2
+    sleep 3
 }
 
 function skip_for_circleci() {