diff --git a/libnetwork/controller.go b/libnetwork/controller.go index ed05d23ad7..f834b7b370 100644 --- a/libnetwork/controller.go +++ b/libnetwork/controller.go @@ -256,7 +256,7 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti return nil, err } - if err := c.updateNetworkToStore(network); err != nil { + if err := c.updateToStore(network); err != nil { log.Warnf("couldnt create network %s: %v", network.name, err) if e := network.Delete(); e != nil { log.Warnf("couldnt cleanup network %s: %v", network.name, err) @@ -293,8 +293,10 @@ func (c *controller) addNetwork(n *network) error { if err := d.CreateNetwork(n.id, n.generic); err != nil { return err } - if err := n.watchEndpoints(); err != nil { - return err + if n.isGlobalScoped() { + if err := n.watchEndpoints(); err != nil { + return err + } } c.Lock() c.networks[n.id] = n diff --git a/libnetwork/drivers.go b/libnetwork/drivers.go index 4bce703832..898bcc4ebe 100644 --- a/libnetwork/drivers.go +++ b/libnetwork/drivers.go @@ -29,9 +29,9 @@ func makeDriverConfig(c *controller, ntype string) map[string]interface{} { config := make(map[string]interface{}) - if c.validateDatastoreConfig() { - config[netlabel.KVProvider] = c.cfg.Datastore.Client.Provider - config[netlabel.KVProviderURL] = c.cfg.Datastore.Client.Address + if c.validateGlobalStoreConfig() { + config[netlabel.KVProvider] = c.cfg.GlobalStore.Client.Provider + config[netlabel.KVProviderURL] = c.cfg.GlobalStore.Client.Address } for _, label := range c.cfg.Daemon.Labels { diff --git a/libnetwork/drivers/remote/driver_test.go b/libnetwork/drivers/remote/driver_test.go index 98d9bee074..627106e9e8 100644 --- a/libnetwork/drivers/remote/driver_test.go +++ b/libnetwork/drivers/remote/driver_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/docker/docker/pkg/plugins" + "github.com/docker/libnetwork/datastore" "github.com/docker/libnetwork/driverapi" _ "github.com/docker/libnetwork/testutils" "github.com/docker/libnetwork/types" @@ -205,8 +206,8 @@ func TestGetExtraCapabilities(t *testing.T) { c, err := d.(*driver).getCapabilities() if err != nil { t.Fatal(err) - } else if c.Scope != driverapi.LocalScope { - t.Fatalf("get capability '%s', expecting 'local'", c.Scope) + } else if c.DataScope != datastore.LocalScope { + t.Fatalf("get capability '%s', expecting 'local'", c.DataScope) } } @@ -343,8 +344,8 @@ func TestRemoteDriver(t *testing.T) { c, err := d.(*driver).getCapabilities() if err != nil { t.Fatal(err) - } else if c.Scope != driverapi.GlobalScope { - t.Fatalf("get capability '%s', expecting 'global'", c.Scope) + } else if c.DataScope != datastore.GlobalScope { + t.Fatalf("get capability '%s', expecting 'global'", c.DataScope) } netID := "dummy-network" diff --git a/libnetwork/endpoint.go b/libnetwork/endpoint.go index ab8645c5e7..6faa71dbbc 100644 --- a/libnetwork/endpoint.go +++ b/libnetwork/endpoint.go @@ -282,8 +282,10 @@ func (ep *endpoint) Join(sbox Sandbox, options ...EndpointOption) error { return err } - if err = network.ctrlr.updateEndpointToStore(ep); err != nil { - return err + if !ep.isLocalScoped() { + if err = network.ctrlr.updateToStore(ep); err != nil { + return err + } } sb.Lock() @@ -355,11 +357,13 @@ func (ep *endpoint) Leave(sbox Sandbox, options ...EndpointOption) error { d := n.driver n.Unlock() - if err := c.updateEndpointToStore(ep); err != nil { - ep.Lock() - ep.sandboxID = sid - ep.Unlock() - return err + if !ep.isLocalScoped() { + if err := c.updateToStore(ep); err != nil { + ep.Lock() + ep.sandboxID = sid + ep.Unlock() + return err + } } if err := d.Leave(n.id, ep.id); err != nil { @@ -395,27 +399,31 @@ func (ep *endpoint) Delete() error { n.Unlock() ep.Unlock() - if err = ctrlr.deleteEndpointFromStore(ep); err != nil { - return err + if !ep.isLocalScoped() { + if err = ctrlr.deleteFromStore(ep); err != nil { + return err + } } defer func() { if err != nil { - ep.SetIndex(0) - if e := ctrlr.updateEndpointToStore(ep); e != nil { - log.Warnf("failed to recreate endpoint in store %s : %v", name, err) + ep.dbExists = false + if !ep.isLocalScoped() { + if e := ctrlr.updateToStore(ep); e != nil { + log.Warnf("failed to recreate endpoint in store %s : %v", name, e) + } } } }() // Update the endpoint count in network and update it in the datastore n.DecEndpointCnt() - if err = ctrlr.updateNetworkToStore(n); err != nil { + if err = ctrlr.updateToStore(n); err != nil { return err } defer func() { if err != nil { n.IncEndpointCnt() - if e := ctrlr.updateNetworkToStore(n); e != nil { + if e := ctrlr.updateToStore(n); e != nil { log.Warnf("failed to update network %s : %v", n.name, e) } } @@ -547,3 +555,7 @@ func (ep *endpoint) DataScope() datastore.DataScope { defer ep.Unlock() return ep.network.dataScope } + +func (ep *endpoint) isLocalScoped() bool { + return ep.DataScope() == datastore.LocalScope +} diff --git a/libnetwork/libnetwork_test.go b/libnetwork/libnetwork_test.go index b1245c29ac..82e3dd642a 100644 --- a/libnetwork/libnetwork_test.go +++ b/libnetwork/libnetwork_test.go @@ -47,6 +47,7 @@ func TestMain(m *testing.M) { } if err := createController(); err != nil { + log.Errorf("Error creating controller: %v", err) os.Exit(1) } @@ -65,7 +66,11 @@ func createController() error { genericOption := make(map[string]interface{}) genericOption[netlabel.GenericData] = option - controller, err = libnetwork.New(config.OptionDriverConfig(bridgeNetType, genericOption)) + cfgOptions, err := libnetwork.OptionBoltdbWithRandomDBFile() + if err != nil { + return err + } + controller, err = libnetwork.New(append(cfgOptions, config.OptionDriverConfig(bridgeNetType, genericOption))...) if err != nil { return err } diff --git a/libnetwork/network.go b/libnetwork/network.go index a677566f33..855a365e1f 100644 --- a/libnetwork/network.go +++ b/libnetwork/network.go @@ -240,13 +240,22 @@ func (n *network) Delete() error { // deleteNetworkFromStore performs an atomic delete operation and the network.endpointCnt field will help // prevent any possible race between endpoint join and network delete - if err = ctrlr.deleteNetworkFromStore(n); err != nil { + if err = ctrlr.deleteFromStore(n); err != nil { if err == datastore.ErrKeyModified { return types.InternalErrorf("operation in progress. delete failed for network %s. Please try again.") } return err } + defer func() { + if err != nil { + n.dbExists = false + if e := ctrlr.updateToStore(n); e != nil { + log.Warnf("failed to recreate network in store %s : %v", n.name, e) + } + } + }() + if err = n.deleteNetwork(); err != nil { return err } @@ -322,13 +331,13 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi n.Unlock() n.IncEndpointCnt() - if err = ctrlr.updateNetworkToStore(n); err != nil { + if err = ctrlr.updateToStore(n); err != nil { return nil, err } defer func() { if err != nil { n.DecEndpointCnt() - if err = ctrlr.updateNetworkToStore(n); err != nil { + if err = ctrlr.updateToStore(n); err != nil { log.Warnf("endpoint count cleanup failed when updating network for %s : %v", name, err) } } @@ -344,8 +353,10 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi } }() - if err = ctrlr.updateEndpointToStore(ep); err != nil { - return nil, err + if !ep.isLocalScoped() { + if err = ctrlr.updateToStore(ep); err != nil { + return nil, err + } } return ep, nil diff --git a/libnetwork/sandbox_test.go b/libnetwork/sandbox_test.go index e573f3511f..60fbf174e7 100644 --- a/libnetwork/sandbox_test.go +++ b/libnetwork/sandbox_test.go @@ -23,7 +23,11 @@ func getTestEnv(t *testing.T) (NetworkController, Network, Network) { genericOption := make(map[string]interface{}) genericOption[netlabel.GenericData] = option - c, err := New(config.OptionDriverConfig(netType, genericOption)) + cfgOptions, err := OptionBoltdbWithRandomDBFile() + if err != nil { + t.Fatal(err) + } + c, err := New(append(cfgOptions, config.OptionDriverConfig(netType, genericOption))...) if err != nil { t.Fatal(err) } diff --git a/libnetwork/store.go b/libnetwork/store.go index b8c7d55f90..4c2bcd58b7 100644 --- a/libnetwork/store.go +++ b/libnetwork/store.go @@ -43,7 +43,7 @@ func (c *controller) initGlobalStore() error { c.globalStore = store c.Unlock() - nws, err := c.getNetworksFromGlobalStore() + nws, err := c.getNetworksFromStore(true) if err == nil { c.processNetworkUpdate(nws, nil) } else if err != datastore.ErrKeyNotFound { @@ -64,24 +64,23 @@ func (c *controller) initLocalStore() error { c.localStore = localStore c.Unlock() - nws, err := c.getNetworksFromLocalStore() + nws, err := c.getNetworksFromStore(false) if err == nil { c.processNetworkUpdate(nws, nil) } else if err != datastore.ErrKeyNotFound { log.Warnf("failed to read networks from localstore during init : %v", err) } - eps, err := c.getEndpointsFromLocalStore() - if err == nil { - c.processEndpointsUpdate(eps, nil) - } else if err != datastore.ErrKeyNotFound { - log.Warnf("failed to read endpoints from localstore during init : %v", err) - } return nil } -func (c *controller) getNetworksFromGlobalStore() ([]*store.KVPair, error) { +func (c *controller) getNetworksFromStore(global bool) ([]*store.KVPair, error) { + var cs datastore.DataStore c.Lock() - cs := c.globalStore + if global { + cs = c.globalStore + } else { + cs = c.localStore + } c.Unlock() return cs.KVStore().List(datastore.Key(datastore.NetworkKeyPrefix)) } @@ -95,30 +94,6 @@ func (c *controller) newNetworkFromStore(n *network) error { return c.addNetwork(n) } -func (c *controller) updateNetworkToStore(n *network) error { - cs := c.getDataStore(n.DataScope()) - if cs == nil { - log.Debugf("datastore not initialized. Network %s is not added to the store", n.Name()) - return nil - } - - return cs.PutObjectAtomic(n) -} - -func (c *controller) deleteNetworkFromStore(n *network) error { - cs := c.getDataStore(n.DataScope()) - if cs == nil { - log.Debugf("datastore not initialized. Network %s is not deleted from datastore", n.Name()) - return nil - } - - if err := cs.DeleteObjectAtomic(n); err != nil { - return err - } - - return nil -} - func (c *controller) newEndpointFromStore(key string, ep *endpoint) error { ep.Lock() n := ep.network @@ -134,27 +109,24 @@ func (c *controller) newEndpointFromStore(key string, ep *endpoint) error { return err } -func (c *controller) updateEndpointToStore(ep *endpoint) error { - ep.Lock() - name := ep.name - ep.Unlock() - cs := c.getDataStore(ep.DataScope()) +func (c *controller) updateToStore(kvObject datastore.KV) error { + cs := c.getDataStore(kvObject.DataScope()) if cs == nil { - log.Debugf("datastore not initialized. endpoint %s is not added to the store", name) + log.Debugf("datastore not initialized. kv object %s is not added to the store", datastore.Key(kvObject.Key()...)) return nil } - return cs.PutObjectAtomic(ep) + return cs.PutObjectAtomic(kvObject) } -func (c *controller) deleteEndpointFromStore(ep *endpoint) error { - cs := c.getDataStore(ep.DataScope()) +func (c *controller) deleteFromStore(kvObject datastore.KV) error { + cs := c.getDataStore(kvObject.DataScope()) if cs == nil { - log.Debugf("datastore not initialized. endpoint %s is not deleted from datastore", ep.Name()) + log.Debugf("datastore not initialized. kv object %s is not deleted from datastore", datastore.Key(kvObject.Key()...)) return nil } - if err := cs.DeleteObjectAtomic(ep); err != nil { + if err := cs.DeleteObjectAtomic(kvObject); err != nil { return err } @@ -367,13 +339,6 @@ func (c *controller) getLocalStoreConfig(cfg *config.Config) *config.DatastoreCf return &defaultLocalStoreConfig } -func (c *controller) getNetworksFromLocalStore() ([]*store.KVPair, error) { - c.Lock() - cs := c.localStore - c.Unlock() - return cs.KVStore().List(datastore.Key(datastore.NetworkKeyPrefix)) -} - func (c *controller) getDataStore(dataScope datastore.DataScope) (dataStore datastore.DataStore) { c.Lock() if dataScope == datastore.GlobalScope { @@ -385,13 +350,6 @@ func (c *controller) getDataStore(dataScope datastore.DataScope) (dataStore data return } -func (c *controller) getEndpointsFromLocalStore() ([]*store.KVPair, error) { - c.Lock() - cs := c.localStore - c.Unlock() - return cs.KVStore().List(datastore.Key(datastore.EndpointKeyPrefix)) -} - func (c *controller) processEndpointsUpdate(eps []*store.KVPair, prune *endpointTable) { for _, epe := range eps { var ep endpoint diff --git a/libnetwork/store_test.go b/libnetwork/store_test.go index d97601ba4e..0c9ad943cd 100644 --- a/libnetwork/store_test.go +++ b/libnetwork/store_test.go @@ -1,20 +1,96 @@ package libnetwork import ( + "fmt" + "io/ioutil" + "os" "testing" + "github.com/docker/libkv/store" "github.com/docker/libnetwork/config" + "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/options" ) func TestZooKeeperBackend(t *testing.T) { - testNewController(t, "zk", "127.0.0.1:2181") + if err := testNewController(t, "zk", "127.0.0.1:2181"); err != nil { + t.Fatal(err) + } } func testNewController(t *testing.T, provider, url string) error { - netOptions := []config.Option{} - netOptions = append(netOptions, config.OptionKVProvider(provider)) - netOptions = append(netOptions, config.OptionKVProviderURL(url)) - - _, err := New(netOptions...) + cfgOptions, err := OptionBoltdbWithRandomDBFile() + if err != nil { + return err + } + cfgOptions = append(cfgOptions, config.OptionKVProvider(provider)) + cfgOptions = append(cfgOptions, config.OptionKVProviderURL(url)) + _, err = New(cfgOptions...) return err } + +func TestBoltdbBackend(t *testing.T) { + defer os.Remove(defaultLocalStoreConfig.Client.Address) + testLocalBackend(t, "", "", nil) + defer os.Remove("/tmp/boltdb.db") + testLocalBackend(t, "boltdb", "/tmp/boltdb.db", &store.Config{Bucket: "testBackend"}) +} + +func testLocalBackend(t *testing.T, provider, url string, storeConfig *store.Config) { + cfgOptions := []config.Option{} + cfgOptions = append(cfgOptions, config.OptionLocalKVProvider(provider)) + cfgOptions = append(cfgOptions, config.OptionLocalKVProviderURL(url)) + cfgOptions = append(cfgOptions, config.OptionLocalKVProviderConfig(storeConfig)) + + driverOptions := options.Generic{} + genericOption := make(map[string]interface{}) + genericOption[netlabel.GenericData] = driverOptions + cfgOptions = append(cfgOptions, config.OptionDriverConfig("host", genericOption)) + + ctrl, err := New(cfgOptions...) + if err != nil { + t.Fatalf("Error new controller: %v", err) + } + nw, err := ctrl.NewNetwork("host", "host") + if err != nil { + t.Fatalf("Error creating default \"host\" network: %v", err) + } + ep, err := nw.CreateEndpoint("newendpoint", []EndpointOption{}...) + if err != nil { + t.Fatalf("Error creating endpoint: %v", err) + } + store := ctrl.(*controller).localStore.KVStore() + if exists, err := store.Exists(datastore.Key(datastore.NetworkKeyPrefix, string(nw.ID()))); !exists || err != nil { + t.Fatalf("Network key should have been created.") + } + if exists, err := store.Exists(datastore.Key([]string{datastore.EndpointKeyPrefix, string(nw.ID()), string(ep.ID())}...)); exists || err != nil { + t.Fatalf("Endpoint key shouldn't have been created.") + } + store.Close() + + // test restore of local store + ctrl, err = New(cfgOptions...) + if err != nil { + t.Fatalf("Error creating controller: %v", err) + } + if _, err = ctrl.NetworkByID(nw.ID()); err != nil { + t.Fatalf("Error getting network %v", err) + } +} + +// OptionBoltdbWithRandomDBFile function returns a random dir for local store backend +func OptionBoltdbWithRandomDBFile() ([]config.Option, error) { + tmp, err := ioutil.TempFile("", "libnetwork-") + if err != nil { + return nil, fmt.Errorf("Error creating temp file: %v", err) + } + if err := tmp.Close(); err != nil { + return nil, fmt.Errorf("Error closing temp file: %v", err) + } + cfgOptions := []config.Option{} + cfgOptions = append(cfgOptions, config.OptionLocalKVProvider("boltdb")) + cfgOptions = append(cfgOptions, config.OptionLocalKVProviderURL(tmp.Name())) + cfgOptions = append(cfgOptions, config.OptionLocalKVProviderConfig(&store.Config{Bucket: "testBackend"})) + return cfgOptions, nil +} diff --git a/libnetwork/test/integration/dnet/helpers.bash b/libnetwork/test/integration/dnet/helpers.bash index 7f6cfba05d..7b89ddc78a 100644 --- a/libnetwork/test/integration/dnet/helpers.bash +++ b/libnetwork/test/integration/dnet/helpers.bash @@ -50,9 +50,9 @@ title = "LibNetwork Configuration file" [daemon] debug = false labels = ["com.docker.network.driver.overlay.bind_interface=eth0"] -[datastore] +[globalstore] embedded = false -[datastore.client] +[globalstore.client] provider = "consul" Address = "${bridge_ip}:8500" EOF