Cache scope in network object

Its safe to cache the scope value in network object and can be reused
for cleanup operations. The current implementation assume the presence
of driver during cleanup operation. Since a remote driver may not be
present, we should not fail such cleanup operations. Hence make use of
the scope variable from network object.

Signed-off-by: Madhu Venugopal <madhu@docker.com>

libnetwork/endpoint.go

@@ -962,7 +962,7 @@ func (c *controller) cleanupLocalEndpoints() {
 		}
 
 		for _, ep := range epl {
-			if err := ep.Delete(false); err != nil {
+			if err := ep.Delete(true); err != nil {
 				log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)
 			}
 		}

libnetwork/network.go

@@ -149,6 +149,7 @@ type network struct {
 	name         string
 	networkType  string
 	id           string
+	scope        string
 	ipamType     string
 	ipamOptions  map[string]string
 	addrSpace    string
@@ -246,6 +247,7 @@ func (n *network) New() datastore.KVObject {
 	return &network{
 		ctrlr:   n.ctrlr,
 		drvOnce: &sync.Once{},
+		scope:   n.scope,
 	}
 }
 
@@ -295,6 +297,7 @@ func (n *network) CopyTo(o datastore.KVObject) error {
 	dstN.name = n.name
 	dstN.id = n.id
 	dstN.networkType = n.networkType
+	dstN.scope = n.scope
 	dstN.ipamType = n.ipamType
 	dstN.enableIPv6 = n.enableIPv6
 	dstN.persist = n.persist
@@ -337,7 +340,7 @@ func (n *network) CopyTo(o datastore.KVObject) error {
 }
 
 func (n *network) DataScope() string {
-	return n.driverScope()
+	return n.Scope()
 }
 
 func (n *network) getEpCnt() *endpointCnt {
@@ -353,6 +356,7 @@ func (n *network) MarshalJSON() ([]byte, error) {
 	netMap["name"] = n.name
 	netMap["id"] = n.id
 	netMap["networkType"] = n.networkType
+	netMap["scope"] = n.scope
 	netMap["ipamType"] = n.ipamType
 	netMap["addrSpace"] = n.addrSpace
 	netMap["enableIPv6"] = n.enableIPv6
@@ -456,6 +460,9 @@ func (n *network) UnmarshalJSON(b []byte) (err error) {
 	if v, ok := netMap["internal"]; ok {
 		n.internal = v.(bool)
 	}
+	if s, ok := netMap["scope"]; ok {
+		n.scope = s.(string)
+	}
 	return nil
 }
 
@@ -585,6 +592,9 @@ func (n *network) driver(load bool) (driverapi.Driver, error) {
 		return nil, nil
 	}
 
+	n.Lock()
+	n.scope = dd.capability.DataScope
+	n.Unlock()
 	return dd.driver, nil
 }
 
@@ -1172,7 +1182,9 @@ func (n *network) DriverOptions() map[string]string {
 }
 
 func (n *network) Scope() string {
-	return n.driverScope()
+	n.Lock()
+	defer n.Unlock()
+	return n.scope
 }
 
 func (n *network) IpamConfig() (string, map[string]string, []*IpamConf, []*IpamConf) {

libnetwork/store.go

@@ -75,6 +75,7 @@ func (c *controller) getNetworkFromStore(nid string) (*network, error) {
 		}
 
 		n.epCnt = ec
+		n.scope = store.Scope()
 		return n, nil
 	}
 
@@ -107,6 +108,7 @@ func (c *controller) getNetworksForScope(scope string) ([]*network, error) {
 		}
 
 		n.epCnt = ec
+		n.scope = scope
 		nl = append(nl, n)
 	}
 
@@ -140,6 +142,7 @@ func (c *controller) getNetworksFromStore() ([]*network, error) {
 			}
 
 			n.epCnt = ec
+			n.scope = store.Scope()
 			nl = append(nl, n)
 		}
 	}
@@ -148,17 +151,21 @@ func (c *controller) getNetworksFromStore() ([]*network, error) {
 }
 
 func (n *network) getEndpointFromStore(eid string) (*endpoint, error) {
-	store := n.ctrlr.getStore(n.Scope())
-	if store == nil {
-		return nil, fmt.Errorf("could not find endpoint %s: datastore not found for scope %s", eid, n.Scope())
+	var errors []string
+	for _, store := range n.ctrlr.getStores() {
+		ep := &endpoint{id: eid, network: n}
+		err := store.GetObject(datastore.Key(ep.Key()...), ep)
+		// Continue searching in the next store if the key is not found in this store
+		if err != nil {
+			if err != datastore.ErrKeyNotFound {
+				errors = append(errors, fmt.Sprintf("{%s:%v}, ", store.Scope(), err))
+				log.Debugf("could not find endpoint %s in %s: %v", eid, store.Scope(), err)
+			}
+			continue
+		}
+		return ep, nil
 	}
-
-	ep := &endpoint{id: eid, network: n}
-	err := store.GetObject(datastore.Key(ep.Key()...), ep)
-	if err != nil {
-		return nil, fmt.Errorf("could not find endpoint %s: %v", eid, err)
-	}
-	return ep, nil
+	return nil, fmt.Errorf("could not find endpoint %s: %v", eid, errors)
 }
 
 func (n *network) getEndpointsFromStore() ([]*endpoint, error) {