diff --git a/libnetwork/agent.go b/libnetwork/agent.go index b4b7bdf693..d67d446fe8 100644 --- a/libnetwork/agent.go +++ b/libnetwork/agent.go @@ -222,7 +222,7 @@ func (c *controller) agentSetup(clusterProvider cluster.Provider) error { return err } c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool { - if capability.DataScope == datastore.GlobalScope { + if capability.ConnectivityScope == datastore.GlobalScope { c.agentDriverNotify(driver) } return false @@ -507,7 +507,7 @@ func (n *network) Services() map[string]ServiceInfo { } func (n *network) isClusterEligible() bool { - if n.driverScope() != datastore.GlobalScope { + if n.scope != datastore.SwarmScope || !n.driverIsMultihost() { return false } return n.getController().getAgent() != nil diff --git a/libnetwork/controller.go b/libnetwork/controller.go index 84c3ee3d37..159e40f646 100644 --- a/libnetwork/controller.go +++ b/libnetwork/controller.go @@ -621,7 +621,7 @@ func (c *controller) pushNodeDiscovery(d driverapi.Driver, cap driverapi.Capabil } } - if d == nil || cap.DataScope != datastore.GlobalScope || nodes == nil { + if d == nil || cap.ConnectivityScope != datastore.GlobalScope || nodes == nil { return } @@ -747,11 +747,17 @@ func (c *controller) NewNetwork(networkType, name string, id string, options ... return nil, err } + if network.scope == datastore.LocalScope && cap.DataScope == datastore.GlobalScope { + return nil, types.ForbiddenErrorf("cannot downgrade network scope for %s networks", networkType) + + } if network.ingress && cap.DataScope != datastore.GlobalScope { return nil, types.ForbiddenErrorf("Ingress network can only be global scope network") } - if cap.DataScope == datastore.GlobalScope && !c.isDistributedControl() && !network.dynamic { + // At this point the network scope is still unknown if not set by user + if (cap.DataScope == datastore.GlobalScope || network.scope == datastore.SwarmScope) && + !c.isDistributedControl() && !network.dynamic { if c.isManager() { // For non-distributed controlled environment, globalscoped non-dynamic networks are redirected to Manager return nil, ManagerRedirectError(name) diff --git a/libnetwork/datastore/datastore.go b/libnetwork/datastore/datastore.go index 19bf0b026b..82feef1c84 100644 --- a/libnetwork/datastore/datastore.go +++ b/libnetwork/datastore/datastore.go @@ -115,7 +115,10 @@ const ( // LocalScope indicates to store the KV object in local datastore such as boltdb LocalScope = "local" // GlobalScope indicates to store the KV object in global datastore such as consul/etcd/zookeeper - GlobalScope = "global" + GlobalScope = "global" + // SwarmScope is not indicating a datastore location. It is defined here + // along with the other two scopes just for consistency. + SwarmScope = "swarm" defaultPrefix = "/var/lib/docker/network/files" ) diff --git a/libnetwork/docs/remote.md b/libnetwork/docs/remote.md index 48fd5e58e9..56125080fe 100644 --- a/libnetwork/docs/remote.md +++ b/libnetwork/docs/remote.md @@ -56,10 +56,12 @@ Other entries in the list value are allowed; `"NetworkDriver"` indicates that th After Handshake, the remote driver will receive another POST message to the URL `/NetworkDriver.GetCapabilities` with no payload. The driver's response should have the form: { - "Scope": "local" + "Scope": "local" + "ConnectivityScope": "global" } -Value of "Scope" should be either "local" or "global" which indicates the capability of remote driver, values beyond these will fail driver's registration and return an error to the caller. +Value of "Scope" should be either "local" or "global" which indicates whether the resource allocations for this driver's network can be done only locally to the node or globally across the cluster of nodes. Any other value will fail driver's registration and return an error to the caller. +Similarly, value of "ConnectivityScope" should be either "local" or "global" which indicates whether the driver's network can provide connectivity only locally to this node or globally across the cluster of nodes. If the value is missing, libnetwork will set it to the value of "Scope". should be either "local" or "global" which indicates ### Create network diff --git a/libnetwork/driverapi/driverapi.go b/libnetwork/driverapi/driverapi.go index 074438ef88..48a14ae57a 100644 --- a/libnetwork/driverapi/driverapi.go +++ b/libnetwork/driverapi/driverapi.go @@ -161,7 +161,8 @@ type DriverCallback interface { // Capability represents the high level capabilities of the drivers which libnetwork can make use of type Capability struct { - DataScope string + DataScope string + ConnectivityScope string } // IPAMData represents the per-network ip related diff --git a/libnetwork/drivers/bridge/bridge.go b/libnetwork/drivers/bridge/bridge.go index e681b8f7c4..dd79f04d91 100644 --- a/libnetwork/drivers/bridge/bridge.go +++ b/libnetwork/drivers/bridge/bridge.go @@ -153,7 +153,8 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { } c := driverapi.Capability{ - DataScope: datastore.LocalScope, + DataScope: datastore.LocalScope, + ConnectivityScope: datastore.LocalScope, } return dc.RegisterDriver(networkType, d, c) } diff --git a/libnetwork/drivers/host/host.go b/libnetwork/drivers/host/host.go index 7b4a986e6c..a71d461380 100644 --- a/libnetwork/drivers/host/host.go +++ b/libnetwork/drivers/host/host.go @@ -19,7 +19,8 @@ type driver struct { // Init registers a new instance of host driver func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { c := driverapi.Capability{ - DataScope: datastore.LocalScope, + DataScope: datastore.LocalScope, + ConnectivityScope: datastore.LocalScope, } return dc.RegisterDriver(networkType, &driver{}, c) } diff --git a/libnetwork/drivers/ipvlan/ipvlan.go b/libnetwork/drivers/ipvlan/ipvlan.go index 296804dc1a..c64ad555a3 100644 --- a/libnetwork/drivers/ipvlan/ipvlan.go +++ b/libnetwork/drivers/ipvlan/ipvlan.go @@ -58,7 +58,8 @@ type network struct { // Init initializes and registers the libnetwork ipvlan driver func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { c := driverapi.Capability{ - DataScope: datastore.LocalScope, + DataScope: datastore.LocalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ networks: networkTable{}, diff --git a/libnetwork/drivers/macvlan/macvlan.go b/libnetwork/drivers/macvlan/macvlan.go index 49b9fbae00..872e6f3ec1 100644 --- a/libnetwork/drivers/macvlan/macvlan.go +++ b/libnetwork/drivers/macvlan/macvlan.go @@ -60,7 +60,8 @@ type network struct { // Init initializes and registers the libnetwork macvlan driver func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { c := driverapi.Capability{ - DataScope: datastore.LocalScope, + DataScope: datastore.LocalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ networks: networkTable{}, diff --git a/libnetwork/drivers/overlay/overlay.go b/libnetwork/drivers/overlay/overlay.go index 88e1010c43..8a932cf370 100644 --- a/libnetwork/drivers/overlay/overlay.go +++ b/libnetwork/drivers/overlay/overlay.go @@ -56,7 +56,8 @@ type driver struct { // Init registers a new instance of overlay driver func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { c := driverapi.Capability{ - DataScope: datastore.GlobalScope, + DataScope: datastore.GlobalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ networks: networkTable{}, diff --git a/libnetwork/drivers/overlay/ovmanager/ovmanager.go b/libnetwork/drivers/overlay/ovmanager/ovmanager.go index dce8f98b8e..3c3c0bb257 100644 --- a/libnetwork/drivers/overlay/ovmanager/ovmanager.go +++ b/libnetwork/drivers/overlay/ovmanager/ovmanager.go @@ -49,7 +49,8 @@ type network struct { func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { var err error c := driverapi.Capability{ - DataScope: datastore.GlobalScope, + DataScope: datastore.GlobalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ diff --git a/libnetwork/drivers/remote/api/api.go b/libnetwork/drivers/remote/api/api.go index f9a341c540..d24f190162 100644 --- a/libnetwork/drivers/remote/api/api.go +++ b/libnetwork/drivers/remote/api/api.go @@ -24,7 +24,8 @@ func (r *Response) GetError() string { // GetCapabilityResponse is the response of GetCapability request type GetCapabilityResponse struct { Response - Scope string + Scope string + ConnectivityScope string } // AllocateNetworkRequest requests allocation of new network by manager diff --git a/libnetwork/drivers/remote/driver.go b/libnetwork/drivers/remote/driver.go index 49a7fb4951..ffe0730c95 100644 --- a/libnetwork/drivers/remote/driver.go +++ b/libnetwork/drivers/remote/driver.go @@ -74,6 +74,17 @@ func (d *driver) getCapabilities() (*driverapi.Capability, error) { return nil, fmt.Errorf("invalid capability: expecting 'local' or 'global', got %s", capResp.Scope) } + switch capResp.ConnectivityScope { + case "global": + c.ConnectivityScope = datastore.GlobalScope + case "local": + c.ConnectivityScope = datastore.LocalScope + case "": + c.ConnectivityScope = c.DataScope + default: + return nil, fmt.Errorf("invalid capability: expecting 'local' or 'global', got %s", capResp.Scope) + } + return c, nil } diff --git a/libnetwork/drivers/remote/driver_test.go b/libnetwork/drivers/remote/driver_test.go index e559a61299..8a97bacffd 100644 --- a/libnetwork/drivers/remote/driver_test.go +++ b/libnetwork/drivers/remote/driver_test.go @@ -238,8 +238,9 @@ func TestGetExtraCapabilities(t *testing.T) { handle(t, mux, "GetCapabilities", func(msg map[string]interface{}) interface{} { return map[string]interface{}{ - "Scope": "local", - "foo": "bar", + "Scope": "local", + "foo": "bar", + "ConnectivityScope": "global", } }) @@ -258,6 +259,8 @@ func TestGetExtraCapabilities(t *testing.T) { t.Fatal(err) } else if c.DataScope != datastore.LocalScope { t.Fatalf("get capability '%s', expecting 'local'", c.DataScope) + } else if c.ConnectivityScope != datastore.GlobalScope { + t.Fatalf("get capability '%s', expecting %q", c.ConnectivityScope, datastore.GlobalScope) } } diff --git a/libnetwork/drivers/solaris/bridge/bridge.go b/libnetwork/drivers/solaris/bridge/bridge.go index 13dd5f14bc..2547016a4b 100644 --- a/libnetwork/drivers/solaris/bridge/bridge.go +++ b/libnetwork/drivers/solaris/bridge/bridge.go @@ -159,7 +159,8 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { } c := driverapi.Capability{ - DataScope: datastore.LocalScope, + DataScope: datastore.LocalScope, + ConnectivityScope: datastore.LocalScope, } return dc.RegisterDriver(networkType, d, c) } diff --git a/libnetwork/drivers/solaris/overlay/overlay.go b/libnetwork/drivers/solaris/overlay/overlay.go index b001f58af7..0a5a1bfdce 100644 --- a/libnetwork/drivers/solaris/overlay/overlay.go +++ b/libnetwork/drivers/solaris/overlay/overlay.go @@ -57,7 +57,8 @@ type driver struct { // Init registers a new instance of overlay driver func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { c := driverapi.Capability{ - DataScope: datastore.GlobalScope, + DataScope: datastore.GlobalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ networks: networkTable{}, diff --git a/libnetwork/drivers/solaris/overlay/ovmanager/ovmanager.go b/libnetwork/drivers/solaris/overlay/ovmanager/ovmanager.go index f053ff9784..a3a7cd4cf8 100644 --- a/libnetwork/drivers/solaris/overlay/ovmanager/ovmanager.go +++ b/libnetwork/drivers/solaris/overlay/ovmanager/ovmanager.go @@ -49,7 +49,8 @@ type network struct { func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { var err error c := driverapi.Capability{ - DataScope: datastore.GlobalScope, + DataScope: datastore.GlobalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ diff --git a/libnetwork/drivers/windows/overlay/overlay_windows.go b/libnetwork/drivers/windows/overlay/overlay_windows.go index 2ae2cec77e..111e517dbd 100644 --- a/libnetwork/drivers/windows/overlay/overlay_windows.go +++ b/libnetwork/drivers/windows/overlay/overlay_windows.go @@ -36,7 +36,8 @@ type driver struct { // Init registers a new instance of overlay driver func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { c := driverapi.Capability{ - DataScope: datastore.GlobalScope, + DataScope: datastore.GlobalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ diff --git a/libnetwork/drivers/windows/windows.go b/libnetwork/drivers/windows/windows.go index b6591a1d6d..84089a5284 100644 --- a/libnetwork/drivers/windows/windows.go +++ b/libnetwork/drivers/windows/windows.go @@ -104,7 +104,8 @@ func GetInit(networkType string) func(dc driverapi.DriverCallback, config map[st } return dc.RegisterDriver(networkType, newDriver(networkType), driverapi.Capability{ - DataScope: datastore.LocalScope, + DataScope: datastore.LocalScope, + ConnectivityScope: datastore.LocalScope, }) } } diff --git a/libnetwork/network.go b/libnetwork/network.go index 7be7b0af9b..5fcca95fc4 100644 --- a/libnetwork/network.go +++ b/libnetwork/network.go @@ -195,7 +195,7 @@ type network struct { networkType string id string created time.Time - scope string + scope string // network data scope labels map[string]string ipamType string ipamOptions map[string]string @@ -514,7 +514,12 @@ func (n *network) CopyTo(o datastore.KVObject) error { } func (n *network) DataScope() string { - return n.Scope() + s := n.Scope() + // All swarm scope networks have local datascope + if s == datastore.SwarmScope { + s = datastore.LocalScope + } + return s } func (n *network) getEpCnt() *endpointCnt { @@ -762,6 +767,14 @@ func NetworkOptionAttachable(attachable bool) NetworkOption { } } +// NetworkOptionScope returns an option setter to overwrite the network's scope. +// By default the network's scope is set to the network driver's datascope. +func NetworkOptionScope(scope string) NetworkOption { + return func(n *network) { + n.scope = scope + } +} + // NetworkOptionIpam function returns an option setter for the ipam configuration for this network func NetworkOptionIpam(ipamDriver string, addrSpace string, ipV4 []*IpamConf, ipV6 []*IpamConf, opts map[string]string) NetworkOption { return func(n *network) { @@ -877,6 +890,14 @@ func (n *network) driverScope() string { return cap.DataScope } +func (n *network) driverIsMultihost() bool { + _, cap, err := n.resolveDriver(n.networkType, true) + if err != nil { + return false + } + return cap.ConnectivityScope == datastore.GlobalScope +} + func (n *network) driver(load bool) (driverapi.Driver, error) { d, cap, err := n.resolveDriver(n.networkType, load) if err != nil { @@ -887,14 +908,14 @@ func (n *network) driver(load bool) (driverapi.Driver, error) { isAgent := c.isAgent() n.Lock() // If load is not required, driver, cap and err may all be nil - if cap != nil { + if n.scope == "" && cap != nil { n.scope = cap.DataScope } - if isAgent || n.dynamic { - // If we are running in agent mode then all networks - // in libnetwork are local scope regardless of the - // backing driver. - n.scope = datastore.LocalScope + if isAgent && n.dynamic { + // If we are running in agent mode and the network + // is dynamic, then the networks are swarm scoped + // regardless of the backing driver. + n.scope = datastore.SwarmScope } n.Unlock() return d, nil diff --git a/libnetwork/store.go b/libnetwork/store.go index cb220561b6..1a8dceca78 100644 --- a/libnetwork/store.go +++ b/libnetwork/store.go @@ -350,17 +350,18 @@ func (c *controller) networkWatchLoop(nw *netWatch, ep *endpoint, ecCh <-chan da } func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoint) { - if !c.isDistributedControl() && ep.getNetwork().driverScope() == datastore.GlobalScope { + n := ep.getNetwork() + if !c.isDistributedControl() && n.Scope() == datastore.SwarmScope && n.driverIsMultihost() { return } c.Lock() - nw, ok := nmap[ep.getNetwork().ID()] + nw, ok := nmap[n.ID()] c.Unlock() if ok { // Update the svc db for the local endpoint join right away - ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), true) + n.updateSvcRecord(ep, c.getLocalEps(nw), true) c.Lock() nw.localEps[ep.ID()] = ep @@ -381,15 +382,15 @@ func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoi // Update the svc db for the local endpoint join right away // Do this before adding this ep to localEps so that we don't // try to update this ep's container's svc records - ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), true) + n.updateSvcRecord(ep, c.getLocalEps(nw), true) c.Lock() nw.localEps[ep.ID()] = ep - nmap[ep.getNetwork().ID()] = nw + nmap[n.ID()] = nw nw.stopCh = make(chan struct{}) c.Unlock() - store := c.getStore(ep.getNetwork().DataScope()) + store := c.getStore(n.DataScope()) if store == nil { return } @@ -398,7 +399,7 @@ func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoi return } - ch, err := store.Watch(ep.getNetwork().getEpCnt(), nw.stopCh) + ch, err := store.Watch(n.getEpCnt(), nw.stopCh) if err != nil { logrus.Warnf("Error creating watch for network: %v", err) return @@ -408,12 +409,13 @@ func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoi } func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoint) { - if !c.isDistributedControl() && ep.getNetwork().driverScope() == datastore.GlobalScope { + n := ep.getNetwork() + if !c.isDistributedControl() && n.Scope() == datastore.SwarmScope && n.driverIsMultihost() { return } c.Lock() - nw, ok := nmap[ep.getNetwork().ID()] + nw, ok := nmap[n.ID()] if ok { delete(nw.localEps, ep.ID()) @@ -422,7 +424,7 @@ func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoi // Update the svc db about local endpoint leave right away // Do this after we remove this ep from localEps so that we // don't try to remove this svc record from this ep's container. - ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), false) + n.updateSvcRecord(ep, c.getLocalEps(nw), false) c.Lock() if len(nw.localEps) == 0 { @@ -430,9 +432,9 @@ func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoi // This is the last container going away for the network. Destroy // this network's svc db entry - delete(c.svcRecords, ep.getNetwork().ID()) + delete(c.svcRecords, n.ID()) - delete(nmap, ep.getNetwork().ID()) + delete(nmap, n.ID()) } } c.Unlock()