Merge pull request #29556 from mavenugo/refcount

Fixing a couple of network plugin life-cycle mgmt issues

daemon/network.go

@@ -12,8 +12,11 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/network"
 	clustertypes "github.com/docker/docker/daemon/cluster/provider"
+	"github.com/docker/docker/pkg/plugingetter"
 	"github.com/docker/docker/runconfig"
 	"github.com/docker/libnetwork"
+	"github.com/docker/libnetwork/driverapi"
+	"github.com/docker/libnetwork/ipamapi"
 	networktypes "github.com/docker/libnetwork/types"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
@@ -298,6 +301,10 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
 		return nil, err
 	}
 
+	daemon.pluginRefCount(driver, driverapi.NetworkPluginEndpointType, plugingetter.ACQUIRE)
+	if create.IPAM != nil {
+		daemon.pluginRefCount(create.IPAM.Driver, ipamapi.PluginEndpointType, plugingetter.ACQUIRE)
+	}
 	daemon.LogNetworkEvent(n, "create")
 
 	return &types.NetworkCreateResponse{
@@ -306,6 +313,29 @@ func (daemon *Daemon) createNetwork(create types.NetworkCreateRequest, id string
 	}, nil
 }
 
+func (daemon *Daemon) pluginRefCount(driver, capability string, mode int) {
+	var builtinDrivers []string
+
+	if capability == driverapi.NetworkPluginEndpointType {
+		builtinDrivers = daemon.netController.BuiltinDrivers()
+	} else if capability == ipamapi.PluginEndpointType {
+		builtinDrivers = daemon.netController.BuiltinIPAMDrivers()
+	}
+
+	for _, d := range builtinDrivers {
+		if d == driver {
+			return
+		}
+	}
+
+	if daemon.PluginStore != nil {
+		_, err := daemon.PluginStore.Get(driver, capability, mode)
+		if err != nil {
+			logrus.WithError(err).WithFields(logrus.Fields{"mode": mode, "driver": driver}).Error("Error handling plugin refcount operation")
+		}
+	}
+}
+
 func getIpamConfig(data []network.IPAMConfig) ([]*libnetwork.IpamConf, []*libnetwork.IpamConf, error) {
 	ipamV4Cfg := []*libnetwork.IpamConf{}
 	ipamV6Cfg := []*libnetwork.IpamConf{}
@@ -420,6 +450,9 @@ func (daemon *Daemon) deleteNetwork(networkID string, dynamic bool) error {
 	if err := nw.Delete(); err != nil {
 		return err
 	}
+	daemon.pluginRefCount(nw.Type(), driverapi.NetworkPluginEndpointType, plugingetter.RELEASE)
+	ipamType, _, _, _ := nw.Info().IpamConfig()
+	daemon.pluginRefCount(ipamType, ipamapi.PluginEndpointType, plugingetter.RELEASE)
 	daemon.LogNetworkEvent(nw, "destroy")
 	return nil
 }

integration-cli/docker_cli_plugins_test.go

@@ -16,8 +16,10 @@ import (
 var (
 	pluginProcessName = "sample-volume-plugin"
 	pName             = "tonistiigi/sample-volume-plugin"
+	npName            = "tonistiigi/test-docker-netplugin"
 	pTag              = "latest"
 	pNameWithTag      = pName + ":" + pTag
+	npNameWithTag     = npName + ":" + pTag
 )
 
 func (s *DockerSuite) TestPluginBasicOps(c *check.C) {
@@ -87,6 +89,33 @@ func (s *DockerSuite) TestPluginActive(c *check.C) {
 	c.Assert(out, checker.Contains, pNameWithTag)
 }
 
+func (s *DockerSuite) TestPluginActiveNetwork(c *check.C) {
+	testRequires(c, DaemonIsLinux, IsAmd64, Network)
+	out, _, err := dockerCmdWithError("plugin", "install", "--grant-all-permissions", npNameWithTag)
+	c.Assert(err, checker.IsNil)
+
+	out, _, err = dockerCmdWithError("network", "create", "-d", npNameWithTag, "test")
+	c.Assert(err, checker.IsNil)
+
+	nID := strings.TrimSpace(out)
+
+	out, _, err = dockerCmdWithError("plugin", "remove", npNameWithTag)
+	c.Assert(out, checker.Contains, "is in use")
+
+	_, _, err = dockerCmdWithError("network", "rm", nID)
+	c.Assert(err, checker.IsNil)
+
+	out, _, err = dockerCmdWithError("plugin", "remove", npNameWithTag)
+	c.Assert(out, checker.Contains, "is enabled")
+
+	_, _, err = dockerCmdWithError("plugin", "disable", npNameWithTag)
+	c.Assert(err, checker.IsNil)
+
+	out, _, err = dockerCmdWithError("plugin", "remove", npNameWithTag)
+	c.Assert(err, checker.IsNil)
+	c.Assert(out, checker.Contains, npNameWithTag)
+}
+
 func (s *DockerSuite) TestPluginInstallDisable(c *check.C) {
 	testRequires(c, DaemonIsLinux, IsAmd64, Network)
 	out, _, err := dockerCmdWithError("plugin", "install", "--grant-all-permissions", "--disable", pName)

integration-cli/docker_cli_swarm_test.go

@@ -499,13 +499,6 @@ func (s *DockerSwarmSuite) TestPsListContainersFilterIsTask(c *check.C) {
 const globalNetworkPlugin = "global-network-plugin"
 const globalIPAMPlugin = "global-ipam-plugin"
 
-func (s *DockerSwarmSuite) SetUpSuite(c *check.C) {
-	mux := http.NewServeMux()
-	s.server = httptest.NewServer(mux)
-	c.Assert(s.server, check.NotNil, check.Commentf("Failed to start an HTTP Server"))
-	setupRemoteGlobalNetworkPlugin(c, mux, s.server.URL, globalNetworkPlugin, globalIPAMPlugin)
-}
-
 func setupRemoteGlobalNetworkPlugin(c *check.C, mux *http.ServeMux, url, netDrv, ipamDrv string) {
 
 	mux.HandleFunc("/Plugin.Activate", func(w http.ResponseWriter, r *http.Request) {
@@ -675,6 +668,16 @@ func setupRemoteGlobalNetworkPlugin(c *check.C, mux *http.ServeMux, url, netDrv,
 }
 
 func (s *DockerSwarmSuite) TestSwarmNetworkPlugin(c *check.C) {
+	mux := http.NewServeMux()
+	s.server = httptest.NewServer(mux)
+	c.Assert(s.server, check.NotNil, check.Commentf("Failed to start an HTTP Server"))
+	setupRemoteGlobalNetworkPlugin(c, mux, s.server.URL, globalNetworkPlugin, globalIPAMPlugin)
+	defer func() {
+		s.server.Close()
+		err := os.RemoveAll("/etc/docker/plugins")
+		c.Assert(err, checker.IsNil)
+	}()
+
 	d := s.AddDaemon(c, true, true)
 
 	out, err := d.Cmd("network", "create", "-d", globalNetworkPlugin, "foo")

vendor.conf

@@ -23,7 +23,7 @@ github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5
 github.com/imdario/mergo 0.2.1
 
 #get libnetwork packages
-github.com/docker/libnetwork b908488a139e81cb8c4091cd836745aeb4d813a4
+github.com/docker/libnetwork 61f01cdbbda7e32e651198b04889f6d825064fa6
 github.com/docker/go-events 18b43f1bc85d9cdd42c05a6cd2d444c7a200a894
 github.com/armon/go-radix e39d623f12e8e41c7b5529e9a9dd67a1e2261f80
 github.com/armon/go-metrics eb0af217e5e9747e41dd5303755356b62d28e3ec

vendor/github.com/docker/libnetwork/controller.go

@@ -79,6 +79,9 @@ type NetworkController interface {
 	// BuiltinDrivers returns list of builtin drivers
 	BuiltinDrivers() []string
 
+	// BuiltinIPAMDrivers returns list of builtin ipam drivers
+	BuiltinIPAMDrivers() []string
+
 	// Config method returns the bootup configuration for the controller
 	Config() config.Config
 
@@ -476,12 +479,23 @@ func (c *controller) ID() string {
 
 func (c *controller) BuiltinDrivers() []string {
 	drivers := []string{}
-	for _, i := range getInitializers(c.cfg.Daemon.Experimental) {
+	c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool {
-		if i.ntype == "remote" {
+		if driver.IsBuiltIn() {
-			continue
+			drivers = append(drivers, name)
 		}
-		drivers = append(drivers, i.ntype)
+		return false
-	}
+	})
+	return drivers
+}
+
+func (c *controller) BuiltinIPAMDrivers() []string {
+	drivers := []string{}
+	c.drvRegistry.WalkIPAMs(func(name string, driver ipamapi.Ipam, cap *ipamapi.Capability) bool {
+		if driver.IsBuiltIn() {
+			drivers = append(drivers, name)
+		}
+		return false
+	})
 	return drivers
 }
 

vendor/github.com/docker/libnetwork/driverapi/driverapi.go

@@ -74,6 +74,9 @@ type Driver interface {
 
 	// Type returns the the type of this driver, the network type this driver manages
 	Type() string
+
+	// IsBuiltIn returns true if it is a built-in driver
+	IsBuiltIn() bool
 }
 
 // NetworkInfo provides a go interface for drivers to provide network

vendor/github.com/docker/libnetwork/drivers/bridge/bridge.go

@@ -1424,6 +1424,10 @@ func (d *driver) Type() string {
 	return networkType
 }
 
+func (d *driver) IsBuiltIn() bool {
+	return true
+}
+
 // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
 func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
 	return nil

vendor/github.com/docker/libnetwork/drivers/host/host.go

@@ -86,6 +86,10 @@ func (d *driver) Type() string {
 	return networkType
 }
 
+func (d *driver) IsBuiltIn() bool {
+	return true
+}
+
 // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
 func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
 	return nil

vendor/github.com/docker/libnetwork/drivers/ipvlan/ipvlan.go

@@ -84,6 +84,10 @@ func (d *driver) Type() string {
 	return ipvlanType
 }
 
+func (d *driver) IsBuiltIn() bool {
+	return true
+}
+
 func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error {
 	return nil
 }

vendor/github.com/docker/libnetwork/drivers/macvlan/macvlan.go

@@ -86,6 +86,10 @@ func (d *driver) Type() string {
 	return macvlanType
 }
 
+func (d *driver) IsBuiltIn() bool {
+	return true
+}
+
 func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error {
 	return nil
 }

vendor/github.com/docker/libnetwork/drivers/null/null.go

@@ -86,6 +86,10 @@ func (d *driver) Type() string {
 	return networkType
 }
 
+func (d *driver) IsBuiltIn() bool {
+	return true
+}
+
 // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
 func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
 	return nil

vendor/github.com/docker/libnetwork/drivers/overlay/overlay.go

@@ -211,6 +211,10 @@ func (d *driver) Type() string {
 	return networkType
 }
 
+func (d *driver) IsBuiltIn() bool {
+	return true
+}
+
 func validateSelf(node string) error {
 	advIP := net.ParseIP(node)
 	if advIP == nil {

vendor/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go

@@ -229,6 +229,10 @@ func (d *driver) Type() string {
 	return networkType
 }
 
+func (d *driver) IsBuiltIn() bool {
+	return true
+}
+
 // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
 func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
 	return types.NotImplementedErrorf("not implemented")

vendor/github.com/docker/libnetwork/drivers/remote/driver.go

@@ -29,12 +29,7 @@ func newDriver(name string, client *plugins.Client) driverapi.Driver {
 // Init makes sure a remote driver is registered when a network driver
 // plugin is activated.
 func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
-	// Unit test code is unaware of a true PluginStore. So we fall back to v1 plugins.
+	newPluginHandler := func(name string, client *plugins.Client) {
-	handleFunc := plugins.Handle
-	if pg := dc.GetPluginGetter(); pg != nil {
-		handleFunc = pg.Handle
-	}
-	handleFunc(driverapi.NetworkPluginEndpointType, func(name string, client *plugins.Client) {
 		// negotiate driver capability with client
 		d := newDriver(name, client)
 		c, err := d.(*driver).getCapabilities()
@@ -45,7 +40,19 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 		if err = dc.RegisterDriver(name, d, *c); err != nil {
 			logrus.Errorf("error registering driver for %s due to %v", name, err)
 		}
-	})
+	}
+
+	// Unit test code is unaware of a true PluginStore. So we fall back to v1 plugins.
+	handleFunc := plugins.Handle
+	if pg := dc.GetPluginGetter(); pg != nil {
+		handleFunc = pg.Handle
+		activePlugins := pg.GetAllManagedPluginsByCap(driverapi.NetworkPluginEndpointType)
+		for _, ap := range activePlugins {
+			newPluginHandler(ap.Name(), ap.Client())
+		}
+	}
+	handleFunc(driverapi.NetworkPluginEndpointType, newPluginHandler)
+
 	return nil
 }
 
@@ -305,6 +312,10 @@ func (d *driver) Type() string {
 	return d.networkType
 }
 
+func (d *driver) IsBuiltIn() bool {
+	return false
+}
+
 // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
 func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
 	if dType != discoverapi.NodeDiscovery {

vendor/github.com/docker/libnetwork/drivers/solaris/bridge/bridge.go

@@ -916,6 +916,10 @@ func (d *driver) Type() string {
 	return networkType
 }
 
+func (d *driver) IsBuiltIn() bool {
+	return true
+}
+
 // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
 func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
 	return nil

vendor/github.com/docker/libnetwork/drivers/solaris/overlay/overlay.go

@@ -200,6 +200,10 @@ func (d *driver) Type() string {
 	return networkType
 }
 
+func (d *driver) IsBuiltIn() bool {
+	return true
+}
+
 func validateSelf(node string) error {
 	advIP := net.ParseIP(node)
 	if advIP == nil {

vendor/github.com/docker/libnetwork/drivers/windows/overlay/overlay_windows.go

@@ -176,6 +176,10 @@ func (d *driver) Type() string {
 	return networkType
 }
 
+func (d *driver) IsBuiltIn() bool {
+	return true
+}
+
 func validateSelf(node string) error {
 	advIP := net.ParseIP(node)
 	if advIP == nil {

vendor/github.com/docker/libnetwork/drivers/windows/windows.go

@@ -698,6 +698,10 @@ func (d *driver) Type() string {
 	return d.name
 }
 
+func (d *driver) IsBuiltIn() bool {
+	return true
+}
+
 // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
 func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
 	return nil

vendor/github.com/docker/libnetwork/drvregistry/drvregistry.go

@@ -164,10 +164,10 @@ func (r *DrvRegistry) RegisterDriver(ntype string, driver driverapi.Driver, capa
 	}
 
 	r.Lock()
-	_, ok := r.drivers[ntype]
+	dd, ok := r.drivers[ntype]
 	r.Unlock()
 
-	if ok {
+	if ok && dd.driver.IsBuiltIn() {
 		return driverapi.ErrActiveRegistration(ntype)
 	}
 
@@ -192,9 +192,9 @@ func (r *DrvRegistry) registerIpamDriver(name string, driver ipamapi.Ipam, caps
 	}
 
 	r.Lock()
-	_, ok := r.ipamDrivers[name]
+	dd, ok := r.ipamDrivers[name]
 	r.Unlock()
-	if ok {
+	if ok && dd.driver.IsBuiltIn() {
 		return types.ForbiddenErrorf("ipam driver %q already registered", name)
 	}
 

vendor/github.com/docker/libnetwork/ipam/allocator.go

@@ -594,3 +594,8 @@ func (a *Allocator) DumpDatabase() string {
 
 	return s
 }
+
+// IsBuiltIn returns true for builtin drivers
+func (a *Allocator) IsBuiltIn() bool {
+	return true
+}

vendor/github.com/docker/libnetwork/ipamapi/contract.go

@@ -80,6 +80,9 @@ type Ipam interface {
 	RequestAddress(string, net.IP, map[string]string) (*net.IPNet, map[string]string, error)
 	// Release the address from the specified pool ID
 	ReleaseAddress(string, net.IP) error
+
+	//IsBuiltIn returns true if it is a built-in driver.
+	IsBuiltIn() bool
 }
 
 // Capability represents the requirements and capabilities of the IPAM driver

vendor/github.com/docker/libnetwork/ipams/null/null.go

@@ -65,6 +65,10 @@ func (a *allocator) DiscoverDelete(dType discoverapi.DiscoveryType, data interfa
 	return nil
 }
 
+func (a *allocator) IsBuiltIn() bool {
+	return true
+}
+
 // Init registers a remote ipam when its plugin is activated
 func Init(ic ipamapi.Callback, l, g interface{}) error {
 	return ic.RegisterIpamDriver(ipamapi.NullIPAM, &allocator{})

vendor/github.com/docker/libnetwork/ipams/remote/remote.go

@@ -31,12 +31,7 @@ func newAllocator(name string, client *plugins.Client) ipamapi.Ipam {
 // Init registers a remote ipam when its plugin is activated
 func Init(cb ipamapi.Callback, l, g interface{}) error {
 
-	// Unit test code is unaware of a true PluginStore. So we fall back to v1 plugins.
+	newPluginHandler := func(name string, client *plugins.Client) {
-	handleFunc := plugins.Handle
-	if pg := cb.GetPluginGetter(); pg != nil {
-		handleFunc = pg.Handle
-	}
-	handleFunc(ipamapi.PluginEndpointType, func(name string, client *plugins.Client) {
 		a := newAllocator(name, client)
 		if cps, err := a.(*allocator).getCapabilities(); err == nil {
 			if err := cb.RegisterIpamDriverWithCapabilities(name, a, cps); err != nil {
@@ -49,7 +44,18 @@ func Init(cb ipamapi.Callback, l, g interface{}) error {
 				logrus.Errorf("error registering remote ipam driver %s due to %v", name, err)
 			}
 		}
-	})
+	}
+
+	// Unit test code is unaware of a true PluginStore. So we fall back to v1 plugins.
+	handleFunc := plugins.Handle
+	if pg := cb.GetPluginGetter(); pg != nil {
+		handleFunc = pg.Handle
+		activePlugins := pg.GetAllManagedPluginsByCap(ipamapi.PluginEndpointType)
+		for _, ap := range activePlugins {
+			newPluginHandler(ap.Name(), ap.Client())
+		}
+	}
+	handleFunc(ipamapi.PluginEndpointType, newPluginHandler)
 	return nil
 }
 
@@ -143,3 +149,7 @@ func (a *allocator) DiscoverNew(dType discoverapi.DiscoveryType, data interface{
 func (a *allocator) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error {
 	return nil
 }
+
+func (a *allocator) IsBuiltIn() bool {
+	return false
+}

vendor/github.com/docker/libnetwork/ipams/windowsipam/windowsipam.go

@@ -101,3 +101,7 @@ func (a *allocator) DiscoverNew(dType discoverapi.DiscoveryType, data interface{
 func (a *allocator) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error {
 	return nil
 }
+
+func (a *allocator) IsBuiltIn() bool {
+	return true
+}