Merge pull request #15921 from srust/volume_driver_register_retry

Retry registering a volume driver

integration-cli/docker_cli_start_volume_driver_unix_test.go

@@ -310,3 +310,43 @@ func (s *DockerExternalVolumeSuite) TestStartExternalVolumeDriverLookupNotBlocke
 		cmd2.Process.Kill()
 	}
 }
+
+func (s *DockerExternalVolumeSuite) TestStartExternalVolumeDriverRetryNotImmediatelyExists(c *check.C) {
+	if err := s.d.StartWithBusybox(); err != nil {
+		c.Fatal(err)
+	}
+
+	specPath := "/etc/docker/plugins/test-external-volume-driver-retry.spec"
+	os.RemoveAll(specPath)
+	defer os.RemoveAll(specPath)
+
+	errchan := make(chan error)
+	go func() {
+		if out, err := s.d.Cmd("run", "--rm", "--name", "test-data-retry", "-v", "external-volume-test:/tmp/external-volume-test", "--volume-driver", "test-external-volume-driver-retry", "busybox:latest"); err != nil {
+			errchan <- fmt.Errorf("%v:\n%s", err, out)
+		}
+		close(errchan)
+	}()
+	go func() {
+		// wait for a retry to occur, then create spec to allow plugin to register
+		time.Sleep(2000 * time.Millisecond)
+		if err := ioutil.WriteFile(specPath, []byte(s.server.URL), 0644); err != nil {
+			c.Fatal(err)
+		}
+	}()
+
+	select {
+	case err := <-errchan:
+		if err != nil {
+			c.Fatal(err)
+		}
+	case <-time.After(8 * time.Second):
+		c.Fatal("volume creates fail when plugin not immediately available")
+	}
+
+	c.Assert(s.ec.activations, check.Equals, 1)
+	c.Assert(s.ec.creations, check.Equals, 1)
+	c.Assert(s.ec.removals, check.Equals, 1)
+	c.Assert(s.ec.mounts, check.Equals, 1)
+	c.Assert(s.ec.unmounts, check.Equals, 1)
+}

pkg/plugins/plugins.go

@@ -25,6 +25,7 @@ package plugins
 import (
 	"errors"
 	"sync"
+	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/pkg/tlsconfig"
@@ -109,27 +110,45 @@ func (p *Plugin) activateWithLock() error {
 }
 
 func load(name string) (*Plugin, error) {
-	storage.Lock()
+	return loadWithRetry(name, true)
+}
+
+func loadWithRetry(name string, retry bool) (*Plugin, error) {
 	registry := newLocalRegistry()
-	pl, err := registry.Plugin(name)
-	if err == nil {
-		storage.plugins[name] = pl
-	}
-	storage.Unlock()
+	start := time.Now()
 
-	if err != nil {
-		return nil, err
-	}
+	var retries int
+	for {
+		pl, err := registry.Plugin(name)
+		if err != nil {
+			if !retry {
+				return nil, err
+			}
 
-	err = pl.activate()
+			timeOff := backoff(retries)
+			if abort(start, timeOff) {
+				return nil, err
+			}
+			retries++
+			logrus.Warnf("Unable to locate plugin: %s, retrying in %v", name, timeOff)
+			time.Sleep(timeOff)
+			continue
+		}
 
-	if err != nil {
 		storage.Lock()
-		delete(storage.plugins, name)
+		storage.plugins[name] = pl
 		storage.Unlock()
-	}
 
-	return pl, err
+		err = pl.activate()
+
+		if err != nil {
+			storage.Lock()
+			delete(storage.plugins, name)
+			storage.Unlock()
+		}
+
+		return pl, err
+	}
 }
 
 func get(name string) (*Plugin, error) {