diff --git a/libnetwork/controller.go b/libnetwork/controller.go index d84f5d4d52..eb99f4d6ca 100644 --- a/libnetwork/controller.go +++ b/libnetwork/controller.go @@ -65,6 +65,9 @@ import ( // NetworkController provides the interface for controller instance which manages // networks. type NetworkController interface { + // ID provides an unique identity for the controller + ID() string + // ConfigureNetworkDriver applies the passed options to the driver instance for the specified network type ConfigureNetworkDriver(networkType string, options map[string]interface{}) error @@ -99,8 +102,8 @@ type NetworkController interface { // SandboxByID returns the Sandbox which has the passed id. If not found, a types.NotFoundError is returned. SandboxByID(id string) (Sandbox, error) - // GC triggers immediate garbage collection of resources which are garbage collected. - GC() + // Stop network controller + Stop() } // NetworkWalker is a client provided function which will be used to walk the Networks. @@ -122,11 +125,13 @@ type endpointTable map[string]*endpoint type sandboxTable map[string]*sandbox type controller struct { - networks networkTable - drivers driverTable - sandboxes sandboxTable - cfg *config.Config - store datastore.DataStore + id string + networks networkTable + drivers driverTable + sandboxes sandboxTable + cfg *config.Config + store datastore.DataStore + extKeyListener net.Listener sync.Mutex } @@ -138,6 +143,7 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { cfg.ProcessOptions(cfgOptions...) } c := &controller{ + id: stringid.GenerateRandomID(), cfg: cfg, networks: networkTable{}, sandboxes: sandboxTable{}, @@ -159,9 +165,17 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { } } + if err := c.startExternalKeyListener(); err != nil { + return nil, err + } + return c, nil } +func (c *controller) ID() string { + return c.id +} + func (c *controller) validateHostDiscoveryConfig() bool { if c.cfg == nil || c.cfg.Cluster.Discovery == "" || c.cfg.Cluster.Address == "" { return false @@ -514,6 +528,7 @@ func (c *controller) isDriverGlobalScoped(networkType string) (bool, error) { return false, nil } -func (c *controller) GC() { +func (c *controller) Stop() { + c.stopExternalKeyListener() osl.GC() } diff --git a/libnetwork/libnetwork_test.go b/libnetwork/libnetwork_test.go index 8366761cca..04d75e7dad 100644 --- a/libnetwork/libnetwork_test.go +++ b/libnetwork/libnetwork_test.go @@ -2,6 +2,7 @@ package libnetwork_test import ( "bytes" + "encoding/json" "flag" "fmt" "io/ioutil" @@ -9,8 +10,10 @@ import ( "net/http" "net/http/httptest" "os" + "os/exec" "runtime" "strconv" + "strings" "sync" "testing" @@ -25,6 +28,8 @@ import ( "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/testutils" "github.com/docker/libnetwork/types" + "github.com/opencontainers/runc/libcontainer" + "github.com/opencontainers/runc/libcontainer/configs" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" ) @@ -1193,6 +1198,14 @@ func (f *fakeSandbox) SetKey(key string) error { } func TestExternalKey(t *testing.T) { + externalKeyTest(t, false) +} + +func TestExternalKeyWithReexec(t *testing.T) { + externalKeyTest(t, true) +} + +func externalKeyTest(t *testing.T, reexec bool) { if !testutils.IsRunningInContainer() { defer testutils.SetupTestOSContext(t)() } @@ -1264,9 +1277,16 @@ func TestExternalKey(t *testing.T) { t.Fatalf("Expected to have a valid Sandbox") } - // Setting an non-existing key (namespace) must fail - if err := sbox.SetKey("this-must-fail"); err == nil { - t.Fatalf("Setkey must fail if the corresponding namespace is not created") + if reexec { + err := reexecSetKey("this-must-fail", containerID, controller.ID()) + if err == nil { + t.Fatalf("SetExternalKey must fail if the corresponding namespace is not created") + } + } else { + // Setting an non-existing key (namespace) must fail + if err := sbox.SetKey("this-must-fail"); err == nil { + t.Fatalf("Setkey must fail if the corresponding namespace is not created") + } } // Create a new OS sandbox using the osl API before using it in SetKey @@ -1274,8 +1294,15 @@ func TestExternalKey(t *testing.T) { t.Fatalf("Failed to create new osl sandbox") } - if err := sbox.SetKey("ValidKey"); err != nil { - t.Fatalf("Setkey failed with %v", err) + if reexec { + err := reexecSetKey("ValidKey", containerID, controller.ID()) + if err != nil { + t.Fatalf("SetExternalKey failed with %v", err) + } + } else { + if err := sbox.SetKey("ValidKey"); err != nil { + t.Fatalf("Setkey failed with %v", err) + } } // Join endpoint to sandbox after SetKey @@ -1299,6 +1326,28 @@ func TestExternalKey(t *testing.T) { checkSandbox(t, ep.Info()) } +func reexecSetKey(key string, containerID string, controllerID string) error { + var ( + state libcontainer.State + b []byte + err error + ) + + state.NamespacePaths = make(map[configs.NamespaceType]string) + state.NamespacePaths[configs.NamespaceType("NEWNET")] = key + if b, err = json.Marshal(state); err != nil { + return err + } + cmd := &exec.Cmd{ + Path: reexec.Self(), + Args: append([]string{"libnetwork-setkey"}, containerID, controllerID), + Stdin: strings.NewReader(string(b)), + Stdout: os.Stdout, + Stderr: os.Stderr, + } + return cmd.Run() +} + func TestEndpointDeleteWithActiveContainer(t *testing.T) { if !testutils.IsRunningInContainer() { defer testutils.SetupTestOSContext(t)() diff --git a/libnetwork/sandbox_externalkey.go b/libnetwork/sandbox_externalkey.go new file mode 100644 index 0000000000..fe863495ab --- /dev/null +++ b/libnetwork/sandbox_externalkey.go @@ -0,0 +1,185 @@ +package libnetwork + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net" + "os" + + "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/reexec" + "github.com/docker/libnetwork/types" + "github.com/opencontainers/runc/libcontainer" + "github.com/opencontainers/runc/libcontainer/configs" +) + +type setKeyData struct { + ContainerID string + Key string +} + +func init() { + reexec.Register("libnetwork-setkey", processSetKeyReexec) +} + +const udsBase = "/var/lib/docker/network/files/" +const success = "success" + +// processSetKeyReexec is a private function that must be called only on an reexec path +// It expects 3 args { [0] = "libnetwork-setkey", [1] = , [2] = } +// It also expects libcontainer.State as a json string in +// Refer to https://github.com/opencontainers/runc/pull/160/ for more information +func processSetKeyReexec() { + var err error + + // Return a failure to the calling process via ExitCode + defer func() { + if err != nil { + logrus.Fatalf("%v", err) + } + }() + + // expecting 3 args {[0]="libnetwork-setkey", [1]=, [2]= } + if len(os.Args) < 3 { + err = fmt.Errorf("Re-exec expects 3 args, received : %d", len(os.Args)) + return + } + containerID := os.Args[1] + + // We expect libcontainer.State as a json string in + stateBuf, err := ioutil.ReadAll(os.Stdin) + if err != nil { + return + } + var state libcontainer.State + if err = json.Unmarshal(stateBuf, &state); err != nil { + return + } + + controllerID := os.Args[2] + key := state.NamespacePaths[configs.NamespaceType("NEWNET")] + + err = SetExternalKey(controllerID, containerID, key) + return +} + +// SetExternalKey provides a convenient way to set an External key to a sandbox +func SetExternalKey(controllerID string, containerID string, key string) error { + keyData := setKeyData{ + ContainerID: containerID, + Key: key} + + c, err := net.Dial("unix", udsBase+controllerID+".sock") + if err != nil { + return err + } + defer c.Close() + + if err = sendKey(c, keyData); err != nil { + return fmt.Errorf("sendKey failed with : %v", err) + } + return processReturn(c) +} + +func sendKey(c net.Conn, data setKeyData) error { + var err error + defer func() { + if err != nil { + c.Close() + } + }() + + var b []byte + if b, err = json.Marshal(data); err != nil { + return err + } + + _, err = c.Write(b) + return err +} + +func processReturn(r io.Reader) error { + buf := make([]byte, 1024) + n, err := r.Read(buf[:]) + if err != nil { + return fmt.Errorf("failed to read buf in processReturn : %v", err) + } + if string(buf[0:n]) != success { + return fmt.Errorf(string(buf[0:n])) + } + return nil +} + +func (c *controller) startExternalKeyListener() error { + if err := os.MkdirAll(udsBase, 0600); err != nil { + return err + } + uds := udsBase + c.id + ".sock" + l, err := net.Listen("unix", uds) + if err != nil { + return err + } + if err := os.Chmod(uds, 0600); err != nil { + l.Close() + return err + } + c.Lock() + c.extKeyListener = l + c.Unlock() + + go c.acceptClientConnections(uds, l) + return nil +} + +func (c *controller) acceptClientConnections(sock string, l net.Listener) { + for { + conn, err := l.Accept() + if err != nil { + if _, err1 := os.Stat(sock); os.IsNotExist(err1) { + logrus.Warnf("Unix socket %s doesnt exist. cannot accept client connections", sock) + return + } + logrus.Errorf("Error accepting connection %v", err) + continue + } + go func() { + err := c.processExternalKey(conn) + ret := success + if err != nil { + ret = err.Error() + } + + _, err = conn.Write([]byte(ret)) + if err != nil { + logrus.Errorf("Error returning to the client %v", err) + } + }() + } +} + +func (c *controller) processExternalKey(conn net.Conn) error { + buf := make([]byte, 1280) + nr, err := conn.Read(buf) + if err != nil { + return err + } + var s setKeyData + if err = json.Unmarshal(buf[0:nr], &s); err != nil { + return err + } + + var sandbox Sandbox + search := SandboxContainerWalker(&sandbox, s.ContainerID) + c.WalkSandboxes(search) + if sandbox == nil { + return types.BadRequestErrorf("no sandbox present for %s", s.ContainerID) + } + + return sandbox.SetKey(s.Key) +} + +func (c *controller) stopExternalKeyListener() { + c.extKeyListener.Close() +}