From 73210671764fc3de133a627205582e069e1ff43d Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 8 Aug 2014 13:18:18 -0700 Subject: [PATCH 1/4] Use argv0 as reexec implementation for dockerinit This changes the way the exec drivers work by not specifing a -driver flag on reexec. For each of the exec drivers they register their own functions that will be matched aginst the argv 0 on exec and called if they match. This also allows any functionality to be added to docker so that the binary can be reexec'd and any type of function can be called. I moved the flag parsing on docker exec to the specific initializers so that the implementations do not bleed into one another. This also allows for more flexability within reexec initializers to specify their own flags and options. Signed-off-by: Michael Crosby --- daemon/execdriver/driver.go | 44 +--------- daemon/execdriver/lxc/driver.go | 33 +------ daemon/execdriver/lxc/init.go | 109 ++++++++++++++++++++++-- daemon/execdriver/lxc/lxc_init_linux.go | 2 +- daemon/execdriver/native/driver.go | 39 +-------- daemon/execdriver/native/init.go | 65 ++++++++++++++ docker/client.go | 4 - docker/daemon.go | 8 +- docker/docker.go | 4 +- dockerinit/dockerinit.go | 7 +- reexec/README.md | 5 ++ reexec/reexec.go | 23 +++++ sysinit/sysinit.go | 50 ----------- 13 files changed, 208 insertions(+), 185 deletions(-) create mode 100644 daemon/execdriver/native/init.go create mode 100644 reexec/README.md create mode 100644 reexec/reexec.go diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index d52a2ac96c..121c6a5a03 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -20,49 +20,7 @@ var ( ErrDriverNotFound = errors.New("The requested docker init has not been found") ) -var dockerInitFcts map[string]InitFunc - -type ( - StartCallback func(*Command) - InitFunc func(i *InitArgs) error -) - -func RegisterInitFunc(name string, fct InitFunc) error { - if dockerInitFcts == nil { - dockerInitFcts = make(map[string]InitFunc) - } - if _, ok := dockerInitFcts[name]; ok { - return ErrDriverAlreadyRegistered - } - dockerInitFcts[name] = fct - return nil -} - -func GetInitFunc(name string) (InitFunc, error) { - fct, ok := dockerInitFcts[name] - if !ok { - return nil, ErrDriverNotFound - } - return fct, nil -} - -// Args provided to the init function for a driver -type InitArgs struct { - User string - Gateway string - Ip string - WorkDir string - Privileged bool - Env []string - Args []string - Mtu int - Driver string - Console string - Pipe int - Root string - CapAdd string - CapDrop string -} +type StartCallback func(*Command) // Driver specific information based on // processes registered with the driver diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index df4b50e3ad..93dd1509ce 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -5,12 +5,10 @@ import ( "fmt" "io" "io/ioutil" - "log" "os" "os/exec" "path" "path/filepath" - "runtime" "strconv" "strings" "syscall" @@ -27,34 +25,6 @@ import ( const DriverName = "lxc" -func init() { - execdriver.RegisterInitFunc(DriverName, func(args *execdriver.InitArgs) error { - runtime.LockOSThread() - if err := setupEnv(args); err != nil { - return err - } - if err := setupHostname(args); err != nil { - return err - } - if err := setupNetworking(args); err != nil { - return err - } - if err := finalizeNamespace(args); err != nil { - return err - } - - path, err := exec.LookPath(args.Args[0]) - if err != nil { - log.Printf("Unable to locate %v", args.Args[0]) - os.Exit(127) - } - if err := syscall.Exec(path, args.Args, os.Environ()); err != nil { - return fmt.Errorf("dockerinit unable to execute %s - %s", path, err) - } - panic("Unreachable") - }) -} - type driver struct { root string // root path for the driver to use initPath string @@ -67,6 +37,7 @@ func NewDriver(root, initPath string, apparmor bool) (*driver, error) { if err := linkLxcStart(root); err != nil { return nil, err } + return &driver{ apparmor: apparmor, root: root, @@ -108,8 +79,6 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba "-f", configPath, "--", c.InitPath, - "-driver", - DriverName, } if c.Network.Interface != nil { diff --git a/daemon/execdriver/lxc/init.go b/daemon/execdriver/lxc/init.go index 246c56f0c1..2a91bbb5f5 100644 --- a/daemon/execdriver/lxc/init.go +++ b/daemon/execdriver/lxc/init.go @@ -2,19 +2,116 @@ package lxc import ( "encoding/json" + "flag" "fmt" "io/ioutil" + "log" "net" "os" + "os/exec" + "runtime" "strings" "syscall" - "github.com/docker/docker/daemon/execdriver" + "github.com/docker/docker/reexec" "github.com/docker/libcontainer/netlink" ) +// Args provided to the init function for a driver +type InitArgs struct { + User string + Gateway string + Ip string + WorkDir string + Privileged bool + Env []string + Args []string + Mtu int + Console string + Pipe int + Root string + CapAdd string + CapDrop string +} + +func init() { + // like always lxc requires a hack to get this to work + reexec.Register("/.dockerinit", dockerInititalizer) +} + +func dockerInititalizer() { + initializer() +} + +// initializer is the lxc driver's init function that is run inside the namespace to setup +// additional configurations +func initializer() { + runtime.LockOSThread() + + args := getArgs() + + if err := setupNamespace(args); err != nil { + log.Fatal(err) + } +} + +func setupNamespace(args *InitArgs) error { + if err := setupEnv(args); err != nil { + return err + } + if err := setupHostname(args); err != nil { + return err + } + if err := setupNetworking(args); err != nil { + return err + } + if err := finalizeNamespace(args); err != nil { + return err + } + + path, err := exec.LookPath(args.Args[0]) + if err != nil { + log.Printf("Unable to locate %v", args.Args[0]) + os.Exit(127) + } + + if err := syscall.Exec(path, args.Args, os.Environ()); err != nil { + return fmt.Errorf("dockerinit unable to execute %s - %s", path, err) + } + + return nil +} + +func getArgs() *InitArgs { + var ( + // Get cmdline arguments + user = flag.String("u", "", "username or uid") + gateway = flag.String("g", "", "gateway address") + ip = flag.String("i", "", "ip address") + workDir = flag.String("w", "", "workdir") + privileged = flag.Bool("privileged", false, "privileged mode") + mtu = flag.Int("mtu", 1500, "interface mtu") + capAdd = flag.String("cap-add", "", "capabilities to add") + capDrop = flag.String("cap-drop", "", "capabilities to drop") + ) + + flag.Parse() + + return &InitArgs{ + User: *user, + Gateway: *gateway, + Ip: *ip, + WorkDir: *workDir, + Privileged: *privileged, + Args: flag.Args(), + Mtu: *mtu, + CapAdd: *capAdd, + CapDrop: *capDrop, + } +} + // Clear environment pollution introduced by lxc-start -func setupEnv(args *execdriver.InitArgs) error { +func setupEnv(args *InitArgs) error { // Get env var env []string content, err := ioutil.ReadFile(".dockerenv") @@ -41,7 +138,7 @@ func setupEnv(args *execdriver.InitArgs) error { return nil } -func setupHostname(args *execdriver.InitArgs) error { +func setupHostname(args *InitArgs) error { hostname := getEnv(args, "HOSTNAME") if hostname == "" { return nil @@ -50,7 +147,7 @@ func setupHostname(args *execdriver.InitArgs) error { } // Setup networking -func setupNetworking(args *execdriver.InitArgs) error { +func setupNetworking(args *InitArgs) error { if args.Ip != "" { // eth0 iface, err := net.InterfaceByName("eth0") @@ -95,7 +192,7 @@ func setupNetworking(args *execdriver.InitArgs) error { } // Setup working directory -func setupWorkingDirectory(args *execdriver.InitArgs) error { +func setupWorkingDirectory(args *InitArgs) error { if args.WorkDir == "" { return nil } @@ -105,7 +202,7 @@ func setupWorkingDirectory(args *execdriver.InitArgs) error { return nil } -func getEnv(args *execdriver.InitArgs, key string) string { +func getEnv(args *InitArgs, key string) string { for _, kv := range args.Env { parts := strings.SplitN(kv, "=", 2) if parts[0] == key && len(parts) == 2 { diff --git a/daemon/execdriver/lxc/lxc_init_linux.go b/daemon/execdriver/lxc/lxc_init_linux.go index 9f294f6458..625caa1608 100644 --- a/daemon/execdriver/lxc/lxc_init_linux.go +++ b/daemon/execdriver/lxc/lxc_init_linux.go @@ -17,7 +17,7 @@ func setHostname(hostname string) error { return syscall.Sethostname([]byte(hostname)) } -func finalizeNamespace(args *execdriver.InitArgs) error { +func finalizeNamespace(args *InitArgs) error { if err := utils.CloseExecFrom(3); err != nil { return err } diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 61edd54d0f..c45188b6bc 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -22,7 +22,6 @@ import ( "github.com/docker/libcontainer/cgroups/systemd" consolepkg "github.com/docker/libcontainer/console" "github.com/docker/libcontainer/namespaces" - "github.com/docker/libcontainer/syncpipe" "github.com/docker/libcontainer/system" ) @@ -31,38 +30,6 @@ const ( Version = "0.2" ) -func init() { - execdriver.RegisterInitFunc(DriverName, func(args *execdriver.InitArgs) error { - var container *libcontainer.Config - f, err := os.Open(filepath.Join(args.Root, "container.json")) - if err != nil { - return err - } - - if err := json.NewDecoder(f).Decode(&container); err != nil { - f.Close() - return err - } - f.Close() - - rootfs, err := os.Getwd() - if err != nil { - return err - } - - syncPipe, err := syncpipe.NewSyncPipeFromFd(0, uintptr(args.Pipe)) - if err != nil { - return err - } - - if err := namespaces.Init(container, rootfs, args.Console, syncPipe, args.Args); err != nil { - return err - } - - return nil - }) -} - type activeContainer struct { container *libcontainer.Config cmd *exec.Cmd @@ -133,13 +100,9 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba } return namespaces.Exec(container, c.Stdin, c.Stdout, c.Stderr, c.Console, c.Rootfs, dataPath, args, func(container *libcontainer.Config, console, rootfs, dataPath, init string, child *os.File, args []string) *exec.Cmd { - // we need to join the rootfs because namespaces will setup the rootfs and chroot - initPath := filepath.Join(c.Rootfs, c.InitPath) - c.Path = d.initPath c.Args = append([]string{ - initPath, - "-driver", DriverName, + DriverName, "-console", console, "-pipe", "3", "-root", filepath.Join(d.root, c.ID), diff --git a/daemon/execdriver/native/init.go b/daemon/execdriver/native/init.go new file mode 100644 index 0000000000..5661454162 --- /dev/null +++ b/daemon/execdriver/native/init.go @@ -0,0 +1,65 @@ +// +build linux + +package native + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/docker/docker/reexec" + "github.com/docker/libcontainer" + "github.com/docker/libcontainer/namespaces" + "github.com/docker/libcontainer/syncpipe" +) + +func init() { + reexec.Register(DriverName, initializer) +} + +func initializer() { + runtime.LockOSThread() + + var ( + pipe = flag.Int("pipe", 0, "sync pipe fd") + console = flag.String("console", "", "console (pty slave) path") + root = flag.String("root", ".", "root path for configuration files") + ) + + flag.Parse() + + var container *libcontainer.Config + f, err := os.Open(filepath.Join(*root, "container.json")) + if err != nil { + writeError(err) + } + + if err := json.NewDecoder(f).Decode(&container); err != nil { + f.Close() + writeError(err) + } + f.Close() + + rootfs, err := os.Getwd() + if err != nil { + writeError(err) + } + + syncPipe, err := syncpipe.NewSyncPipeFromFd(0, uintptr(*pipe)) + if err != nil { + writeError(err) + } + + if err := namespaces.Init(container, rootfs, *console, syncPipe, flag.Args()); err != nil { + writeError(err) + } + + panic("Unreachable") +} + +func writeError(err error) { + fmt.Sprint(os.Stderr, err) +} diff --git a/docker/client.go b/docker/client.go index 4565d6d6a1..27001cc557 100644 --- a/docker/client.go +++ b/docker/client.go @@ -8,10 +8,6 @@ import ( const CanDaemon = false -func mainSysinit() { - log.Fatal("This is a client-only binary - running it as 'dockerinit' is not supported.") -} - func mainDaemon() { log.Fatal("This is a client-only binary - running the Docker daemon is not supported.") } diff --git a/docker/daemon.go b/docker/daemon.go index 15d9c4856f..e886f99a1b 100644 --- a/docker/daemon.go +++ b/docker/daemon.go @@ -7,20 +7,16 @@ import ( "net" "github.com/docker/docker/builtins" + _ "github.com/docker/docker/daemon/execdriver/lxc" + _ "github.com/docker/docker/daemon/execdriver/native" "github.com/docker/docker/dockerversion" "github.com/docker/docker/engine" flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/signal" - "github.com/docker/docker/sysinit" ) const CanDaemon = true -func mainSysinit() { - // Running in init mode - sysinit.SysInit() -} - func mainDaemon() { if flag.NArg() != 0 { flag.Usage() diff --git a/docker/docker.go b/docker/docker.go index 38a4da0d44..b20db7c16b 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -13,6 +13,7 @@ import ( "github.com/docker/docker/api/client" "github.com/docker/docker/dockerversion" flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/reexec" "github.com/docker/docker/utils" ) @@ -23,8 +24,7 @@ const ( ) func main() { - if selfPath := utils.SelfPath(); strings.Contains(selfPath, ".dockerinit") { - mainSysinit() + if reexec.Init() { return } diff --git a/dockerinit/dockerinit.go b/dockerinit/dockerinit.go index 3507d3e5fe..c5bba782b0 100644 --- a/dockerinit/dockerinit.go +++ b/dockerinit/dockerinit.go @@ -1,11 +1,12 @@ package main import ( - "github.com/docker/docker/sysinit" + _ "github.com/docker/docker/daemon/execdriver/lxc" + _ "github.com/docker/docker/daemon/execdriver/native" + "github.com/docker/docker/reexec" ) func main() { // Running in init mode - sysinit.SysInit() - return + reexec.Init() } diff --git a/reexec/README.md b/reexec/README.md new file mode 100644 index 0000000000..45592ce85a --- /dev/null +++ b/reexec/README.md @@ -0,0 +1,5 @@ +## reexec + +The `reexec` package facilitates the busybox style reexec of the docker binary that we require because +of the forking limitations of using Go. Handlers can be registered with a name and the argv 0 of +the exec of the binary will be used to find and execute custom init paths. diff --git a/reexec/reexec.go b/reexec/reexec.go new file mode 100644 index 0000000000..b46b879882 --- /dev/null +++ b/reexec/reexec.go @@ -0,0 +1,23 @@ +package reexec + +import "os" + +var registeredInitializers = make(map[string]func()) + +// Register adds an initialization func under the specified name +func Register(name string, initializer func()) { + registeredInitializers[name] = initializer +} + +// Init is called as the first part of the exec process and returns true if an +// initialization function was called. +func Init() bool { + initializer, exists := registeredInitializers[os.Args[0]] + if exists { + initializer() + + return true + } + + return false +} diff --git a/sysinit/sysinit.go b/sysinit/sysinit.go index 65d7bf4e7c..ed8d11827f 100644 --- a/sysinit/sysinit.go +++ b/sysinit/sysinit.go @@ -1,25 +1,11 @@ package sysinit import ( - "flag" "fmt" - "log" "os" "runtime" - - "github.com/docker/docker/daemon/execdriver" - _ "github.com/docker/docker/daemon/execdriver/lxc" - _ "github.com/docker/docker/daemon/execdriver/native" ) -func executeProgram(args *execdriver.InitArgs) error { - dockerInitFct, err := execdriver.GetInitFunc(args.Driver) - if err != nil { - panic(err) - } - return dockerInitFct(args) -} - // Sys Init code // This code is run INSIDE the container and is responsible for setting // up the environment before running the actual process @@ -33,40 +19,4 @@ func SysInit() { os.Exit(1) } - var ( - // Get cmdline arguments - user = flag.String("u", "", "username or uid") - gateway = flag.String("g", "", "gateway address") - ip = flag.String("i", "", "ip address") - workDir = flag.String("w", "", "workdir") - privileged = flag.Bool("privileged", false, "privileged mode") - mtu = flag.Int("mtu", 1500, "interface mtu") - driver = flag.String("driver", "", "exec driver") - pipe = flag.Int("pipe", 0, "sync pipe fd") - console = flag.String("console", "", "console (pty slave) path") - root = flag.String("root", ".", "root path for configuration files") - capAdd = flag.String("cap-add", "", "capabilities to add") - capDrop = flag.String("cap-drop", "", "capabilities to drop") - ) - flag.Parse() - - args := &execdriver.InitArgs{ - User: *user, - Gateway: *gateway, - Ip: *ip, - WorkDir: *workDir, - Privileged: *privileged, - Args: flag.Args(), - Mtu: *mtu, - Driver: *driver, - Console: *console, - Pipe: *pipe, - Root: *root, - CapAdd: *capAdd, - CapDrop: *capDrop, - } - - if err := executeProgram(args); err != nil { - log.Fatal(err) - } } From 29af9c14e442f5cf3760f439d8c0fffaf84e2268 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 8 Aug 2014 16:21:53 -0700 Subject: [PATCH 2/4] Remove unsued sysinit package Signed-off-by: Michael Crosby --- sysinit/README.md | 4 ---- sysinit/sysinit.go | 22 ---------------------- 2 files changed, 26 deletions(-) delete mode 100644 sysinit/README.md delete mode 100644 sysinit/sysinit.go diff --git a/sysinit/README.md b/sysinit/README.md deleted file mode 100644 index c28d0298b8..0000000000 --- a/sysinit/README.md +++ /dev/null @@ -1,4 +0,0 @@ -Sys Init code - -This code is run INSIDE the container and is responsible for setting -up the environment before running the actual process diff --git a/sysinit/sysinit.go b/sysinit/sysinit.go deleted file mode 100644 index ed8d11827f..0000000000 --- a/sysinit/sysinit.go +++ /dev/null @@ -1,22 +0,0 @@ -package sysinit - -import ( - "fmt" - "os" - "runtime" -) - -// Sys Init code -// This code is run INSIDE the container and is responsible for setting -// up the environment before running the actual process -func SysInit() { - // The very first thing that we should do is lock the thread so that other - // system level options will work and not have issues, i.e. setns - runtime.LockOSThread() - - if len(os.Args) <= 1 { - fmt.Println("You should not invoke dockerinit manually") - os.Exit(1) - } - -} From 9694209241074c10d4cc625b24dfedcb2387de93 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 8 Aug 2014 16:27:13 -0700 Subject: [PATCH 3/4] Fix hack in old integration test for new init Signed-off-by: Michael Crosby --- integration/runtime_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration/runtime_test.go b/integration/runtime_test.go index 45a2a29cf5..5148af01ba 100644 --- a/integration/runtime_test.go +++ b/integration/runtime_test.go @@ -20,8 +20,8 @@ import ( "github.com/docker/docker/engine" "github.com/docker/docker/image" "github.com/docker/docker/nat" + "github.com/docker/docker/reexec" "github.com/docker/docker/runconfig" - "github.com/docker/docker/sysinit" "github.com/docker/docker/utils" ) @@ -94,8 +94,7 @@ func init() { os.Setenv("DOCKER_TMPDIR", unitTestDockerTmpdir) // Hack to run sys init during unit testing - if selfPath := utils.SelfPath(); strings.Contains(selfPath, ".dockerinit") { - sysinit.SysInit() + if reexec.Init() { return } From 1a249a5feb9880465594299c1d4965b160e4a14e Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 11 Aug 2014 18:13:27 -0700 Subject: [PATCH 4/4] Panic if trying to register an func with the same name Signed-off-by: Michael Crosby --- reexec/reexec.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/reexec/reexec.go b/reexec/reexec.go index b46b879882..6a199e1ff1 100644 --- a/reexec/reexec.go +++ b/reexec/reexec.go @@ -1,11 +1,18 @@ package reexec -import "os" +import ( + "fmt" + "os" +) var registeredInitializers = make(map[string]func()) // Register adds an initialization func under the specified name func Register(name string, initializer func()) { + if _, exists := registeredInitializers[name]; exists { + panic(fmt.Sprintf("reexec func already registred under name %q", name)) + } + registeredInitializers[name] = initializer }