From fac41af25bd5f42269424a788783a4280dd7fc9c Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sat, 22 Feb 2014 01:21:26 -0800 Subject: [PATCH] Refactor driver to use Exec function from nsini Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- execdriver/lxc/term.go | 37 ++-- execdriver/namespaces/driver.go | 296 +++++++++---------------------- pkg/libcontainer/nsinit/state.go | 10 +- pkg/libcontainer/nsinit/term.go | 9 + 4 files changed, 113 insertions(+), 239 deletions(-) diff --git a/execdriver/lxc/term.go b/execdriver/lxc/term.go index d772f60972..db58c3181a 100644 --- a/execdriver/lxc/term.go +++ b/execdriver/lxc/term.go @@ -6,6 +6,7 @@ import ( "github.com/kr/pty" "io" "os" + "os/exec" ) func SetTerminal(command *execdriver.Command, pipes *execdriver.Pipes) error { @@ -26,8 +27,8 @@ func SetTerminal(command *execdriver.Command, pipes *execdriver.Pipes) error { } type TtyConsole struct { - master *os.File - slave *os.File + MasterPty *os.File + SlavePty *os.File } func NewTtyConsole(command *execdriver.Command, pipes *execdriver.Pipes) (*TtyConsole, error) { @@ -36,28 +37,28 @@ func NewTtyConsole(command *execdriver.Command, pipes *execdriver.Pipes) (*TtyCo return nil, err } tty := &TtyConsole{ - master: ptyMaster, - slave: ptySlave, + MasterPty: ptyMaster, + SlavePty: ptySlave, } - if err := tty.attach(command, pipes); err != nil { + if err := tty.AttachPipes(&command.Cmd, pipes); err != nil { tty.Close() return nil, err } + command.Console = tty.SlavePty.Name() return tty, nil } func (t *TtyConsole) Master() *os.File { - return t.master + return t.MasterPty } func (t *TtyConsole) Resize(h, w int) error { - return term.SetWinsize(t.master.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)}) + return term.SetWinsize(t.MasterPty.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)}) } -func (t *TtyConsole) attach(command *execdriver.Command, pipes *execdriver.Pipes) error { - command.Stdout = t.slave - command.Stderr = t.slave - command.Console = t.slave.Name() +func (t *TtyConsole) AttachPipes(command *exec.Cmd, pipes *execdriver.Pipes) error { + command.Stdout = t.SlavePty + command.Stderr = t.SlavePty go func() { if wb, ok := pipes.Stdout.(interface { @@ -65,24 +66,24 @@ func (t *TtyConsole) attach(command *execdriver.Command, pipes *execdriver.Pipes }); ok { defer wb.CloseWriters() } - io.Copy(pipes.Stdout, t.master) + io.Copy(pipes.Stdout, t.MasterPty) }() if pipes.Stdin != nil { - command.Stdin = t.slave + command.Stdin = t.SlavePty command.SysProcAttr.Setctty = true go func() { defer pipes.Stdin.Close() - io.Copy(t.master, pipes.Stdin) + io.Copy(t.MasterPty, pipes.Stdin) }() } return nil } func (t *TtyConsole) Close() error { - t.slave.Close() - return t.master.Close() + t.SlavePty.Close() + return t.MasterPty.Close() } type StdConsole struct { @@ -91,13 +92,13 @@ type StdConsole struct { func NewStdConsole(command *execdriver.Command, pipes *execdriver.Pipes) (*StdConsole, error) { std := &StdConsole{} - if err := std.attach(command, pipes); err != nil { + if err := std.AttachPipes(&command.Cmd, pipes); err != nil { return nil, err } return std, nil } -func (s *StdConsole) attach(command *execdriver.Command, pipes *execdriver.Pipes) error { +func (s *StdConsole) AttachPipes(command *exec.Cmd, pipes *execdriver.Pipes) error { command.Stdout = pipes.Stdout command.Stderr = pipes.Stderr diff --git a/execdriver/namespaces/driver.go b/execdriver/namespaces/driver.go index e243c64703..a3f095f464 100644 --- a/execdriver/namespaces/driver.go +++ b/execdriver/namespaces/driver.go @@ -5,15 +5,10 @@ import ( "errors" "fmt" "github.com/dotcloud/docker/execdriver" + "github.com/dotcloud/docker/execdriver/lxc" "github.com/dotcloud/docker/pkg/libcontainer" - "github.com/dotcloud/docker/pkg/libcontainer/network" "github.com/dotcloud/docker/pkg/libcontainer/nsinit" - "github.com/dotcloud/docker/pkg/libcontainer/utils" - "github.com/dotcloud/docker/pkg/system" - "github.com/dotcloud/docker/pkg/term" - "io" "io/ioutil" - "log" "os" "os/exec" "path/filepath" @@ -44,129 +39,31 @@ func NewDriver() (*driver, error) { } func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (int, error) { - container := createContainer(c) + var ( + term nsinit.Terminal + container = createContainer(c) + factory = &dockerCommandFactory{c} + stateWriter = &dockerStateWriter{ + callback: startCallback, + c: c, + dsw: &nsinit.DefaultStateWriter{c.Rootfs}, + } + ) + if c.Tty { + term = &dockerTtyTerm{ + pipes: pipes, + } + } else { + term = &dockerStdTerm{ + pipes: pipes, + } + } + c.Terminal = term if err := writeContainerFile(container, c.Rootfs); err != nil { return -1, err } - - var ( - console string - master *os.File - err error - - inPipe io.WriteCloser - outPipe, errPipe io.ReadCloser - ) - - if container.Tty { - log.Printf("setting up master and console") - master, console, err = createMasterAndConsole() - if err != nil { - return -1, err - } - } - c.Terminal = NewTerm(pipes, master) - - // create a pipe so that we can syncronize with the namespaced process and - // pass the veth name to the child - r, w, err := os.Pipe() - if err != nil { - return -1, err - } - system.UsetCloseOnExec(r.Fd()) - args := append([]string{c.Entrypoint}, c.Arguments...) - createCommand(c, container, console, "/nsinit.logs", r.Fd(), args) - command := c - - if !container.Tty { - log.Printf("opening pipes on command") - if inPipe, err = command.StdinPipe(); err != nil { - return -1, err - } - if outPipe, err = command.StdoutPipe(); err != nil { - return -1, err - } - if errPipe, err = command.StderrPipe(); err != nil { - return -1, err - } - } - - log.Printf("staring init") - if err := command.Start(); err != nil { - return -1, err - } - log.Printf("writting state file") - if err := writePidFile(c.Rootfs, command.Process.Pid); err != nil { - command.Process.Kill() - return -1, err - } - defer deletePidFile(c.Rootfs) - - // Do this before syncing with child so that no children - // can escape the cgroup - if container.Cgroups != nil { - log.Printf("setting up cgroups") - if err := container.Cgroups.Apply(command.Process.Pid); err != nil { - command.Process.Kill() - return -1, err - } - } - - if container.Network != nil { - log.Printf("creating veth pair") - vethPair, err := initializeContainerVeth(container.Network.Bridge, container.Network.Mtu, command.Process.Pid) - if err != nil { - return -1, err - } - log.Printf("sending %s as veth pair name", vethPair) - sendVethName(w, vethPair) - } - - // Sync with child - log.Printf("closing sync pipes") - w.Close() - r.Close() - - if container.Tty { - log.Printf("starting copy for tty") - go io.Copy(pipes.Stdout, master) - if pipes.Stdin != nil { - go io.Copy(master, pipes.Stdin) - } - - /* - state, err := setupWindow(master) - if err != nil { - command.Process.Kill() - return -1, err - } - defer term.RestoreTerminal(uintptr(syscall.Stdin), state) - */ - } else { - log.Printf("starting copy for std pipes") - if pipes.Stdin != nil { - go func() { - defer inPipe.Close() - io.Copy(inPipe, pipes.Stdin) - }() - } - go io.Copy(pipes.Stdout, outPipe) - go io.Copy(pipes.Stderr, errPipe) - } - - if startCallback != nil { - startCallback(c) - } - - log.Printf("waiting on process") - if err := command.Wait(); err != nil { - if _, ok := err.(*exec.ExitError); !ok { - return -1, err - } - } - log.Printf("process ended") - return command.ProcessState.Sys().(syscall.WaitStatus).ExitStatus(), nil + return nsinit.Exec(container, factory, stateWriter, term, "/nsinit.log", args) } func (d *driver) Kill(p *execdriver.Command, sig int) error { @@ -207,107 +104,22 @@ func getEnv(key string, env []string) string { return "" } -// sendVethName writes the veth pair name to the child's stdin then closes the -// pipe so that the child stops waiting for more data -func sendVethName(pipe io.Writer, name string) { - fmt.Fprint(pipe, name) -} - -// initializeContainerVeth will create a veth pair and setup the host's -// side of the pair by setting the specified bridge as the master and bringing -// up the interface. -// -// Then will with set the other side of the veth pair into the container's namespaced -// using the pid and returns the veth's interface name to provide to the container to -// finish setting up the interface inside the namespace -func initializeContainerVeth(bridge string, mtu, nspid int) (string, error) { - name1, name2, err := createVethPair() - if err != nil { - return "", err - } - log.Printf("veth pair created %s <> %s", name1, name2) - if err := network.SetInterfaceMaster(name1, bridge); err != nil { - return "", err - } - if err := network.SetMtu(name1, mtu); err != nil { - return "", err - } - if err := network.InterfaceUp(name1); err != nil { - return "", err - } - log.Printf("setting %s inside %d namespace", name2, nspid) - if err := network.SetInterfaceInNamespacePid(name2, nspid); err != nil { - return "", err - } - return name2, nil -} - -func setupWindow(master *os.File) (*term.State, error) { - ws, err := term.GetWinsize(os.Stdin.Fd()) - if err != nil { - return nil, err - } - if err := term.SetWinsize(master.Fd(), ws); err != nil { - return nil, err - } - return term.SetRawTerminal(os.Stdin.Fd()) -} - -// createMasterAndConsole will open /dev/ptmx on the host and retreive the -// pts name for use as the pty slave inside the container -func createMasterAndConsole() (*os.File, string, error) { - master, err := os.OpenFile("/dev/ptmx", syscall.O_RDWR|syscall.O_NOCTTY|syscall.O_CLOEXEC, 0) - if err != nil { - return nil, "", err - } - console, err := system.Ptsname(master) - if err != nil { - return nil, "", err - } - if err := system.Unlockpt(master); err != nil { - return nil, "", err - } - return master, console, nil -} - -// createVethPair will automatically generage two random names for -// the veth pair and ensure that they have been created -func createVethPair() (name1 string, name2 string, err error) { - name1, err = utils.GenerateRandomName("dock", 4) - if err != nil { - return - } - name2, err = utils.GenerateRandomName("dock", 4) - if err != nil { - return - } - if err = network.CreateVethPair(name1, name2); err != nil { - return - } - return -} - -// writePidFile writes the namespaced processes pid to .nspid in the rootfs for the container -func writePidFile(rootfs string, pid int) error { - return ioutil.WriteFile(filepath.Join(rootfs, ".nspid"), []byte(fmt.Sprint(pid)), 0655) -} - -func deletePidFile(rootfs string) error { - return os.Remove(filepath.Join(rootfs, ".nspid")) +type dockerCommandFactory struct { + c *execdriver.Command } // createCommand will return an exec.Cmd with the Cloneflags set to the proper namespaces // defined on the container's configuration and use the current binary as the init with the // args provided -func createCommand(c *execdriver.Command, container *libcontainer.Container, - console, logFile string, pipe uintptr, args []string) { - +func (d *dockerCommandFactory) Create(container *libcontainer.Container, + console, logFile string, syncFd uintptr, args []string) *exec.Cmd { + c := d.c aname, _ := exec.LookPath("nsinit") c.Path = aname c.Args = append([]string{ aname, "-console", console, - "-pipe", fmt.Sprint(pipe), + "-pipe", fmt.Sprint(syncFd), "-log", logFile, "init", }, args...) @@ -316,6 +128,26 @@ func createCommand(c *execdriver.Command, container *libcontainer.Container, } c.Env = container.Env c.Dir = c.Rootfs + + return &c.Cmd +} + +type dockerStateWriter struct { + dsw nsinit.StateWriter + c *execdriver.Command + callback execdriver.StartCallback +} + +func (d *dockerStateWriter) WritePid(pid int) error { + err := d.dsw.WritePid(pid) + if d.callback != nil { + d.callback(d.c) + } + return err +} + +func (d *dockerStateWriter) DeletePid() error { + return d.dsw.DeletePid() } func createContainer(c *execdriver.Command) *libcontainer.Container { @@ -334,7 +166,11 @@ func createContainer(c *execdriver.Command) *libcontainer.Container { Mtu: c.Network.Mtu, Address: fmt.Sprintf("%s/%d", c.Network.IPAddress, c.Network.IPPrefixLen), Gateway: c.Network.Gateway, - Bridge: c.Network.Bridge, + Type: "veth", + Context: libcontainer.Context{ + "prefix": "dock", + "bridge": c.Network.Bridge, + }, } } if c.Privileged { @@ -347,3 +183,29 @@ func createContainer(c *execdriver.Command) *libcontainer.Container { } return container } + +type dockerStdTerm struct { + lxc.StdConsole + pipes *execdriver.Pipes +} + +func (d *dockerStdTerm) Attach(cmd *exec.Cmd) error { + return d.AttachPipes(cmd, d.pipes) +} + +func (d *dockerStdTerm) SetMaster(master *os.File) { + // do nothing +} + +type dockerTtyTerm struct { + lxc.TtyConsole + pipes *execdriver.Pipes +} + +func (t *dockerTtyTerm) Attach(cmd *exec.Cmd) error { + return t.AttachPipes(cmd, t.pipes) +} + +func (t *dockerTtyTerm) SetMaster(master *os.File) { + t.MasterPty = master +} diff --git a/pkg/libcontainer/nsinit/state.go b/pkg/libcontainer/nsinit/state.go index 1f0fedd110..2dbaaa5977 100644 --- a/pkg/libcontainer/nsinit/state.go +++ b/pkg/libcontainer/nsinit/state.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" ) type StateWriter interface { @@ -12,13 +13,14 @@ type StateWriter interface { } type DefaultStateWriter struct { + Root string } // writePidFile writes the namespaced processes pid to .nspid in the rootfs for the container -func (*DefaultStateWriter) WritePid(pid int) error { - return ioutil.WriteFile(".nspid", []byte(fmt.Sprint(pid)), 0655) +func (d *DefaultStateWriter) WritePid(pid int) error { + return ioutil.WriteFile(filepath.Join(d.Root, ".nspid"), []byte(fmt.Sprint(pid)), 0655) } -func (*DefaultStateWriter) DeletePid() error { - return os.Remove(".nspid") +func (d *DefaultStateWriter) DeletePid() error { + return os.Remove(filepath.Join(d.Root, ".nspid")) } diff --git a/pkg/libcontainer/nsinit/term.go b/pkg/libcontainer/nsinit/term.go index 649246891e..58dccab2b8 100644 --- a/pkg/libcontainer/nsinit/term.go +++ b/pkg/libcontainer/nsinit/term.go @@ -11,6 +11,7 @@ type Terminal interface { io.Closer SetMaster(*os.File) Attach(*exec.Cmd) error + Resize(h, w int) error } func NewTerminal(stdin io.Reader, stdout, stderr io.Writer, tty bool) Terminal { @@ -35,6 +36,10 @@ type TtyTerminal struct { state *term.State } +func (t *TtyTerminal) Resize(h, w int) error { + return term.SetWinsize(t.master.Fd(), &term.Winsize{Height: uint16(h), Width: uint16(w)}) +} + func (t *TtyTerminal) SetMaster(master *os.File) { t.master = master } @@ -83,6 +88,10 @@ func (s *StdTerminal) Close() error { return nil } +func (s *StdTerminal) Resize(h, w int) error { + return nil +} + func (s *StdTerminal) Attach(command *exec.Cmd) error { inPipe, err := command.StdinPipe() if err != nil {