Ensure a reliable way to kill ghost containers on reboot

Docker-DCO-1.1-Signed-off-by: Michael Crosby <michael@crosbymichael.com> (github: crosbymichael)

pkg/libcontainer/nsinit/exec.go

@@ -50,8 +50,13 @@ func (ns *linuxNs) Exec(container *libcontainer.Container, term Terminal, args [
 	if err := command.Start(); err != nil {
 		return -1, err
 	}
+
+	started, err := system.GetProcessStartTime(command.Process.Pid)
+	if err != nil {
+		return -1, err
+	}
 	ns.logger.Printf("writting pid %d to file\n", command.Process.Pid)
-	if err := ns.stateWriter.WritePid(command.Process.Pid); err != nil {
+	if err := ns.stateWriter.WritePid(command.Process.Pid, started); err != nil {
 		command.Process.Kill()
 		return -1, err
 	}

pkg/libcontainer/nsinit/state.go

@@ -10,7 +10,7 @@ import (
 // StateWriter handles writing and deleting the pid file
 // on disk
 type StateWriter interface {
-	WritePid(pid int) error
+	WritePid(pid int, startTime string) error
 	DeletePid() error
 }
 
@@ -19,10 +19,18 @@ type DefaultStateWriter struct {
 }
 
 // writePidFile writes the namespaced processes pid to pid in the rootfs for the container
-func (d *DefaultStateWriter) WritePid(pid int) error {
-	return ioutil.WriteFile(filepath.Join(d.Root, "pid"), []byte(fmt.Sprint(pid)), 0655)
+func (d *DefaultStateWriter) WritePid(pid int, startTime string) error {
+	err := ioutil.WriteFile(filepath.Join(d.Root, "pid"), []byte(fmt.Sprint(pid)), 0655)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(filepath.Join(d.Root, "start"), []byte(startTime), 0655)
 }
 
 func (d *DefaultStateWriter) DeletePid() error {
-	return os.Remove(filepath.Join(d.Root, "pid"))
+	err := os.Remove(filepath.Join(d.Root, "pid"))
+	if serr := os.Remove(filepath.Join(d.Root, "start")); err == nil {
+		err = serr
+	}
+	return err
 }

pkg/system/proc.go

@@ -0,0 +1,26 @@
+package system
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+// look in /proc to find the process start time so that we can verify
+// that this pid has started after ourself
+func GetProcessStartTime(pid int) (string, error) {
+	data, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "stat"))
+	if err != nil {
+		return "", err
+	}
+	parts := strings.Split(string(data), " ")
+	// the starttime is located at pos 22
+	// from the man page
+	//
+	// starttime %llu (was %lu before Linux 2.6)
+	// (22)  The  time the process started after system boot.  In kernels before Linux 2.6, this
+	// value was expressed in jiffies.  Since Linux 2.6, the value is expressed in  clock  ticks
+	// (divide by sysconf(_SC_CLK_TCK)).
+	return parts[22-1], nil // starts at 1
+}

runtime/execdriver/driver.go

@@ -84,6 +84,7 @@ type Driver interface {
 	Name() string                                 // Driver name
 	Info(id string) Info                          // "temporary" hack (until we move state from core to plugins)
 	GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container.
+	Terminate(c *Command) error                   // kill it with fire
 }
 
 // Network settings of the container

runtime/execdriver/lxc/driver.go

@@ -204,6 +204,10 @@ func (d *driver) Kill(c *execdriver.Command, sig int) error {
 	return KillLxc(c.ID, sig)
 }
 
+func (d *driver) Terminate(c *execdriver.Command) error {
+	return KillLxc(c.ID, 9)
+}
+
 func (d *driver) version() string {
 	var (
 		version string

runtime/execdriver/native/driver.go

@@ -117,9 +117,39 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 }
 
 func (d *driver) Kill(p *execdriver.Command, sig int) error {
-	err := syscall.Kill(p.Process.Pid, syscall.Signal(sig))
+	return syscall.Kill(p.Process.Pid, syscall.Signal(sig))
+}
+
+func (d *driver) Terminate(p *execdriver.Command) error {
+	// lets check the start time for the process
+	started, err := d.readStartTime(p)
+	if err != nil {
+		// if we don't have the data on disk then we can assume the process is gone
+		// because this is only removed after we know the process has stopped
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return err
+	}
+
+	currentStartTime, err := system.GetProcessStartTime(p.Process.Pid)
+	if err != nil {
+		return err
+	}
+	if started == currentStartTime {
+		err = syscall.Kill(p.Process.Pid, 9)
+	}
 	d.removeContainerRoot(p.ID)
 	return err
+
+}
+
+func (d *driver) readStartTime(p *execdriver.Command) (string, error) {
+	data, err := ioutil.ReadFile(filepath.Join(d.root, p.ID, "start"))
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
 }
 
 func (d *driver) Info(id string) execdriver.Info {
@@ -235,9 +265,9 @@ type dockerStateWriter struct {
 	callback execdriver.StartCallback
 }
 
-func (d *dockerStateWriter) WritePid(pid int) error {
+func (d *dockerStateWriter) WritePid(pid int, started string) error {
 	d.c.ContainerPid = pid
-	err := d.dsw.WritePid(pid)
+	err := d.dsw.WritePid(pid, started)
 	if d.callback != nil {
 		d.callback(d.c)
 	}

runtime/runtime.go

@@ -192,7 +192,7 @@ func (runtime *Runtime) Register(container *Container) error {
 				if err != nil {
 					utils.Debugf("cannot find existing process for %d", existingPid)
 				}
-				runtime.execDriver.Kill(cmd, 9)
+				runtime.execDriver.Terminate(cmd)
 			}
 			if err := container.Unmount(); err != nil {
 				utils.Debugf("ghost unmount error %s", err)