From b1d2f52bb2bb900cddb526320b13da18634fe518 Mon Sep 17 00:00:00 2001 From: David Calavera Date: Wed, 26 Aug 2015 14:00:01 +0200 Subject: [PATCH] Improvements to the original sharing implementation. - Print the mount table as in /proc/self/mountinfo - Do not exit prematurely when one of the ipc mounts doesn't exist. - Do not exit prematurely when one of the ipc mounts cannot be unmounted. - Add a unit test to see if the cleanup really works. - Use syscall.MNT_DETACH to cleanup mounts after a crash. - Unmount IPC mounts when the daemon unregisters an old running container. Signed-off-by: David Calavera --- daemon/container.go | 6 +-- daemon/container_unix.go | 34 +++++++++---- daemon/daemon.go | 10 ++-- daemon/daemon_linux.go | 24 +++++++--- daemon/daemon_linux_test.go | 58 +++++++++++++++++++++++ integration-cli/docker_cli_daemon_test.go | 6 ++- 6 files changed, 114 insertions(+), 24 deletions(-) create mode 100644 daemon/daemon_linux_test.go diff --git a/daemon/container.go b/daemon/container.go index ebc985b172..c510d321c5 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -296,7 +296,7 @@ func (container *Container) Start() (err error) { return err } - if !(container.hostConfig.IpcMode.IsContainer() || container.hostConfig.IpcMode.IsHost()) { + if !container.hostConfig.IpcMode.IsContainer() && !container.hostConfig.IpcMode.IsHost() { if err := container.setupIpcDirs(); err != nil { return err } @@ -366,11 +366,11 @@ func (container *Container) cleanup() { container.releaseNetwork() if err := container.unmountIpcMounts(); err != nil { - logrus.Errorf("%v: Failed to umount ipc filesystems: %v", container.ID, err) + logrus.Errorf("%s: Failed to umount ipc filesystems: %v", container.ID, err) } if err := container.Unmount(); err != nil { - logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err) + logrus.Errorf("%s: Failed to umount filesystem: %v", container.ID, err) } for _, eConfig := range container.execCommands.s { diff --git a/daemon/container_unix.go b/daemon/container_unix.go index 697e8b6063..c55079dcc1 100644 --- a/daemon/container_unix.go +++ b/daemon/container_unix.go @@ -1242,10 +1242,10 @@ func (container *Container) removeMountPoints(rm bool) error { } func (container *Container) shmPath() (string, error) { - return container.GetRootResourcePath("shm") + return container.getRootResourcePath("shm") } func (container *Container) mqueuePath() (string, error) { - return container.GetRootResourcePath("mqueue") + return container.getRootResourcePath("mqueue") } func (container *Container) setupIpcDirs() error { @@ -1258,7 +1258,7 @@ func (container *Container) setupIpcDirs() error { return err } - if err := syscall.Mount("shm", shmPath, "tmpfs", uintptr(syscall.MS_NOEXEC|syscall.MS_NOSUID|syscall.MS_NODEV), label.FormatMountLabel("mode=1777,size=65536k", container.GetMountLabel())); err != nil { + if err := syscall.Mount("shm", shmPath, "tmpfs", uintptr(syscall.MS_NOEXEC|syscall.MS_NOSUID|syscall.MS_NODEV), label.FormatMountLabel("mode=1777,size=65536k", container.getMountLabel())); err != nil { return fmt.Errorf("mounting shm tmpfs: %s", err) } @@ -1283,22 +1283,32 @@ func (container *Container) unmountIpcMounts() error { return nil } + var errors []string shmPath, err := container.shmPath() if err != nil { - return fmt.Errorf("shm path does not exist %v", err) - } + logrus.Error(err) + errors = append(errors, err.Error()) + } else { + if err := detachMounted(shmPath); err != nil { + logrus.Errorf("failed to umount %s: %v", shmPath, err) + errors = append(errors, err.Error()) + } - if err := syscall.Unmount(shmPath, syscall.MNT_DETACH); err != nil { - return fmt.Errorf("failed to umount %s filesystem %v", shmPath, err) } mqueuePath, err := container.mqueuePath() if err != nil { - return fmt.Errorf("mqueue path does not exist %v", err) + logrus.Error(err) + errors = append(errors, err.Error()) + } else { + if err := detachMounted(mqueuePath); err != nil { + logrus.Errorf("failed to umount %s: %v", mqueuePath, err) + errors = append(errors, err.Error()) + } } - if err := syscall.Unmount(mqueuePath, syscall.MNT_DETACH); err != nil { - return fmt.Errorf("failed to umount %s filesystem %v", mqueuePath, err) + if len(errors) > 0 { + return fmt.Errorf("failed to cleanup ipc mounts:\n%v", strings.Join(errors, "\n")) } return nil @@ -1322,3 +1332,7 @@ func (container *Container) ipcMounts() []execdriver.Mount { }) return mounts } + +func detachMounted(path string) error { + return syscall.Unmount(path, syscall.MNT_DETACH) +} diff --git a/daemon/daemon.go b/daemon/daemon.go index 7c1673ffdf..18cc09a399 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -209,6 +209,9 @@ func (daemon *Daemon) Register(container *Container) error { } daemon.execDriver.Terminate(cmd) + if err := container.unmountIpcMounts(); err != nil { + logrus.Errorf("%s: Failed to umount ipc filesystems: %v", container.ID, err) + } if err := container.Unmount(); err != nil { logrus.Debugf("unmount error %s", err) } @@ -756,13 +759,14 @@ func NewDaemon(config *Config, registryService *registry.Service) (daemon *Daemo d.EventsService = eventsService d.volumes = volStore d.root = config.Root - go d.execCommandGC() - if err := d.restore(); err != nil { + if err := d.cleanupMounts(); err != nil { return nil, err } - if err := d.cleanupMounts(); err != nil { + go d.execCommandGC() + + if err := d.restore(); err != nil { return nil, err } diff --git a/daemon/daemon_linux.go b/daemon/daemon_linux.go index 0c52de3734..0ae9096522 100644 --- a/daemon/daemon_linux.go +++ b/daemon/daemon_linux.go @@ -2,12 +2,13 @@ package daemon import ( "bufio" + "fmt" + "io" "os" "path/filepath" "strings" "github.com/Sirupsen/logrus" - "github.com/docker/docker/pkg/mount" ) // cleanupMounts umounts shm/mqueue mounts for old containers @@ -19,17 +20,24 @@ func (daemon *Daemon) cleanupMounts() error { } defer f.Close() - sc := bufio.NewScanner(f) + return daemon.cleanupMountsFromReader(f, detachMounted) +} + +func (daemon *Daemon) cleanupMountsFromReader(reader io.Reader, unmount func(target string) error) error { + sc := bufio.NewScanner(reader) + var errors []string for sc.Scan() { line := sc.Text() - fields := strings.Split(line, " ") + fields := strings.Fields(line) if strings.HasPrefix(fields[4], daemon.repository) { + logrus.Debugf("Mount base: %v, repository %s", fields[4], daemon.repository) mnt := fields[4] mountBase := filepath.Base(mnt) if mountBase == "mqueue" || mountBase == "shm" { - logrus.Debugf("Unmounting %+v", mnt) - if err := mount.Unmount(mnt); err != nil { - return err + logrus.Debugf("Unmounting %v", mnt) + if err := unmount(mnt); err != nil { + logrus.Error(err) + errors = append(errors, err.Error()) } } } @@ -39,6 +47,10 @@ func (daemon *Daemon) cleanupMounts() error { return err } + if len(errors) > 0 { + return fmt.Errorf("Error cleaningup mounts:\n%v", strings.Join(errors, "\n")) + } + logrus.Debugf("Cleaning up old shm/mqueue mounts: done.") return nil } diff --git a/daemon/daemon_linux_test.go b/daemon/daemon_linux_test.go new file mode 100644 index 0000000000..6179c700d0 --- /dev/null +++ b/daemon/daemon_linux_test.go @@ -0,0 +1,58 @@ +// +build linux + +package daemon + +import ( + "strings" + "testing" +) + +func TestCleanupMounts(t *testing.T) { + fixture := `230 138 0:60 / / rw,relatime - overlay overlay rw,lowerdir=/var/lib/docker/overlay/0ef9f93d5d365c1385b09d54bbee6afff3d92002c16f22eccb6e1549b2ff97d8/root,upperdir=/var/lib/docker/overlay/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb/upper,workdir=/var/lib/docker/overlay/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb/work +231 230 0:56 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw +232 230 0:57 / /dev rw,nosuid - tmpfs tmpfs rw,mode=755 +233 232 0:58 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +234 232 0:59 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +235 232 0:55 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw +236 230 0:61 / /sys rw,nosuid,nodev,noexec,relatime - sysfs sysfs rw +237 236 0:62 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime - tmpfs tmpfs rw +238 237 0:21 /system.slice/docker.service /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd +239 237 0:23 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/perf_event rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,perf_event +240 237 0:24 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpuset,clone_children +241 237 0:25 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,devices +242 237 0:26 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/freezer rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,freezer +243 237 0:27 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,cpu,cpuacct +244 237 0:28 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/blkio rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,blkio +245 237 0:29 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/net_cls,net_prio rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,net_cls,net_prio +246 237 0:30 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/hugetlb rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,hugetlb +247 237 0:31 /docker/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime - cgroup cgroup rw,memory +248 230 253:1 /var/lib/docker/volumes/510cc41ac68c48bd4eac932e3e09711673876287abf1b185312cfbfe6261a111/_data /var/lib/docker rw,relatime - ext4 /dev/disk/by-uuid/ba70ea0c-1a8f-4ee4-9687-cb393730e2b5 rw,errors=remount-ro,data=ordered +250 230 253:1 /var/lib/docker/containers/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb/hostname /etc/hostname rw,relatime - ext4 /dev/disk/by-uuid/ba70ea0c-1a8f-4ee4-9687-cb393730e2b5 rw,errors=remount-ro,data=ordered +251 230 253:1 /var/lib/docker/containers/dfac036ce135a8914e292cb2f6fea114f7339983c186366aa26d0051e93162cb/hosts /etc/hosts rw,relatime - ext4 /dev/disk/by-uuid/ba70ea0c-1a8f-4ee4-9687-cb393730e2b5 rw,errors=remount-ro,data=ordered +252 232 0:13 /1 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=000 +139 236 0:11 / /sys/kernel/security rw,relatime - securityfs none rw +140 230 0:54 / /tmp rw,relatime - tmpfs none rw +145 230 0:3 / /run/docker/netns/default rw - nsfs nsfs rw +130 140 0:45 / /tmp/docker_recursive_mount_test312125472/tmpfs rw,relatime - tmpfs tmpfs rw +131 230 0:3 / /run/docker/netns/47903e2e6701 rw - nsfs nsfs rw +133 230 0:55 / /go/src/github.com/docker/docker/bundles/1.9.0-dev/test-integration-cli/d45526097/graph/containers/47903e2e67014246eba27607809d5f5c2437c3bf84c2986393448f84093cc40b/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw` + + d := &Daemon{ + repository: "/go/src/github.com/docker/docker/bundles/1.9.0-dev/test-integration-cli/d45526097/graph/containers/", + } + + expected := "/go/src/github.com/docker/docker/bundles/1.9.0-dev/test-integration-cli/d45526097/graph/containers/47903e2e67014246eba27607809d5f5c2437c3bf84c2986393448f84093cc40b/mqueue" + var unmounted bool + unmount := func(target string) error { + if target == expected { + unmounted = true + } + return nil + } + + d.cleanupMountsFromReader(strings.NewReader(fixture), unmount) + + if !unmounted { + t.Fatalf("Expected to unmount the mqueue") + } +} diff --git a/integration-cli/docker_cli_daemon_test.go b/integration-cli/docker_cli_daemon_test.go index 46843c596e..c9e9ed49d9 100644 --- a/integration-cli/docker_cli_daemon_test.go +++ b/integration-cli/docker_cli_daemon_test.go @@ -1474,9 +1474,11 @@ func (s *DockerDaemonSuite) TestCleanupMountsAfterCrash(c *check.C) { id := strings.TrimSpace(out) c.Assert(s.d.cmd.Process.Signal(os.Kill), check.IsNil) c.Assert(s.d.Start(), check.IsNil) - mountOut, err := exec.Command("mount").CombinedOutput() + mountOut, err := ioutil.ReadFile("/proc/self/mountinfo") c.Assert(err, check.IsNil, check.Commentf("Output: %s", mountOut)) - c.Assert(strings.Contains(string(mountOut), id), check.Equals, false, check.Commentf("Something mounted from older daemon start: %s", mountOut)) + + comment := check.Commentf("%s is still mounted from older daemon start:\nDaemon root repository %s\n%s", id, s.d.folder, mountOut) + c.Assert(strings.Contains(string(mountOut), id), check.Equals, false, comment) } func (s *DockerDaemonSuite) TestRunContainerWithBridgeNone(c *check.C) {