From 7ed3d265a4499ec03f10537fea0aac3ebaa0cec6 Mon Sep 17 00:00:00 2001 From: Anusha Ragunathan Date: Thu, 24 Mar 2016 09:18:03 -0700 Subject: [PATCH] When using systemd, pass expected cgroupsPath and cli options to runc. runc expects a systemd cgroupsPath to be in slice:scopePrefix:containerName format and the "--systemd-cgroup" option to be set. Update docker accordingly. Fixes 21475 Signed-off-by: Anusha Ragunathan --- Dockerfile | 4 +-- Dockerfile.aarch64 | 4 +-- Dockerfile.armhf | 4 +-- Dockerfile.gccgo | 4 +-- Dockerfile.ppc64le | 4 +-- Dockerfile.s390x | 4 +-- Dockerfile.simple | 4 +-- daemon/daemon_unix.go | 41 ++++++++++++++++++---------- daemon/oci_linux.go | 25 +++++++++++------ docker/daemon_unix.go | 4 +++ docs/reference/commandline/daemon.md | 7 +++-- libcontainerd/remote_linux.go | 25 ++++++++++++++++- man/docker.1.md | 5 ++-- 13 files changed, 92 insertions(+), 43 deletions(-) diff --git a/Dockerfile b/Dockerfile index 23a4c5c656..05d3bd91ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -243,7 +243,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Install runc -ENV RUNC_COMMIT d563bd134293c1026976a8f5764d5df5612f1dbf +ENV RUNC_COMMIT 5439bd2d95229c4e213a219174c7b9da284e3487 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \ @@ -253,7 +253,7 @@ RUN set -x \ && cp runc /usr/local/bin/docker-runc # Install containerd -ENV CONTAINERD_COMMIT c761085e92be09df9d5298f852c328b538f5dc2f +ENV CONTAINERD_COMMIT 471bb075214cf0ad85f74f003ca00c7651638c79 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \ diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 index 3d50968eec..812b87496d 100644 --- a/Dockerfile.aarch64 +++ b/Dockerfile.aarch64 @@ -181,7 +181,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Install runc -ENV RUNC_COMMIT d563bd134293c1026976a8f5764d5df5612f1dbf +ENV RUNC_COMMIT 5439bd2d95229c4e213a219174c7b9da284e3487 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \ @@ -191,7 +191,7 @@ RUN set -x \ && cp runc /usr/local/bin/docker-runc # Install containerd -ENV CONTAINERD_COMMIT c761085e92be09df9d5298f852c328b538f5dc2f +ENV CONTAINERD_COMMIT 471bb075214cf0ad85f74f003ca00c7651638c79 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \ diff --git a/Dockerfile.armhf b/Dockerfile.armhf index aa016a56dc..c3e8e06d22 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -198,7 +198,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Install runc -ENV RUNC_COMMIT d563bd134293c1026976a8f5764d5df5612f1dbf +ENV RUNC_COMMIT 5439bd2d95229c4e213a219174c7b9da284e3487 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \ @@ -208,7 +208,7 @@ RUN set -x \ && cp runc /usr/local/bin/docker-runc # Install containerd -ENV CONTAINERD_COMMIT c761085e92be09df9d5298f852c328b538f5dc2f +ENV CONTAINERD_COMMIT 471bb075214cf0ad85f74f003ca00c7651638c79 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \ diff --git a/Dockerfile.gccgo b/Dockerfile.gccgo index 8135bc1c5a..7c755a57c4 100644 --- a/Dockerfile.gccgo +++ b/Dockerfile.gccgo @@ -74,7 +74,7 @@ WORKDIR /go/src/github.com/docker/docker ENV DOCKER_BUILDTAGS apparmor seccomp selinux # Install runc -ENV RUNC_COMMIT d563bd134293c1026976a8f5764d5df5612f1dbf +ENV RUNC_COMMIT 5439bd2d95229c4e213a219174c7b9da284e3487 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \ @@ -84,7 +84,7 @@ RUN set -x \ && cp runc /usr/local/bin/docker-runc # Install containerd -ENV CONTAINERD_COMMIT c761085e92be09df9d5298f852c328b538f5dc2f +ENV CONTAINERD_COMMIT 471bb075214cf0ad85f74f003ca00c7651638c79 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \ diff --git a/Dockerfile.ppc64le b/Dockerfile.ppc64le index 6c5f8372bf..2dfaeb8e17 100644 --- a/Dockerfile.ppc64le +++ b/Dockerfile.ppc64le @@ -199,7 +199,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Install runc -ENV RUNC_COMMIT d563bd134293c1026976a8f5764d5df5612f1dbf +ENV RUNC_COMMIT 5439bd2d95229c4e213a219174c7b9da284e3487 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \ @@ -209,7 +209,7 @@ RUN set -x \ && cp runc /usr/local/bin/docker-runc # Install containerd -ENV CONTAINERD_COMMIT c761085e92be09df9d5298f852c328b538f5dc2f +ENV CONTAINERD_COMMIT 471bb075214cf0ad85f74f003ca00c7651638c79 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \ diff --git a/Dockerfile.s390x b/Dockerfile.s390x index fdec13b24d..b97c2b9c60 100644 --- a/Dockerfile.s390x +++ b/Dockerfile.s390x @@ -178,7 +178,7 @@ RUN set -x \ && rm -rf "$GOPATH" # Install runc -ENV RUNC_COMMIT d563bd134293c1026976a8f5764d5df5612f1dbf +ENV RUNC_COMMIT 5439bd2d95229c4e213a219174c7b9da284e3487 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \ @@ -188,7 +188,7 @@ RUN set -x \ && cp runc /usr/local/bin/docker-runc # Install containerd -ENV CONTAINERD_COMMIT c761085e92be09df9d5298f852c328b538f5dc2f +ENV CONTAINERD_COMMIT 471bb075214cf0ad85f74f003ca00c7651638c79 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \ diff --git a/Dockerfile.simple b/Dockerfile.simple index 4ed165ed64..061631ab1b 100644 --- a/Dockerfile.simple +++ b/Dockerfile.simple @@ -57,7 +57,7 @@ ENV GOPATH /go:/go/src/github.com/docker/docker/vendor ENV CGO_LDFLAGS -L/lib # Install runc -ENV RUNC_COMMIT d563bd134293c1026976a8f5764d5df5612f1dbf +ENV RUNC_COMMIT 5439bd2d95229c4e213a219174c7b9da284e3487 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \ @@ -67,7 +67,7 @@ RUN set -x \ && cp runc /usr/local/bin/docker-runc # Install containerd -ENV CONTAINERD_COMMIT c761085e92be09df9d5298f852c328b538f5dc2f +ENV CONTAINERD_COMMIT 471bb075214cf0ad85f74f003ca00c7651638c79 RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone git://github.com/docker/containerd.git "$GOPATH/src/github.com/docker/containerd" \ diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index 4ca122abad..1ef7dc2ebe 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -472,28 +472,36 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi func (daemon *Daemon) getCgroupDriver() string { cgroupDriver := cgroupFsDriver - // No other cgroup drivers are supported at the moment. Warn the - // user if they tried to set one other than cgroupfs - for _, option := range daemon.configStore.ExecOptions { + if UsingSystemd(daemon.configStore) { + cgroupDriver = cgroupSystemdDriver + } + return cgroupDriver +} + +// getCD gets the raw value of the native.cgroupdriver option, if set. +func getCD(config *Config) string { + for _, option := range config.ExecOptions { key, val, err := parsers.ParseKeyValueOpt(option) if err != nil || !strings.EqualFold(key, "native.cgroupdriver") { continue } - if val != cgroupFsDriver { - logrus.Warnf("cgroupdriver '%s' is not supported", val) - } + return val } - - return cgroupDriver + return "" } -func usingSystemd(config *Config) bool { - // No support for systemd cgroup atm - return false +// VerifyCgroupDriver validates native.cgroupdriver +func VerifyCgroupDriver(config *Config) error { + cd := getCD(config) + if cd == "" || cd == cgroupFsDriver || cd == cgroupSystemdDriver { + return nil + } + return fmt.Errorf("native.cgroupdriver option %s not supported", cd) } -func (daemon *Daemon) usingSystemd() bool { - return daemon.getCgroupDriver() == cgroupSystemdDriver +// UsingSystemd returns true if cli option includes native.cgroupdriver=systemd +func UsingSystemd(config *Config) bool { + return getCD(config) == cgroupSystemdDriver } // verifyPlatformContainerSettings performs platform-specific validation of the @@ -539,7 +547,7 @@ func verifyPlatformContainerSettings(daemon *Daemon, hostConfig *containertypes. return warnings, fmt.Errorf("Cannot use the --read-only option when user namespaces are enabled") } } - if hostConfig.CgroupParent != "" && daemon.usingSystemd() { + if hostConfig.CgroupParent != "" && UsingSystemd(daemon.configStore) { // CgroupParent for systemd cgroup should be named as "xxx.slice" if len(hostConfig.CgroupParent) <= 6 || !strings.HasSuffix(hostConfig.CgroupParent, ".slice") { return warnings, fmt.Errorf("cgroup-parent for systemd cgroup should be a valid slice named as \"xxx.slice\"") @@ -560,7 +568,10 @@ func verifyDaemonSettings(config *Config) error { if !config.bridgeConfig.EnableIPTables && config.bridgeConfig.EnableIPMasq { config.bridgeConfig.EnableIPMasq = false } - if config.CgroupParent != "" && usingSystemd(config) { + if err := VerifyCgroupDriver(config); err != nil { + return err + } + if config.CgroupParent != "" && UsingSystemd(config) { if len(config.CgroupParent) <= 6 || !strings.HasSuffix(config.CgroupParent, ".slice") { return fmt.Errorf("cgroup-parent for systemd cgroup should be a valid slice named as \"xxx.slice\"") } diff --git a/daemon/oci_linux.go b/daemon/oci_linux.go index 9f14ed078c..34602e2223 100644 --- a/daemon/oci_linux.go +++ b/daemon/oci_linux.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" + "github.com/Sirupsen/logrus" "github.com/docker/docker/container" "github.com/docker/docker/daemon/caps" "github.com/docker/docker/libcontainerd" @@ -583,16 +584,24 @@ func (daemon *Daemon) createSpec(c *container.Container) (*libcontainerd.Spec, e } var cgroupsPath string + scopePrefix := "docker" + parent := "/docker" + useSystemd := UsingSystemd(daemon.configStore) + if useSystemd { + parent = "system.slice" + } + if c.HostConfig.CgroupParent != "" { - cgroupsPath = filepath.Join(c.HostConfig.CgroupParent, c.ID) + parent = c.HostConfig.CgroupParent + } else if daemon.configStore.CgroupParent != "" { + parent = daemon.configStore.CgroupParent + } + + if useSystemd { + cgroupsPath = parent + ":" + scopePrefix + ":" + c.ID + logrus.Debugf("createSpec: cgroupsPath: %s", cgroupsPath) } else { - defaultCgroupParent := "/docker" - if daemon.configStore.CgroupParent != "" { - defaultCgroupParent = daemon.configStore.CgroupParent - } else if daemon.usingSystemd() { - defaultCgroupParent = "system.slice" - } - cgroupsPath = filepath.Join(defaultCgroupParent, c.ID) + cgroupsPath = filepath.Join(parent, c.ID) } s.Linux.CgroupsPath = &cgroupsPath diff --git a/docker/daemon_unix.go b/docker/daemon_unix.go index 1e7785dbc6..809a756334 100644 --- a/docker/daemon_unix.go +++ b/docker/daemon_unix.go @@ -75,6 +75,10 @@ func (cli *DaemonCli) getPlatformRemoteOptions() []libcontainerd.RemoteOption { } else { opts = append(opts, libcontainerd.WithStartDaemon(true)) } + if daemon.UsingSystemd(cli.Config) { + args := []string{"--systemd-cgroup=true"} + opts = append(opts, libcontainerd.WithRuntimeArgs(args)) + } return opts } diff --git a/docs/reference/commandline/daemon.md b/docs/reference/commandline/daemon.md index 7788eefde4..02e9a2a6d4 100644 --- a/docs/reference/commandline/daemon.md +++ b/docs/reference/commandline/daemon.md @@ -490,12 +490,13 @@ with the `--exec-opt` flag. All the flag's options have the `native` prefix. A single `native.cgroupdriver` option is available. The `native.cgroupdriver` option specifies the management of the container's -cgroups. You can specify only specify `cgroupfs` at the moment. If you omit the +cgroups. You can specify only specify `cgroupfs` or `systemd`. If you specify +`systemd` and it is not available, the system errors out. If you omit the `native.cgroupdriver` option,` cgroupfs` is used. -This example explicitely sets the `cgroupdriver` to `cgroupfs`: +This example sets the `cgroupdriver` to `systemd`: - $ sudo docker daemon --exec-opt native.cgroupdriver=cgroupfs + $ sudo docker daemon --exec-opt native.cgroupdriver=systemd Setting this option applies to all containers the daemon launches. diff --git a/libcontainerd/remote_linux.go b/libcontainerd/remote_linux.go index f0284ca5ca..ec836dd7ef 100644 --- a/libcontainerd/remote_linux.go +++ b/libcontainerd/remote_linux.go @@ -45,6 +45,7 @@ type remote struct { clients []*client eventTsPath string pastEvents map[string]*containerd.Event + runtimeArgs []string } // New creates a fresh instance of libcontainerd remote. @@ -340,7 +341,14 @@ func (r *remote) runContainerdDaemon() error { // Start a new instance args := []string{"-l", r.rpcAddr, "--runtime", "docker-runc"} if r.debugLog { - args = append(args, "--debug", "true") + args = append(args, "--debug") + } + if len(r.runtimeArgs) > 0 { + for _, v := range r.runtimeArgs { + args = append(args, "--runtime-args") + args = append(args, v) + } + logrus.Debugf("runContainerdDaemon: runtimeArgs: %s", args) } cmd := exec.Command(containerdBinary, args...) // TODO: store logs? @@ -375,6 +383,21 @@ func (a rpcAddr) Apply(r Remote) error { return fmt.Errorf("WithRemoteAddr option not supported for this remote") } +// WithRuntimeArgs sets the list of runtime args passed to containerd +func WithRuntimeArgs(args []string) RemoteOption { + return runtimeArgs(args) +} + +type runtimeArgs []string + +func (rt runtimeArgs) Apply(r Remote) error { + if remote, ok := r.(*remote); ok { + remote.runtimeArgs = rt + return nil + } + return fmt.Errorf("WithRuntimeArgs option not supported for this remote") +} + // WithStartDaemon defines if libcontainerd should also run containerd daemon. func WithStartDaemon(start bool) RemoteOption { return startDaemon(start) diff --git a/man/docker.1.md b/man/docker.1.md index c34d9c8d9a..6f4f6ab836 100644 --- a/man/docker.1.md +++ b/man/docker.1.md @@ -230,8 +230,9 @@ Use the **--exec-opt** flags to specify options to the execution driver. The following options are available: #### native.cgroupdriver -Specifies the management of the container's `cgroups`. Only `cgroupfs` can be specified -`cgroupfs` at the moment. +Specifies the management of the container's `cgroups`. You can specify `cgroupfs` +or `systemd`. If you specify `systemd` and it is not available, the system errors +out. #### Client For specific client examples please see the man page for the specific Docker