From ed89041433a031cafc0a0f19cfe573c31688d377 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Wed, 3 Jun 2020 22:37:14 +0900 Subject: [PATCH] test-integration: support cgroup2 Usage: DOCKER_BUILD_ARGS="--build-arg CONTAINERD_COMMIT=master --build-arg RUNC_COMMIT=master" DOCKER_EXPERIMENTAL=1 TEST_SKIP_INTEGRATION_CLI=1 make test-integration Depends on containerd master (v1.4) and runc master (v1.0.0-rc91). Currently `TEST_SKIP_INTEGRATION_CLI=1` must be specified. Signed-off-by: Akihiro Suda --- hack/dind | 11 +++ hack/make/.integration-daemon-start | 7 ++ .../container/run_cgroupns_linux_test.go | 1 + integration/container/update_linux_test.go | 88 ++++++++++++++----- 4 files changed, 87 insertions(+), 20 deletions(-) diff --git a/hack/dind b/hack/dind index 4188c1befe..9d3d28e8f7 100755 --- a/hack/dind +++ b/hack/dind @@ -25,6 +25,17 @@ if ! mountpoint -q /tmp; then mount -t tmpfs none /tmp fi +# cgroup v2: enable nesting +if [ -f /sys/fs/cgroup/cgroup.controllers ]; then + # move the init process (PID 1) from the root group to the /init group, + # otherwise writing subtree_control fails with EBUSY. + mkdir -p /sys/fs/cgroup/init + echo 1 > /sys/fs/cgroup/init/cgroup.procs + # enable controllers + sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \ + > /sys/fs/cgroup/cgroup.subtree_control +fi + if [ $# -gt 0 ]; then exec "$@" fi diff --git a/hack/make/.integration-daemon-start b/hack/make/.integration-daemon-start index 57a95cdc8a..df7bd1edb3 100644 --- a/hack/make/.integration-daemon-start +++ b/hack/make/.integration-daemon-start @@ -64,6 +64,13 @@ if [ "$DOCKER_EXPERIMENTAL" ]; then fi dockerd="dockerd" +if [ -f "/sys/fs/cgroup/cgroup.controllers" ]; then + if [ -z "$TEST_SKIP_INTEGRATION_CLI" ]; then + echo >&2 '# cgroup v2 requires TEST_SKIP_INTEGRATION_CLI to be set' + exit 1 + fi +fi + if [ -n "$DOCKER_ROOTLESS" ]; then if [ -z "$TEST_SKIP_INTEGRATION_CLI" ]; then echo >&2 '# DOCKER_ROOTLESS requires TEST_SKIP_INTEGRATION_CLI to be set' diff --git a/integration/container/run_cgroupns_linux_test.go b/integration/container/run_cgroupns_linux_test.go index adf691c3c7..9f17f40371 100644 --- a/integration/container/run_cgroupns_linux_test.go +++ b/integration/container/run_cgroupns_linux_test.go @@ -69,6 +69,7 @@ func TestCgroupNamespacesRunPrivileged(t *testing.T) { skip.If(t, testEnv.DaemonInfo.OSType != "linux") skip.If(t, testEnv.IsRemoteDaemon()) skip.If(t, !requirement.CgroupNamespacesEnabled()) + skip.If(t, testEnv.DaemonInfo.CgroupVersion == "2", "on cgroup v2, privileged containers use private cgroupns") // When the daemon defaults to private cgroup namespaces, privileged containers // launched should not be inside their own cgroup namespaces diff --git a/integration/container/update_linux_test.go b/integration/container/update_linux_test.go index 6e131714fc..4d43124900 100644 --- a/integration/container/update_linux_test.go +++ b/integration/container/update_linux_test.go @@ -53,19 +53,34 @@ func TestUpdateMemory(t *testing.T) { assert.Check(t, is.Equal(setMemory, inspect.HostConfig.Memory)) assert.Check(t, is.Equal(setMemorySwap, inspect.HostConfig.MemorySwap)) + memoryFile := "/sys/fs/cgroup/memory/memory.limit_in_bytes" + if testEnv.DaemonInfo.CgroupVersion == "2" { + memoryFile = "/sys/fs/cgroup/memory.max" + } res, err := container.Exec(ctx, client, cID, - []string{"cat", "/sys/fs/cgroup/memory/memory.limit_in_bytes"}) + []string{"cat", memoryFile}) assert.NilError(t, err) assert.Assert(t, is.Len(res.Stderr(), 0)) assert.Equal(t, 0, res.ExitCode) assert.Check(t, is.Equal(strconv.FormatInt(setMemory, 10), strings.TrimSpace(res.Stdout()))) - res, err = container.Exec(ctx, client, cID, - []string{"cat", "/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes"}) - assert.NilError(t, err) - assert.Assert(t, is.Len(res.Stderr(), 0)) - assert.Equal(t, 0, res.ExitCode) - assert.Check(t, is.Equal(strconv.FormatInt(setMemorySwap, 10), strings.TrimSpace(res.Stdout()))) + // see ConvertMemorySwapToCgroupV2Value() for the convention: + // https://github.com/opencontainers/runc/commit/c86be8a2c118ca7bad7bbe9eaf106c659a83940d + if testEnv.DaemonInfo.CgroupVersion == "2" { + res, err = container.Exec(ctx, client, cID, + []string{"cat", "/sys/fs/cgroup/memory.swap.max"}) + assert.NilError(t, err) + assert.Assert(t, is.Len(res.Stderr(), 0)) + assert.Equal(t, 0, res.ExitCode) + assert.Check(t, is.Equal(strconv.FormatInt(setMemorySwap-setMemory, 10), strings.TrimSpace(res.Stdout()))) + } else { + res, err = container.Exec(ctx, client, cID, + []string{"cat", "/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes"}) + assert.NilError(t, err) + assert.Assert(t, is.Len(res.Stderr(), 0)) + assert.Equal(t, 0, res.ExitCode) + assert.Check(t, is.Equal(strconv.FormatInt(setMemorySwap, 10), strings.TrimSpace(res.Stdout()))) + } } func TestUpdateCPUQuota(t *testing.T) { @@ -85,24 +100,53 @@ func TestUpdateCPUQuota(t *testing.T) { {desc: "a lower value", update: 10000}, {desc: "unset value", update: -1}, } { - _, err := client.ContainerUpdate(ctx, cID, containertypes.UpdateConfig{ - Resources: containertypes.Resources{ - CPUQuota: test.update, - }, - }) - assert.NilError(t, err) + if testEnv.DaemonInfo.CgroupVersion == "2" { + // On v2, specifying CPUQuota without CPUPeriod is currently broken: + // https://github.com/opencontainers/runc/issues/2456 + // As a workaround we set them together. + _, err := client.ContainerUpdate(ctx, cID, containertypes.UpdateConfig{ + Resources: containertypes.Resources{ + CPUQuota: test.update, + CPUPeriod: 100000, + }, + }) + assert.NilError(t, err) + } else { + _, err := client.ContainerUpdate(ctx, cID, containertypes.UpdateConfig{ + Resources: containertypes.Resources{ + CPUQuota: test.update, + }, + }) + assert.NilError(t, err) + } inspect, err := client.ContainerInspect(ctx, cID) assert.NilError(t, err) assert.Check(t, is.Equal(test.update, inspect.HostConfig.CPUQuota)) - res, err := container.Exec(ctx, client, cID, - []string{"/bin/cat", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"}) - assert.NilError(t, err) - assert.Assert(t, is.Len(res.Stderr(), 0)) - assert.Equal(t, 0, res.ExitCode) + if testEnv.DaemonInfo.CgroupVersion == "2" { + res, err := container.Exec(ctx, client, cID, + []string{"/bin/cat", "/sys/fs/cgroup/cpu.max"}) + assert.NilError(t, err) + assert.Assert(t, is.Len(res.Stderr(), 0)) + assert.Equal(t, 0, res.ExitCode) - assert.Check(t, is.Equal(strconv.FormatInt(test.update, 10), strings.TrimSpace(res.Stdout()))) + quotaPeriodPair := strings.Fields(res.Stdout()) + quota := quotaPeriodPair[0] + if test.update == -1 { + assert.Check(t, is.Equal("max", quota)) + } else { + assert.Check(t, is.Equal(strconv.FormatInt(test.update, 10), quota)) + } + } else { + res, err := container.Exec(ctx, client, cID, + []string{"/bin/cat", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"}) + assert.NilError(t, err) + assert.Assert(t, is.Len(res.Stderr(), 0)) + assert.Equal(t, 0, res.ExitCode) + + assert.Check(t, is.Equal(strconv.FormatInt(test.update, 10), strings.TrimSpace(res.Stdout()))) + } } } @@ -160,7 +204,11 @@ func TestUpdatePidsLimit(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() - res, err := container.Exec(ctx, c, cID, []string{"cat", "/sys/fs/cgroup/pids/pids.max"}) + pidsFile := "/sys/fs/cgroup/pids/pids.max" + if testEnv.DaemonInfo.CgroupVersion == "2" { + pidsFile = "/sys/fs/cgroup/pids.max" + } + res, err := container.Exec(ctx, c, cID, []string{"cat", pidsFile}) assert.NilError(t, err) assert.Assert(t, is.Len(res.Stderr(), 0))