test-integration: support cgroup2

Usage: DOCKER_BUILD_ARGS="--build-arg CONTAINERD_COMMIT=master --build-arg RUNC_COMMIT=master" DOCKER_EXPERIMENTAL=1 TEST_SKIP_INTEGRATION_CLI=1 make test-integration

Depends on containerd master (v1.4) and runc master (v1.0.0-rc91).

Currently `TEST_SKIP_INTEGRATION_CLI=1` must be specified.

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

hack/dind

@@ -25,6 +25,17 @@ if ! mountpoint -q /tmp; then
 	mount -t tmpfs none /tmp
 fi
 
+# cgroup v2: enable nesting
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+	# move the init process (PID 1) from the root group to the /init group,
+	# otherwise writing subtree_control fails with EBUSY.
+	mkdir -p /sys/fs/cgroup/init
+	echo 1 > /sys/fs/cgroup/init/cgroup.procs
+	# enable controllers
+	sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
+		> /sys/fs/cgroup/cgroup.subtree_control
+fi
+
 if [ $# -gt 0 ]; then
 	exec "$@"
 fi

hack/make/.integration-daemon-start

@@ -64,6 +64,13 @@ if [ "$DOCKER_EXPERIMENTAL" ]; then
 fi
 
 dockerd="dockerd"
+if [ -f "/sys/fs/cgroup/cgroup.controllers" ]; then
+	if [ -z "$TEST_SKIP_INTEGRATION_CLI" ]; then
+		echo >&2 '# cgroup v2 requires TEST_SKIP_INTEGRATION_CLI to be set'
+		exit 1
+	fi
+fi
+
 if [ -n "$DOCKER_ROOTLESS" ]; then
 	if [ -z "$TEST_SKIP_INTEGRATION_CLI" ]; then
 		echo >&2 '# DOCKER_ROOTLESS requires TEST_SKIP_INTEGRATION_CLI to be set'

integration/container/run_cgroupns_linux_test.go

@@ -69,6 +69,7 @@ func TestCgroupNamespacesRunPrivileged(t *testing.T) {
 	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
 	skip.If(t, testEnv.IsRemoteDaemon())
 	skip.If(t, !requirement.CgroupNamespacesEnabled())
+	skip.If(t, testEnv.DaemonInfo.CgroupVersion == "2", "on cgroup v2, privileged containers use private cgroupns")
 
 	// When the daemon defaults to private cgroup namespaces, privileged containers
 	// launched should not be inside their own cgroup namespaces

integration/container/update_linux_test.go

@@ -53,19 +53,34 @@ func TestUpdateMemory(t *testing.T) {
 	assert.Check(t, is.Equal(setMemory, inspect.HostConfig.Memory))
 	assert.Check(t, is.Equal(setMemorySwap, inspect.HostConfig.MemorySwap))
 
+	memoryFile := "/sys/fs/cgroup/memory/memory.limit_in_bytes"
+	if testEnv.DaemonInfo.CgroupVersion == "2" {
+		memoryFile = "/sys/fs/cgroup/memory.max"
+	}
 	res, err := container.Exec(ctx, client, cID,
-		[]string{"cat", "/sys/fs/cgroup/memory/memory.limit_in_bytes"})
+		[]string{"cat", memoryFile})
 	assert.NilError(t, err)
 	assert.Assert(t, is.Len(res.Stderr(), 0))
 	assert.Equal(t, 0, res.ExitCode)
 	assert.Check(t, is.Equal(strconv.FormatInt(setMemory, 10), strings.TrimSpace(res.Stdout())))
 
-	res, err = container.Exec(ctx, client, cID,
-		[]string{"cat", "/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes"})
-	assert.NilError(t, err)
-	assert.Assert(t, is.Len(res.Stderr(), 0))
-	assert.Equal(t, 0, res.ExitCode)
-	assert.Check(t, is.Equal(strconv.FormatInt(setMemorySwap, 10), strings.TrimSpace(res.Stdout())))
+	// see ConvertMemorySwapToCgroupV2Value() for the convention:
+	// https://github.com/opencontainers/runc/commit/c86be8a2c118ca7bad7bbe9eaf106c659a83940d
+	if testEnv.DaemonInfo.CgroupVersion == "2" {
+		res, err = container.Exec(ctx, client, cID,
+			[]string{"cat", "/sys/fs/cgroup/memory.swap.max"})
+		assert.NilError(t, err)
+		assert.Assert(t, is.Len(res.Stderr(), 0))
+		assert.Equal(t, 0, res.ExitCode)
+		assert.Check(t, is.Equal(strconv.FormatInt(setMemorySwap-setMemory, 10), strings.TrimSpace(res.Stdout())))
+	} else {
+		res, err = container.Exec(ctx, client, cID,
+			[]string{"cat", "/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes"})
+		assert.NilError(t, err)
+		assert.Assert(t, is.Len(res.Stderr(), 0))
+		assert.Equal(t, 0, res.ExitCode)
+		assert.Check(t, is.Equal(strconv.FormatInt(setMemorySwap, 10), strings.TrimSpace(res.Stdout())))
+	}
 }
 
 func TestUpdateCPUQuota(t *testing.T) {
@@ -85,24 +100,53 @@ func TestUpdateCPUQuota(t *testing.T) {
 		{desc: "a lower value", update: 10000},
 		{desc: "unset value", update: -1},
 	} {
-		_, err := client.ContainerUpdate(ctx, cID, containertypes.UpdateConfig{
-			Resources: containertypes.Resources{
-				CPUQuota: test.update,
-			},
-		})
-		assert.NilError(t, err)
+		if testEnv.DaemonInfo.CgroupVersion == "2" {
+			// On v2, specifying CPUQuota without CPUPeriod is currently broken:
+			// https://github.com/opencontainers/runc/issues/2456
+			// As a workaround we set them together.
+			_, err := client.ContainerUpdate(ctx, cID, containertypes.UpdateConfig{
+				Resources: containertypes.Resources{
+					CPUQuota:  test.update,
+					CPUPeriod: 100000,
+				},
+			})
+			assert.NilError(t, err)
+		} else {
+			_, err := client.ContainerUpdate(ctx, cID, containertypes.UpdateConfig{
+				Resources: containertypes.Resources{
+					CPUQuota: test.update,
+				},
+			})
+			assert.NilError(t, err)
+		}
 
 		inspect, err := client.ContainerInspect(ctx, cID)
 		assert.NilError(t, err)
 		assert.Check(t, is.Equal(test.update, inspect.HostConfig.CPUQuota))
 
-		res, err := container.Exec(ctx, client, cID,
-			[]string{"/bin/cat", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"})
-		assert.NilError(t, err)
-		assert.Assert(t, is.Len(res.Stderr(), 0))
-		assert.Equal(t, 0, res.ExitCode)
+		if testEnv.DaemonInfo.CgroupVersion == "2" {
+			res, err := container.Exec(ctx, client, cID,
+				[]string{"/bin/cat", "/sys/fs/cgroup/cpu.max"})
+			assert.NilError(t, err)
+			assert.Assert(t, is.Len(res.Stderr(), 0))
+			assert.Equal(t, 0, res.ExitCode)
 
-		assert.Check(t, is.Equal(strconv.FormatInt(test.update, 10), strings.TrimSpace(res.Stdout())))
+			quotaPeriodPair := strings.Fields(res.Stdout())
+			quota := quotaPeriodPair[0]
+			if test.update == -1 {
+				assert.Check(t, is.Equal("max", quota))
+			} else {
+				assert.Check(t, is.Equal(strconv.FormatInt(test.update, 10), quota))
+			}
+		} else {
+			res, err := container.Exec(ctx, client, cID,
+				[]string{"/bin/cat", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"})
+			assert.NilError(t, err)
+			assert.Assert(t, is.Len(res.Stderr(), 0))
+			assert.Equal(t, 0, res.ExitCode)
+
+			assert.Check(t, is.Equal(strconv.FormatInt(test.update, 10), strings.TrimSpace(res.Stdout())))
+		}
 	}
 }
 
@@ -160,7 +204,11 @@ func TestUpdatePidsLimit(t *testing.T) {
 			ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
 			defer cancel()
 
-			res, err := container.Exec(ctx, c, cID, []string{"cat", "/sys/fs/cgroup/pids/pids.max"})
+			pidsFile := "/sys/fs/cgroup/pids/pids.max"
+			if testEnv.DaemonInfo.CgroupVersion == "2" {
+				pidsFile = "/sys/fs/cgroup/pids.max"
+			}
+			res, err := container.Exec(ctx, c, cID, []string{"cat", pidsFile})
 			assert.NilError(t, err)
 			assert.Assert(t, is.Len(res.Stderr(), 0))