Merge pull request #40689 from AkihiroSuda/test-rootless2

test-integration: support more rootless tests

hack/dockerfile/install/rootlesskit.installer

@@ -1,7 +1,7 @@
 #!/bin/sh
 
-# v0.9.1
-: ${ROOTLESSKIT_COMMIT:=db9657404cd538820e9e83d90dab2a78d8b833e6}
+# v0.9.2
+: ${ROOTLESSKIT_COMMIT:=eefbc3f7fb73d9a993605c9ff9a36bfcad6c1270}
 
 install_rootlesskit() {
 	case "$1" in

integration/container/daemon_linux_test.go

@@ -30,6 +30,7 @@ import (
 func TestContainerStartOnDaemonRestart(t *testing.T) {
 	skip.If(t, testEnv.IsRemoteDaemon, "cannot start daemon on remote test run")
 	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
+	skip.If(t, testEnv.IsRootless)
 	t.Parallel()
 
 	d := daemon.New(t)
@@ -129,6 +130,7 @@ func TestDaemonRestartIpcMode(t *testing.T) {
 func TestDaemonHostGatewayIP(t *testing.T) {
 	skip.If(t, testEnv.IsRemoteDaemon)
 	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
 	t.Parallel()
 
 	// Verify the IP in /etc/hosts is same as host-gateway-ip

integration/container/ipcmode_linux_test.go

@@ -278,6 +278,7 @@ func TestDaemonIpcModePrivate(t *testing.T) {
 
 // used to check if an IpcMode given in config works as intended
 func testDaemonIpcFromConfig(t *testing.T, mode string, mustExist bool) {
+	skip.If(t, testEnv.IsRootless, "cannot test /dev/shm in rootless")
 	config := `{"default-ipc-mode": "` + mode + `"}`
 	file := fs.NewFile(t, "test-daemon-ipc-config", fs.WithContent(config))
 	defer file.Remove()

integration/container/restart_test.go

@@ -16,6 +16,7 @@ import (
 func TestDaemonRestartKillContainers(t *testing.T) {
 	skip.If(t, testEnv.IsRemoteDaemon, "cannot start daemon on remote test run")
 	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
+	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support live-restore")
 	type testCase struct {
 		desc       string
 		config     *container.Config

integration/image/remove_unix_test.go

@@ -34,6 +34,7 @@ func TestRemoveImageGarbageCollector(t *testing.T) {
 	// daemon for remove image layer.
 	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
 	skip.If(t, os.Getenv("DOCKER_ENGINE_GOARCH") != "amd64")
+	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support overlay2 on most distros")
 
 	// Create daemon with overlay2 graphdriver because vfs uses disk differently
 	// and this test case would not work with it.

integration/internal/swarm/service.go

@@ -53,6 +53,7 @@ func NewSwarm(t *testing.T, testEnv *environment.Execution, ops ...daemon.Option
 	t.Helper()
 	skip.If(t, testEnv.IsRemoteDaemon)
 	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
+	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode")
 	if testEnv.DaemonInfo.ExperimentalBuild {
 		ops = append(ops, daemon.WithExperimental())
 	}

integration/network/inspect_test.go

@@ -14,6 +14,7 @@ import (
 
 func TestInspectNetwork(t *testing.T) {
 	skip.If(t, testEnv.OSType == "windows", "FIXME")
+	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode")
 	defer setupTest(t)()
 	d := swarm.NewSwarm(t, testEnv)
 	defer d.Stop(t)

integration/network/macvlan/macvlan_test.go

@@ -19,6 +19,7 @@ import (
 func TestDockerNetworkMacvlanPersistance(t *testing.T) {
 	// verify the driver automatically provisions the 802.1q link (dm-dummy0.60)
 	skip.If(t, testEnv.IsRemoteDaemon)
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
 
 	d := daemon.New(t)
 	d.StartWithBusybox(t)
@@ -41,6 +42,7 @@ func TestDockerNetworkMacvlanPersistance(t *testing.T) {
 
 func TestDockerNetworkMacvlan(t *testing.T) {
 	skip.If(t, testEnv.IsRemoteDaemon)
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
 
 	for _, tc := range []struct {
 		name string

integration/network/network_test.go

@@ -23,6 +23,7 @@ func TestRunContainerWithBridgeNone(t *testing.T) {
 	skip.If(t, testEnv.IsRemoteDaemon, "cannot start daemon on remote test run")
 	skip.If(t, testEnv.DaemonInfo.OSType != "linux")
 	skip.If(t, IsUserNamespace())
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
 
 	d := daemon.New(t)
 	d.StartWithBusybox(t, "-b", "none")
@@ -95,6 +96,7 @@ func TestNetworkInvalidJSON(t *testing.T) {
 func TestHostIPv4BridgeLabel(t *testing.T) {
 	skip.If(t, testEnv.OSType == "windows")
 	skip.If(t, testEnv.IsRemoteDaemon)
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
 	d := daemon.New(t)
 	d.Start(t)
 	defer d.Stop(t)

integration/plugin/authz/main_test.go

@@ -49,6 +49,7 @@ func TestMain(m *testing.M) {
 func setupTest(t *testing.T) func() {
 	skip.If(t, testEnv.IsRemoteDaemon, "cannot run daemon when remote daemon")
 	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of localhost")
 	environment.ProtectAll(t, testEnv)
 
 	d = daemon.New(t, daemon.WithExperimental())

integration/plugin/graphdriver/external_test.go

@@ -48,6 +48,7 @@ func TestExternalGraphDriver(t *testing.T) {
 	skip.If(t, runtime.GOOS == "windows")
 	skip.If(t, testEnv.IsRemoteDaemon, "cannot run daemon when remote daemon")
 	skip.If(t, !requirement.HasHubConnectivity(t))
+	skip.If(t, testEnv.IsRootless, "rootless mode doesn't support external graph driver")
 
 	// Setup plugin(s)
 	ec := make(map[string]*graphEventsCounter)

integration/plugin/volumes/mounts_test.go

@@ -18,6 +18,7 @@ import (
 func TestPluginWithDevMounts(t *testing.T) {
 	skip.If(t, testEnv.IsRemoteDaemon, "cannot run daemon when remote daemon")
 	skip.If(t, testEnv.DaemonInfo.OSType == "windows")
+	skip.If(t, testEnv.IsRootless)
 	t.Parallel()
 
 	d := daemon.New(t)

integration/system/info_linux_test.go

@@ -92,6 +92,7 @@ func TestInfoDiscoveryInvalidAdvertise(t *testing.T) {
 // configured with interface name properly show the advertise ip-address in info output.
 func TestInfoDiscoveryAdvertiseInterfaceName(t *testing.T) {
 	skip.If(t, testEnv.IsRemoteDaemon, "cannot run daemon when remote daemon")
+	skip.If(t, testEnv.IsRootless, "rootless mode has different view of network")
 	// TODO should we check for networking availability (integration-cli suite checks for networking through `Network()`)
 
 	d := daemon.New(t)

testutil/daemon/daemon.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"os/user"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -40,8 +41,9 @@ type nopLog struct{}
 func (nopLog) Logf(string, ...interface{}) {}
 
 const (
-	defaultDockerdBinary    = "dockerd"
-	defaultContainerdSocket = "/var/run/docker/containerd/containerd.sock"
+	defaultDockerdBinary         = "dockerd"
+	defaultContainerdSocket      = "/var/run/docker/containerd/containerd.sock"
+	defaultDockerdRootlessBinary = "dockerd-rootless.sh"
 )
 
 var errDaemonNotStarted = errors.New("daemon not started")
@@ -77,6 +79,8 @@ type Daemon struct {
 	pidFile                    string
 	args                       []string
 	containerdSocket           string
+	rootlessUser               *user.User
+	rootlessXDGRuntimeDir      string
 
 	// swarm related field
 	swarmListenAddr string
@@ -134,6 +138,46 @@ func NewDaemon(workingDir string, ops ...Option) (*Daemon, error) {
 		op(d)
 	}
 
+	if d.rootlessUser != nil {
+		if err := os.Chmod(SockRoot, 0777); err != nil {
+			return nil, err
+		}
+		uid, err := strconv.Atoi(d.rootlessUser.Uid)
+		if err != nil {
+			return nil, err
+		}
+		gid, err := strconv.Atoi(d.rootlessUser.Gid)
+		if err != nil {
+			return nil, err
+		}
+		if err := os.Chown(d.Folder, uid, gid); err != nil {
+			return nil, err
+		}
+		if err := os.Chown(d.Root, uid, gid); err != nil {
+			return nil, err
+		}
+		if err := os.MkdirAll(filepath.Dir(d.execRoot), 0700); err != nil {
+			return nil, err
+		}
+		if err := os.Chown(filepath.Dir(d.execRoot), uid, gid); err != nil {
+			return nil, err
+		}
+		if err := os.MkdirAll(d.execRoot, 0700); err != nil {
+			return nil, err
+		}
+		if err := os.Chown(d.execRoot, uid, gid); err != nil {
+			return nil, err
+		}
+		d.rootlessXDGRuntimeDir = filepath.Join(d.Folder, "xdgrun")
+		if err := os.MkdirAll(d.rootlessXDGRuntimeDir, 0700); err != nil {
+			return nil, err
+		}
+		if err := os.Chown(d.rootlessXDGRuntimeDir, uid, gid); err != nil {
+			return nil, err
+		}
+		d.containerdSocket = ""
+	}
+
 	return d, nil
 }
 
@@ -152,11 +196,22 @@ func New(t testing.TB, ops ...Option) *Daemon {
 	assert.Check(t, dest != "", "Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable")
 
 	if os.Getenv("DOCKER_ROOTLESS") != "" {
-		t.Skip("github.com/docker/docker/testutil/daemon.Daemon doesn't support DOCKER_ROOTLESS")
+		if os.Getenv("DOCKER_REMAP_ROOT") != "" {
+			t.Skip("DOCKER_ROOTLESS doesn't support DOCKER_REMAP_ROOT currently")
+		}
+		if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" {
+			if val, err := strconv.ParseBool(env); err == nil && !val {
+				t.Skip("DOCKER_ROOTLESS doesn't support DOCKER_USERLANDPROXY=false")
+			}
+		}
+		ops = append(ops, WithRootlessUser("unprivilegeduser"), WithExperimental())
 	}
 
 	d, err := NewDaemon(dest, ops...)
 	assert.NilError(t, err, "could not create daemon at %q", dest)
+	if d.rootlessUser != nil && d.dockerdBinary != defaultDockerdBinary {
+		t.Skipf("DOCKER_ROOTLESS doesn't support specifying non-default dockerd binary path %q", d.dockerdBinary)
+	}
 
 	return d
 }
@@ -231,9 +286,6 @@ func (d *Daemon) Cleanup(t testing.TB) {
 // Start starts the daemon and return once it is ready to receive requests.
 func (d *Daemon) Start(t testing.TB, args ...string) {
 	t.Helper()
-	if os.Getenv("DOCKER_ROOTLESS") != "" {
-		t.Skip("github.com/docker/docker/testutil/daemon.Daemon doesn't support DOCKER_ROOTLESS")
-	}
 	if err := d.StartWithError(args...); err != nil {
 		t.Fatalf("[%s] failed to start daemon with arguments %v : %v", d.id, d.args, err)
 	}
@@ -262,14 +314,30 @@ func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error {
 		d.pidFile = filepath.Join(d.Folder, "docker.pid")
 	}
 
-	d.args = []string{
+	d.args = []string{}
+	if d.rootlessUser != nil {
+		if d.dockerdBinary != defaultDockerdBinary {
+			return errors.Errorf("[%s] DOCKER_ROOTLESS doesn't support non-default dockerd binary path %q", d.id, d.dockerdBinary)
+		}
+		dockerdBinary = "sudo"
+		d.args = append(d.args,
+			"-u", d.rootlessUser.Username,
+			"-E", "XDG_RUNTIME_DIR="+d.rootlessXDGRuntimeDir,
+			"-E", "HOME="+d.rootlessUser.HomeDir,
+			"-E", "PATH="+os.Getenv("PATH"),
+			"--",
+			defaultDockerdRootlessBinary,
+		)
+	}
+
+	d.args = append(d.args,
 		"--data-root", d.Root,
 		"--exec-root", d.execRoot,
 		"--pidfile", d.pidFile,
 		fmt.Sprintf("--userland-proxy=%t", d.userlandProxy),
 		"--containerd-namespace", d.id,
-		"--containerd-plugins-namespace", d.id + "p",
-	}
+		"--containerd-plugins-namespace", d.id+"p",
+	)
 	if d.containerdSocket != "" {
 		d.args = append(d.args, "--containerd", d.containerdSocket)
 	}
@@ -315,6 +383,10 @@ func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error {
 	d.cmd.Stdout = out
 	d.cmd.Stderr = out
 	d.logFile = out
+	if d.rootlessUser != nil {
+		// sudo requires this for propagating signals
+		setsid(d.cmd)
+	}
 
 	if err := d.cmd.Start(); err != nil {
 		return errors.Wrapf(err, "[%s] could not start daemon container", d.id)

testutil/daemon/daemon_unix.go

@@ -5,8 +5,10 @@ package daemon // import "github.com/docker/docker/testutil/daemon"
 import (
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
+	"syscall"
 	"testing"
 
 	"golang.org/x/sys/unix"
@@ -46,3 +48,10 @@ func SignalDaemonDump(pid int) {
 func signalDaemonReload(pid int) error {
 	return unix.Kill(pid, unix.SIGHUP)
 }
+
+func setsid(cmd *exec.Cmd) {
+	if cmd.SysProcAttr == nil {
+		cmd.SysProcAttr = &syscall.SysProcAttr{}
+	}
+	cmd.SysProcAttr.Setsid = true
+}

testutil/daemon/daemon_windows.go

@@ -2,6 +2,7 @@ package daemon
 
 import (
 	"fmt"
+	"os/exec"
 	"strconv"
 	"testing"
 
@@ -30,3 +31,6 @@ func (d *Daemon) CgroupNamespace(t testing.TB) string {
 	assert.Assert(t, false)
 	return "cgroup namespaces are not supported on Windows"
 }
+
+func setsid(cmd *exec.Cmd) {
+}

testutil/daemon/ops.go

@@ -1,6 +1,8 @@
 package daemon
 
 import (
+	"os/user"
+
 	"github.com/docker/docker/testutil/environment"
 )
 
@@ -102,3 +104,14 @@ func WithStorageDriver(driver string) Option {
 		d.storageDriver = driver
 	}
 }
+
+// WithRootlessUser sets the daemon to be rootless
+func WithRootlessUser(username string) Option {
+	return func(d *Daemon) {
+		u, err := user.Lookup(username)
+		if err != nil {
+			panic(err)
+		}
+		d.rootlessUser = u
+	}
+}