From f310bd29bd5ab52caa0536c85d564461a70abf16 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Sun, 15 Mar 2020 12:24:23 +0900 Subject: [PATCH 1/2] rootless: support forwarding signals from RootlessKit to dockerd See https://github.com/rootless-containers/rootlesskit/pull/127 RootlessKit changes: https://github.com/rootless-containers/rootlesskit/compare/v0.9.1...v0.9.2 Signed-off-by: Akihiro Suda --- hack/dockerfile/install/rootlesskit.installer | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/dockerfile/install/rootlesskit.installer b/hack/dockerfile/install/rootlesskit.installer index 66e478d228..08d465698a 100755 --- a/hack/dockerfile/install/rootlesskit.installer +++ b/hack/dockerfile/install/rootlesskit.installer @@ -1,7 +1,7 @@ #!/bin/sh -# v0.9.1 -: ${ROOTLESSKIT_COMMIT:=db9657404cd538820e9e83d90dab2a78d8b833e6} +# v0.9.2 +: ${ROOTLESSKIT_COMMIT:=eefbc3f7fb73d9a993605c9ff9a36bfcad6c1270} install_rootlesskit() { case "$1" in From 5e1b246b9a292255f3b732757adabb020309b383 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Fri, 13 Mar 2020 22:37:09 +0900 Subject: [PATCH 2/2] test-integration: support more rootless tests Signed-off-by: Akihiro Suda --- integration/container/daemon_linux_test.go | 2 + integration/container/ipcmode_linux_test.go | 1 + integration/container/restart_test.go | 1 + integration/image/remove_unix_test.go | 1 + integration/internal/swarm/service.go | 1 + integration/network/inspect_test.go | 1 + integration/network/macvlan/macvlan_test.go | 2 + integration/network/network_test.go | 2 + integration/plugin/authz/main_test.go | 1 + .../plugin/graphdriver/external_test.go | 1 + integration/plugin/volumes/mounts_test.go | 1 + integration/system/info_linux_test.go | 1 + testutil/daemon/daemon.go | 90 +++++++++++++++++-- testutil/daemon/daemon_unix.go | 9 ++ testutil/daemon/daemon_windows.go | 4 + testutil/daemon/ops.go | 13 +++ 16 files changed, 122 insertions(+), 9 deletions(-) diff --git a/integration/container/daemon_linux_test.go b/integration/container/daemon_linux_test.go index e43f8633c7..328bd12caa 100644 --- a/integration/container/daemon_linux_test.go +++ b/integration/container/daemon_linux_test.go @@ -30,6 +30,7 @@ import ( func TestContainerStartOnDaemonRestart(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon, "cannot start daemon on remote test run") skip.If(t, testEnv.DaemonInfo.OSType == "windows") + skip.If(t, testEnv.IsRootless) t.Parallel() d := daemon.New(t) @@ -129,6 +130,7 @@ func TestDaemonRestartIpcMode(t *testing.T) { func TestDaemonHostGatewayIP(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon) skip.If(t, testEnv.DaemonInfo.OSType == "windows") + skip.If(t, testEnv.IsRootless, "rootless mode has different view of network") t.Parallel() // Verify the IP in /etc/hosts is same as host-gateway-ip diff --git a/integration/container/ipcmode_linux_test.go b/integration/container/ipcmode_linux_test.go index b80f4cff79..abd5f10890 100644 --- a/integration/container/ipcmode_linux_test.go +++ b/integration/container/ipcmode_linux_test.go @@ -278,6 +278,7 @@ func TestDaemonIpcModePrivate(t *testing.T) { // used to check if an IpcMode given in config works as intended func testDaemonIpcFromConfig(t *testing.T, mode string, mustExist bool) { + skip.If(t, testEnv.IsRootless, "cannot test /dev/shm in rootless") config := `{"default-ipc-mode": "` + mode + `"}` file := fs.NewFile(t, "test-daemon-ipc-config", fs.WithContent(config)) defer file.Remove() diff --git a/integration/container/restart_test.go b/integration/container/restart_test.go index e74809d4f5..1aed0d4745 100644 --- a/integration/container/restart_test.go +++ b/integration/container/restart_test.go @@ -16,6 +16,7 @@ import ( func TestDaemonRestartKillContainers(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon, "cannot start daemon on remote test run") skip.If(t, testEnv.DaemonInfo.OSType == "windows") + skip.If(t, testEnv.IsRootless, "rootless mode doesn't support live-restore") type testCase struct { desc string config *container.Config diff --git a/integration/image/remove_unix_test.go b/integration/image/remove_unix_test.go index c134ff2a6e..f1815bb330 100644 --- a/integration/image/remove_unix_test.go +++ b/integration/image/remove_unix_test.go @@ -34,6 +34,7 @@ func TestRemoveImageGarbageCollector(t *testing.T) { // daemon for remove image layer. skip.If(t, testEnv.DaemonInfo.OSType != "linux") skip.If(t, os.Getenv("DOCKER_ENGINE_GOARCH") != "amd64") + skip.If(t, testEnv.IsRootless, "rootless mode doesn't support overlay2 on most distros") // Create daemon with overlay2 graphdriver because vfs uses disk differently // and this test case would not work with it. diff --git a/integration/internal/swarm/service.go b/integration/internal/swarm/service.go index ed30045c2d..efd52609bc 100644 --- a/integration/internal/swarm/service.go +++ b/integration/internal/swarm/service.go @@ -53,6 +53,7 @@ func NewSwarm(t *testing.T, testEnv *environment.Execution, ops ...daemon.Option t.Helper() skip.If(t, testEnv.IsRemoteDaemon) skip.If(t, testEnv.DaemonInfo.OSType == "windows") + skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode") if testEnv.DaemonInfo.ExperimentalBuild { ops = append(ops, daemon.WithExperimental()) } diff --git a/integration/network/inspect_test.go b/integration/network/inspect_test.go index 0a6caac83c..0a97154d8e 100644 --- a/integration/network/inspect_test.go +++ b/integration/network/inspect_test.go @@ -14,6 +14,7 @@ import ( func TestInspectNetwork(t *testing.T) { skip.If(t, testEnv.OSType == "windows", "FIXME") + skip.If(t, testEnv.IsRootless, "rootless mode doesn't support Swarm-mode") defer setupTest(t)() d := swarm.NewSwarm(t, testEnv) defer d.Stop(t) diff --git a/integration/network/macvlan/macvlan_test.go b/integration/network/macvlan/macvlan_test.go index b71149ff68..bdf1c71c6b 100644 --- a/integration/network/macvlan/macvlan_test.go +++ b/integration/network/macvlan/macvlan_test.go @@ -19,6 +19,7 @@ import ( func TestDockerNetworkMacvlanPersistance(t *testing.T) { // verify the driver automatically provisions the 802.1q link (dm-dummy0.60) skip.If(t, testEnv.IsRemoteDaemon) + skip.If(t, testEnv.IsRootless, "rootless mode has different view of network") d := daemon.New(t) d.StartWithBusybox(t) @@ -41,6 +42,7 @@ func TestDockerNetworkMacvlanPersistance(t *testing.T) { func TestDockerNetworkMacvlan(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon) + skip.If(t, testEnv.IsRootless, "rootless mode has different view of network") for _, tc := range []struct { name string diff --git a/integration/network/network_test.go b/integration/network/network_test.go index fd2480470e..8a08cb3450 100644 --- a/integration/network/network_test.go +++ b/integration/network/network_test.go @@ -23,6 +23,7 @@ func TestRunContainerWithBridgeNone(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon, "cannot start daemon on remote test run") skip.If(t, testEnv.DaemonInfo.OSType != "linux") skip.If(t, IsUserNamespace()) + skip.If(t, testEnv.IsRootless, "rootless mode has different view of network") d := daemon.New(t) d.StartWithBusybox(t, "-b", "none") @@ -95,6 +96,7 @@ func TestNetworkInvalidJSON(t *testing.T) { func TestHostIPv4BridgeLabel(t *testing.T) { skip.If(t, testEnv.OSType == "windows") skip.If(t, testEnv.IsRemoteDaemon) + skip.If(t, testEnv.IsRootless, "rootless mode has different view of network") d := daemon.New(t) d.Start(t) defer d.Stop(t) diff --git a/integration/plugin/authz/main_test.go b/integration/plugin/authz/main_test.go index 7c1bf56c57..ec02c09fd9 100644 --- a/integration/plugin/authz/main_test.go +++ b/integration/plugin/authz/main_test.go @@ -49,6 +49,7 @@ func TestMain(m *testing.M) { func setupTest(t *testing.T) func() { skip.If(t, testEnv.IsRemoteDaemon, "cannot run daemon when remote daemon") skip.If(t, testEnv.DaemonInfo.OSType == "windows") + skip.If(t, testEnv.IsRootless, "rootless mode has different view of localhost") environment.ProtectAll(t, testEnv) d = daemon.New(t, daemon.WithExperimental()) diff --git a/integration/plugin/graphdriver/external_test.go b/integration/plugin/graphdriver/external_test.go index c9c99ff070..4c848aa075 100644 --- a/integration/plugin/graphdriver/external_test.go +++ b/integration/plugin/graphdriver/external_test.go @@ -48,6 +48,7 @@ func TestExternalGraphDriver(t *testing.T) { skip.If(t, runtime.GOOS == "windows") skip.If(t, testEnv.IsRemoteDaemon, "cannot run daemon when remote daemon") skip.If(t, !requirement.HasHubConnectivity(t)) + skip.If(t, testEnv.IsRootless, "rootless mode doesn't support external graph driver") // Setup plugin(s) ec := make(map[string]*graphEventsCounter) diff --git a/integration/plugin/volumes/mounts_test.go b/integration/plugin/volumes/mounts_test.go index 87ec9caeca..991b1e1105 100644 --- a/integration/plugin/volumes/mounts_test.go +++ b/integration/plugin/volumes/mounts_test.go @@ -18,6 +18,7 @@ import ( func TestPluginWithDevMounts(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon, "cannot run daemon when remote daemon") skip.If(t, testEnv.DaemonInfo.OSType == "windows") + skip.If(t, testEnv.IsRootless) t.Parallel() d := daemon.New(t) diff --git a/integration/system/info_linux_test.go b/integration/system/info_linux_test.go index ed350661ca..36641a573c 100644 --- a/integration/system/info_linux_test.go +++ b/integration/system/info_linux_test.go @@ -92,6 +92,7 @@ func TestInfoDiscoveryInvalidAdvertise(t *testing.T) { // configured with interface name properly show the advertise ip-address in info output. func TestInfoDiscoveryAdvertiseInterfaceName(t *testing.T) { skip.If(t, testEnv.IsRemoteDaemon, "cannot run daemon when remote daemon") + skip.If(t, testEnv.IsRootless, "rootless mode has different view of network") // TODO should we check for networking availability (integration-cli suite checks for networking through `Network()`) d := daemon.New(t) diff --git a/testutil/daemon/daemon.go b/testutil/daemon/daemon.go index a6acb0ef1a..3df1cf9b86 100644 --- a/testutil/daemon/daemon.go +++ b/testutil/daemon/daemon.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "os/exec" + "os/user" "path/filepath" "strconv" "strings" @@ -40,8 +41,9 @@ type nopLog struct{} func (nopLog) Logf(string, ...interface{}) {} const ( - defaultDockerdBinary = "dockerd" - defaultContainerdSocket = "/var/run/docker/containerd/containerd.sock" + defaultDockerdBinary = "dockerd" + defaultContainerdSocket = "/var/run/docker/containerd/containerd.sock" + defaultDockerdRootlessBinary = "dockerd-rootless.sh" ) var errDaemonNotStarted = errors.New("daemon not started") @@ -77,6 +79,8 @@ type Daemon struct { pidFile string args []string containerdSocket string + rootlessUser *user.User + rootlessXDGRuntimeDir string // swarm related field swarmListenAddr string @@ -134,6 +138,46 @@ func NewDaemon(workingDir string, ops ...Option) (*Daemon, error) { op(d) } + if d.rootlessUser != nil { + if err := os.Chmod(SockRoot, 0777); err != nil { + return nil, err + } + uid, err := strconv.Atoi(d.rootlessUser.Uid) + if err != nil { + return nil, err + } + gid, err := strconv.Atoi(d.rootlessUser.Gid) + if err != nil { + return nil, err + } + if err := os.Chown(d.Folder, uid, gid); err != nil { + return nil, err + } + if err := os.Chown(d.Root, uid, gid); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(d.execRoot), 0700); err != nil { + return nil, err + } + if err := os.Chown(filepath.Dir(d.execRoot), uid, gid); err != nil { + return nil, err + } + if err := os.MkdirAll(d.execRoot, 0700); err != nil { + return nil, err + } + if err := os.Chown(d.execRoot, uid, gid); err != nil { + return nil, err + } + d.rootlessXDGRuntimeDir = filepath.Join(d.Folder, "xdgrun") + if err := os.MkdirAll(d.rootlessXDGRuntimeDir, 0700); err != nil { + return nil, err + } + if err := os.Chown(d.rootlessXDGRuntimeDir, uid, gid); err != nil { + return nil, err + } + d.containerdSocket = "" + } + return d, nil } @@ -152,11 +196,22 @@ func New(t testing.TB, ops ...Option) *Daemon { assert.Check(t, dest != "", "Please set the DOCKER_INTEGRATION_DAEMON_DEST or the DEST environment variable") if os.Getenv("DOCKER_ROOTLESS") != "" { - t.Skip("github.com/docker/docker/testutil/daemon.Daemon doesn't support DOCKER_ROOTLESS") + if os.Getenv("DOCKER_REMAP_ROOT") != "" { + t.Skip("DOCKER_ROOTLESS doesn't support DOCKER_REMAP_ROOT currently") + } + if env := os.Getenv("DOCKER_USERLANDPROXY"); env != "" { + if val, err := strconv.ParseBool(env); err == nil && !val { + t.Skip("DOCKER_ROOTLESS doesn't support DOCKER_USERLANDPROXY=false") + } + } + ops = append(ops, WithRootlessUser("unprivilegeduser"), WithExperimental()) } d, err := NewDaemon(dest, ops...) assert.NilError(t, err, "could not create daemon at %q", dest) + if d.rootlessUser != nil && d.dockerdBinary != defaultDockerdBinary { + t.Skipf("DOCKER_ROOTLESS doesn't support specifying non-default dockerd binary path %q", d.dockerdBinary) + } return d } @@ -231,9 +286,6 @@ func (d *Daemon) Cleanup(t testing.TB) { // Start starts the daemon and return once it is ready to receive requests. func (d *Daemon) Start(t testing.TB, args ...string) { t.Helper() - if os.Getenv("DOCKER_ROOTLESS") != "" { - t.Skip("github.com/docker/docker/testutil/daemon.Daemon doesn't support DOCKER_ROOTLESS") - } if err := d.StartWithError(args...); err != nil { t.Fatalf("[%s] failed to start daemon with arguments %v : %v", d.id, d.args, err) } @@ -262,14 +314,30 @@ func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { d.pidFile = filepath.Join(d.Folder, "docker.pid") } - d.args = []string{ + d.args = []string{} + if d.rootlessUser != nil { + if d.dockerdBinary != defaultDockerdBinary { + return errors.Errorf("[%s] DOCKER_ROOTLESS doesn't support non-default dockerd binary path %q", d.id, d.dockerdBinary) + } + dockerdBinary = "sudo" + d.args = append(d.args, + "-u", d.rootlessUser.Username, + "-E", "XDG_RUNTIME_DIR="+d.rootlessXDGRuntimeDir, + "-E", "HOME="+d.rootlessUser.HomeDir, + "-E", "PATH="+os.Getenv("PATH"), + "--", + defaultDockerdRootlessBinary, + ) + } + + d.args = append(d.args, "--data-root", d.Root, "--exec-root", d.execRoot, "--pidfile", d.pidFile, fmt.Sprintf("--userland-proxy=%t", d.userlandProxy), "--containerd-namespace", d.id, - "--containerd-plugins-namespace", d.id + "p", - } + "--containerd-plugins-namespace", d.id+"p", + ) if d.containerdSocket != "" { d.args = append(d.args, "--containerd", d.containerdSocket) } @@ -315,6 +383,10 @@ func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error { d.cmd.Stdout = out d.cmd.Stderr = out d.logFile = out + if d.rootlessUser != nil { + // sudo requires this for propagating signals + setsid(d.cmd) + } if err := d.cmd.Start(); err != nil { return errors.Wrapf(err, "[%s] could not start daemon container", d.id) diff --git a/testutil/daemon/daemon_unix.go b/testutil/daemon/daemon_unix.go index f638951eb8..fac5297480 100644 --- a/testutil/daemon/daemon_unix.go +++ b/testutil/daemon/daemon_unix.go @@ -5,8 +5,10 @@ package daemon // import "github.com/docker/docker/testutil/daemon" import ( "fmt" "os" + "os/exec" "path/filepath" "strings" + "syscall" "testing" "golang.org/x/sys/unix" @@ -46,3 +48,10 @@ func SignalDaemonDump(pid int) { func signalDaemonReload(pid int) error { return unix.Kill(pid, unix.SIGHUP) } + +func setsid(cmd *exec.Cmd) { + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{} + } + cmd.SysProcAttr.Setsid = true +} diff --git a/testutil/daemon/daemon_windows.go b/testutil/daemon/daemon_windows.go index 3cc87e2bf9..601f60a54c 100644 --- a/testutil/daemon/daemon_windows.go +++ b/testutil/daemon/daemon_windows.go @@ -2,6 +2,7 @@ package daemon import ( "fmt" + "os/exec" "strconv" "testing" @@ -30,3 +31,6 @@ func (d *Daemon) CgroupNamespace(t testing.TB) string { assert.Assert(t, false) return "cgroup namespaces are not supported on Windows" } + +func setsid(cmd *exec.Cmd) { +} diff --git a/testutil/daemon/ops.go b/testutil/daemon/ops.go index 7875822a59..66d169d1de 100644 --- a/testutil/daemon/ops.go +++ b/testutil/daemon/ops.go @@ -1,6 +1,8 @@ package daemon import ( + "os/user" + "github.com/docker/docker/testutil/environment" ) @@ -102,3 +104,14 @@ func WithStorageDriver(driver string) Option { d.storageDriver = driver } } + +// WithRootlessUser sets the daemon to be rootless +func WithRootlessUser(username string) Option { + return func(d *Daemon) { + u, err := user.Lookup(username) + if err != nil { + panic(err) + } + d.rootlessUser = u + } +}