Merge pull request #41854 from cpuguy83/for-linux-1169-plugins-custom-runtime-panic

Add shim config for custom runtimes for plugins

daemon/daemon.go

@@ -965,8 +965,12 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		}
 
 		var rt types.Runtime
-		if runtime := config.GetRuntime(config.GetDefaultRuntimeName()); runtime != nil {
+		if runtime.GOOS != "windows" {
-			rt = *runtime
+			rtPtr, err := d.getRuntime(config.GetDefaultRuntimeName())
+			if err != nil {
+				return nil, err
+			}
+			rt = *rtPtr
 		}
 		return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, config.ContainerdPluginNamespace, m, rt)
 	}

daemon/runtime_unix.go

@@ -10,10 +10,12 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/containerd/cgroups"
 	"github.com/containerd/containerd/runtime/linux/runctypes"
 	v2runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/daemon/config"
+	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -96,14 +98,15 @@ func (daemon *Daemon) initRuntimes(runtimes map[string]types.Runtime) (err error
 	}()
 
 	for name, rt := range runtimes {
-		if len(rt.Args) == 0 {
+		if len(rt.Args) > 0 {
-			continue
+			script := filepath.Join(tmpDir, name)
+			content := fmt.Sprintf("#!/bin/sh\n%s %s $@\n", rt.Path, strings.Join(rt.Args, " "))
+			if err := ioutil.WriteFile(script, []byte(content), 0700); err != nil {
+				return err
+			}
 		}
-
+		if rt.Shim == nil {
-		script := filepath.Join(tmpDir, name)
+			rt.Shim = defaultV2ShimConfig(daemon.configStore, rt.Path)
-		content := fmt.Sprintf("#!/bin/sh\n%s %s $@\n", rt.Path, strings.Join(rt.Args, " "))
-		if err := ioutil.WriteFile(script, []byte(content), 0700); err != nil {
-			return err
 		}
 	}
 	return nil
@@ -124,3 +127,32 @@ func (daemon *Daemon) rewriteRuntimePath(name, p string, args []string) (string,
 
 	return filepath.Join(daemon.configStore.Root, "runtimes", name), nil
 }
+
+func (daemon *Daemon) getRuntime(name string) (*types.Runtime, error) {
+	rt := daemon.configStore.GetRuntime(name)
+	if rt == nil {
+		return nil, errdefs.InvalidParameter(errors.Errorf("runtime not found in config: %s", name))
+	}
+
+	if len(rt.Args) > 0 {
+		p, err := daemon.rewriteRuntimePath(name, rt.Path, rt.Args)
+		if err != nil {
+			return nil, err
+		}
+		rt.Path = p
+		rt.Args = nil
+	}
+
+	if rt.Shim == nil {
+		rt.Shim = defaultV2ShimConfig(daemon.configStore, rt.Path)
+	}
+
+	if rt.Shim.Binary == linuxShimV1 {
+		if cgroups.Mode() == cgroups.Unified {
+			return nil, errdefs.InvalidParameter(errors.Errorf("runtime %q is not supported while cgroups v2 (unified hierarchy) is being used", name))
+		}
+		logrus.Warnf("Configured runtime %q is deprecated and will be removed in the next release", name)
+	}
+
+	return rt, nil
+}

daemon/runtime_windows.go

@@ -0,0 +1,10 @@
+package daemon
+
+import (
+	"github.com/docker/docker/api/types"
+	"github.com/pkg/errors"
+)
+
+func (daemon *Daemon) getRuntime(name string) (*types.Runtime, error) {
+	return nil, errors.New("not implemented")
+}

daemon/start_unix.go

@@ -3,11 +3,7 @@
 package daemon // import "github.com/docker/docker/daemon"
 
 import (
-	"github.com/containerd/cgroups"
 	"github.com/docker/docker/container"
-	"github.com/docker/docker/errdefs"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 )
 
 // getLibcontainerdCreateOptions callers must hold a lock on the container
@@ -18,19 +14,9 @@ func (daemon *Daemon) getLibcontainerdCreateOptions(container *container.Contain
 		container.CheckpointTo(daemon.containersReplica)
 	}
 
-	rt := daemon.configStore.GetRuntime(container.HostConfig.Runtime)
+	rt, err := daemon.getRuntime(container.HostConfig.Runtime)
-	if rt.Shim == nil {
+	if err != nil {
-		p, err := daemon.rewriteRuntimePath(container.HostConfig.Runtime, rt.Path, rt.Args)
+		return "", nil, translateContainerdStartErr(container.Path, container.SetExitCode, err)
-		if err != nil {
-			return "", nil, translateContainerdStartErr(container.Path, container.SetExitCode, err)
-		}
-		rt.Shim = defaultV2ShimConfig(daemon.configStore, p)
-	}
-	if rt.Shim.Binary == linuxShimV1 {
-		if cgroups.Mode() == cgroups.Unified {
-			return "", nil, errdefs.InvalidParameter(errors.Errorf("runtime %q is not supported while cgroups v2 (unified hierarchy) is being used", container.HostConfig.Runtime))
-		}
-		logrus.Warnf("Configured runtime %q is deprecated and will be removed in the next release", container.HostConfig.Runtime)
 	}
 
 	return rt.Shim.Binary, rt.Shim.Opts, nil

integration/plugin/common/plugin_test.go

@@ -4,11 +4,14 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
+	"os"
 	"path"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -157,3 +160,66 @@ func TestPluginInstall(t *testing.T) {
 	})
 	// TODO: test insecure registry with https
 }
+
+func TestPluginsWithRuntimes(t *testing.T) {
+	skip.If(t, testEnv.IsRemoteDaemon, "cannot run daemon when remote daemon")
+	skip.If(t, testEnv.IsRootless, "Test not supported on rootless due to buggy daemon setup in rootless mode due to daemon restart")
+	skip.If(t, testEnv.OSType == "windows")
+
+	dir, err := ioutil.TempDir("", t.Name())
+	assert.NilError(t, err)
+	defer os.RemoveAll(dir)
+
+	d := daemon.New(t)
+	defer d.Cleanup(t)
+
+	d.Start(t)
+	defer d.Stop(t)
+
+	ctx := context.Background()
+	client := d.NewClientT(t)
+
+	assert.NilError(t, plugin.Create(ctx, client, "test:latest"))
+	defer client.PluginRemove(ctx, "test:latest", types.PluginRemoveOptions{Force: true})
+
+	assert.NilError(t, client.PluginEnable(ctx, "test:latest", types.PluginEnableOptions{Timeout: 30}))
+
+	p := filepath.Join(dir, "myrt")
+	script := fmt.Sprintf(`#!/bin/sh
+	file="%s/success"
+	if [ "$1" = "someArg" ]; then
+		shift
+		file="${file}_someArg"
+	fi
+
+	touch $file
+	exec runc $@
+	`, dir)
+
+	assert.NilError(t, ioutil.WriteFile(p, []byte(script), 0777))
+
+	type config struct {
+		Runtimes map[string]types.Runtime `json:"runtimes"`
+	}
+
+	cfg, err := json.Marshal(config{
+		Runtimes: map[string]types.Runtime{
+			"myrt":     {Path: p},
+			"myrtArgs": {Path: p, Args: []string{"someArg"}},
+		},
+	})
+	configPath := filepath.Join(dir, "config.json")
+	ioutil.WriteFile(configPath, cfg, 0644)
+
+	t.Run("No Args", func(t *testing.T) {
+		d.Restart(t, "--default-runtime=myrt", "--config-file="+configPath)
+		_, err = os.Stat(filepath.Join(dir, "success"))
+		assert.NilError(t, err)
+	})
+
+	t.Run("With Args", func(t *testing.T) {
+		d.Restart(t, "--default-runtime=myrtArgs", "--config-file="+configPath)
+		_, err = os.Stat(filepath.Join(dir, "success_someArg"))
+		assert.NilError(t, err)
+	})
+}