cgroup2: use shim V2
* Requires containerd binaries from containerd/containerd#3799 . Metrics are unimplemented yet. * Works with crun v0.10.4, but `--security-opt seccomp=unconfined` is needed unless using master version of libseccomp ( containers/crun#156, seccomp/libseccomp#177 ) * Doesn't work with master runc yet * Resource limitations are unimplemented Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>
This commit is contained in:
parent
e6f6c35b79
commit
612343618d
|
@ -794,6 +794,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
|
||||||
PluginStore: pluginStore,
|
PluginStore: pluginStore,
|
||||||
startupDone: make(chan struct{}),
|
startupDone: make(chan struct{}),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure the daemon is properly shutdown if there is a failure during
|
// Ensure the daemon is properly shutdown if there is a failure during
|
||||||
// initialization
|
// initialization
|
||||||
defer func() {
|
defer func() {
|
||||||
|
@ -914,7 +915,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, config.ContainerdPluginNamespace, m)
|
return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, config.ContainerdPluginNamespace, m, d.useShimV2())
|
||||||
}
|
}
|
||||||
|
|
||||||
// Plugin system initialization should happen before restore. Do not change order.
|
// Plugin system initialization should happen before restore. Do not change order.
|
||||||
|
@ -1063,7 +1064,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
|
||||||
|
|
||||||
go d.execCommandGC()
|
go d.execCommandGC()
|
||||||
|
|
||||||
d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), config.ContainerdNamespace, d)
|
d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), config.ContainerdNamespace, d, d.useShimV2())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -1639,3 +1639,7 @@ func (daemon *Daemon) setupSeccompProfile() error {
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (daemon *Daemon) useShimV2() bool {
|
||||||
|
return cgroups.IsCgroup2UnifiedMode()
|
||||||
|
}
|
||||||
|
|
|
@ -653,3 +653,7 @@ func (daemon *Daemon) initRuntimes(_ map[string]types.Runtime) error {
|
||||||
|
|
||||||
func setupResolvConf(config *config.Config) {
|
func setupResolvConf(config *config.Config) {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (daemon *Daemon) useShimV2() bool {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
|
@ -8,6 +8,7 @@ import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"github.com/containerd/containerd/runtime/linux/runctypes"
|
"github.com/containerd/containerd/runtime/linux/runctypes"
|
||||||
|
v2runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
|
||||||
"github.com/docker/docker/container"
|
"github.com/docker/docker/container"
|
||||||
"github.com/docker/docker/errdefs"
|
"github.com/docker/docker/errdefs"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
@ -43,6 +44,20 @@ func (daemon *Daemon) getLibcontainerdCreateOptions(container *container.Contain
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if daemon.useShimV2() {
|
||||||
|
opts := &v2runcoptions.Options{
|
||||||
|
BinaryName: path,
|
||||||
|
Root: filepath.Join(daemon.configStore.ExecRoot,
|
||||||
|
fmt.Sprintf("runtime-%s", container.HostConfig.Runtime)),
|
||||||
|
}
|
||||||
|
|
||||||
|
if UsingSystemd(daemon.configStore) {
|
||||||
|
opts.SystemdCgroup = true
|
||||||
|
}
|
||||||
|
|
||||||
|
return opts, nil
|
||||||
|
|
||||||
|
}
|
||||||
opts := &runctypes.RuncOptions{
|
opts := &runctypes.RuncOptions{
|
||||||
Runtime: path,
|
Runtime: path,
|
||||||
RuntimeRoot: filepath.Join(daemon.configStore.ExecRoot,
|
RuntimeRoot: filepath.Join(daemon.configStore.ExecRoot,
|
||||||
|
|
|
@ -9,6 +9,6 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewClient creates a new libcontainerd client from a containerd client
|
// NewClient creates a new libcontainerd client from a containerd client
|
||||||
func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) {
|
func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend, useShimV2 bool) (libcontainerdtypes.Client, error) {
|
||||||
return remote.NewClient(ctx, cli, stateDir, ns, b)
|
return remote.NewClient(ctx, cli, stateDir, ns, b, useShimV2)
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,9 +11,10 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
// NewClient creates a new libcontainerd client from a containerd client
|
// NewClient creates a new libcontainerd client from a containerd client
|
||||||
func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) {
|
func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend, useShimV2 bool) (libcontainerdtypes.Client, error) {
|
||||||
if !system.ContainerdRuntimeSupported() {
|
if !system.ContainerdRuntimeSupported() {
|
||||||
|
// useShimV2 is ignored for windows
|
||||||
return local.NewClient(ctx, cli, stateDir, ns, b)
|
return local.NewClient(ctx, cli, stateDir, ns, b)
|
||||||
}
|
}
|
||||||
return remote.NewClient(ctx, cli, stateDir, ns, b)
|
return remote.NewClient(ctx, cli, stateDir, ns, b, useShimV2)
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,6 +23,7 @@ import (
|
||||||
"github.com/containerd/containerd/events"
|
"github.com/containerd/containerd/events"
|
||||||
"github.com/containerd/containerd/images"
|
"github.com/containerd/containerd/images"
|
||||||
"github.com/containerd/containerd/runtime/linux/runctypes"
|
"github.com/containerd/containerd/runtime/linux/runctypes"
|
||||||
|
v2runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
|
||||||
"github.com/containerd/typeurl"
|
"github.com/containerd/typeurl"
|
||||||
"github.com/docker/docker/errdefs"
|
"github.com/docker/docker/errdefs"
|
||||||
"github.com/docker/docker/libcontainerd/queue"
|
"github.com/docker/docker/libcontainerd/queue"
|
||||||
|
@ -45,21 +46,27 @@ type client struct {
|
||||||
logger *logrus.Entry
|
logger *logrus.Entry
|
||||||
ns string
|
ns string
|
||||||
|
|
||||||
backend libcontainerdtypes.Backend
|
backend libcontainerdtypes.Backend
|
||||||
eventQ queue.Queue
|
eventQ queue.Queue
|
||||||
oomMu sync.Mutex
|
oomMu sync.Mutex
|
||||||
oom map[string]bool
|
oom map[string]bool
|
||||||
|
useShimV2 bool
|
||||||
|
v2runcoptionsMu sync.Mutex
|
||||||
|
// v2runcoptions is used for copying options specified on Create() to Start()
|
||||||
|
v2runcoptions map[string]v2runcoptions.Options
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewClient creates a new libcontainerd client from a containerd client
|
// NewClient creates a new libcontainerd client from a containerd client
|
||||||
func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) {
|
func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend, useShimV2 bool) (libcontainerdtypes.Client, error) {
|
||||||
c := &client{
|
c := &client{
|
||||||
client: cli,
|
client: cli,
|
||||||
stateDir: stateDir,
|
stateDir: stateDir,
|
||||||
logger: logrus.WithField("module", "libcontainerd").WithField("namespace", ns),
|
logger: logrus.WithField("module", "libcontainerd").WithField("namespace", ns),
|
||||||
ns: ns,
|
ns: ns,
|
||||||
backend: b,
|
backend: b,
|
||||||
oom: make(map[string]bool),
|
oom: make(map[string]bool),
|
||||||
|
useShimV2: useShimV2,
|
||||||
|
v2runcoptions: make(map[string]v2runcoptions.Options),
|
||||||
}
|
}
|
||||||
|
|
||||||
go c.processEventStream(ctx, ns)
|
go c.processEventStream(ctx, ns)
|
||||||
|
@ -126,9 +133,13 @@ func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, run
|
||||||
bdir := c.bundleDir(id)
|
bdir := c.bundleDir(id)
|
||||||
c.logger.WithField("bundle", bdir).WithField("root", ociSpec.Root.Path).Debug("bundle dir created")
|
c.logger.WithField("bundle", bdir).WithField("root", ociSpec.Root.Path).Debug("bundle dir created")
|
||||||
|
|
||||||
|
rt := runtimeName
|
||||||
|
if c.useShimV2 {
|
||||||
|
rt = shimV2RuntimeName
|
||||||
|
}
|
||||||
newOpts := []containerd.NewContainerOpts{
|
newOpts := []containerd.NewContainerOpts{
|
||||||
containerd.WithSpec(ociSpec),
|
containerd.WithSpec(ociSpec),
|
||||||
containerd.WithRuntime(runtimeName, runtimeOptions),
|
containerd.WithRuntime(rt, runtimeOptions),
|
||||||
WithBundle(bdir, ociSpec),
|
WithBundle(bdir, ociSpec),
|
||||||
}
|
}
|
||||||
opts = append(opts, newOpts...)
|
opts = append(opts, newOpts...)
|
||||||
|
@ -140,6 +151,13 @@ func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, run
|
||||||
}
|
}
|
||||||
return wrapError(err)
|
return wrapError(err)
|
||||||
}
|
}
|
||||||
|
if c.useShimV2 {
|
||||||
|
if x, ok := runtimeOptions.(*v2runcoptions.Options); ok {
|
||||||
|
c.v2runcoptionsMu.Lock()
|
||||||
|
c.v2runcoptions[id] = *x
|
||||||
|
c.v2runcoptionsMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -200,11 +218,26 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin
|
||||||
|
|
||||||
if runtime.GOOS != "windows" {
|
if runtime.GOOS != "windows" {
|
||||||
taskOpts = append(taskOpts, func(_ context.Context, _ *containerd.Client, info *containerd.TaskInfo) error {
|
taskOpts = append(taskOpts, func(_ context.Context, _ *containerd.Client, info *containerd.TaskInfo) error {
|
||||||
info.Options = &runctypes.CreateOptions{
|
if c.useShimV2 {
|
||||||
IoUid: uint32(uid),
|
// For v2, we need to inherit options specified on Create
|
||||||
IoGid: uint32(gid),
|
c.v2runcoptionsMu.Lock()
|
||||||
NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "",
|
opts, ok := c.v2runcoptions[id]
|
||||||
|
c.v2runcoptionsMu.Unlock()
|
||||||
|
if !ok {
|
||||||
|
opts = v2runcoptions.Options{}
|
||||||
|
}
|
||||||
|
opts.IoUid = uint32(uid)
|
||||||
|
opts.IoGid = uint32(gid)
|
||||||
|
opts.NoPivotRoot = os.Getenv("DOCKER_RAMDISK") != ""
|
||||||
|
info.Options = &opts
|
||||||
|
} else {
|
||||||
|
info.Options = &runctypes.CreateOptions{
|
||||||
|
IoUid: uint32(uid),
|
||||||
|
IoGid: uint32(gid),
|
||||||
|
NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "",
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
|
@ -466,6 +499,9 @@ func (c *client) Delete(ctx context.Context, containerID string) error {
|
||||||
c.oomMu.Lock()
|
c.oomMu.Lock()
|
||||||
delete(c.oom, containerID)
|
delete(c.oom, containerID)
|
||||||
c.oomMu.Unlock()
|
c.oomMu.Unlock()
|
||||||
|
c.v2runcoptionsMu.Lock()
|
||||||
|
delete(c.v2runcoptions, containerID)
|
||||||
|
c.v2runcoptionsMu.Unlock()
|
||||||
if os.Getenv("LIBCONTAINERD_NOCLEAN") != "1" {
|
if os.Getenv("LIBCONTAINERD_NOCLEAN") != "1" {
|
||||||
if err := os.RemoveAll(bundle); err != nil {
|
if err := os.RemoveAll(bundle); err != nil {
|
||||||
c.logger.WithError(err).WithFields(logrus.Fields{
|
c.logger.WithError(err).WithFields(logrus.Fields{
|
||||||
|
|
|
@ -16,7 +16,10 @@ import (
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
const runtimeName = "io.containerd.runtime.v1.linux"
|
const (
|
||||||
|
runtimeName = "io.containerd.runtime.v1.linux"
|
||||||
|
shimV2RuntimeName = "io.containerd.runc.v2"
|
||||||
|
)
|
||||||
|
|
||||||
func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) {
|
func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) {
|
||||||
return &libcontainerdtypes.Summary{}, nil
|
return &libcontainerdtypes.Summary{}, nil
|
||||||
|
|
|
@ -16,7 +16,10 @@ import (
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
const runtimeName = "io.containerd.runhcs.v1"
|
const (
|
||||||
|
runtimeName = "io.containerd.runhcs.v1"
|
||||||
|
shimV2RuntimeName = runtimeName
|
||||||
|
)
|
||||||
|
|
||||||
func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) {
|
func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) {
|
||||||
switch pd := i.(type) {
|
switch pd := i.(type) {
|
||||||
|
|
|
@ -26,13 +26,13 @@ type ExitHandler interface {
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new containerd plugin executor
|
// New creates a new containerd plugin executor
|
||||||
func New(ctx context.Context, rootDir string, cli *containerd.Client, ns string, exitHandler ExitHandler) (*Executor, error) {
|
func New(ctx context.Context, rootDir string, cli *containerd.Client, ns string, exitHandler ExitHandler, useShimV2 bool) (*Executor, error) {
|
||||||
e := &Executor{
|
e := &Executor{
|
||||||
rootDir: rootDir,
|
rootDir: rootDir,
|
||||||
exitHandler: exitHandler,
|
exitHandler: exitHandler,
|
||||||
}
|
}
|
||||||
|
|
||||||
client, err := libcontainerd.NewClient(ctx, cli, rootDir, ns, e)
|
client, err := libcontainerd.NewClient(ctx, cli, rootDir, ns, e, useShimV2)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Wrap(err, "error creating containerd exec client")
|
return nil, errors.Wrap(err, "error creating containerd exec client")
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue