Windows: Experimental: Allow containerd for runtime

Signed-off-by: John Howard <jhoward@microsoft.com>

This is the first step in refactoring moby (dockerd) to use containerd on Windows.
Similar to the current model in Linux, this adds the option to enable it for runtime.
It does not switch the graphdriver to containerd snapshotters.

 - Refactors libcontainerd to a series of subpackages so that either a
  "local" containerd (1) or a "remote" (2) containerd can be loaded as opposed
  to conditional compile as "local" for Windows and "remote" for Linux.

 - Updates libcontainerd such that Windows has an option to allow the use of a
   "remote" containerd. Here, it communicates over a named pipe using GRPC.
   This is currently guarded behind the experimental flag, an environment variable,
   and the providing of a pipename to connect to containerd.

 - Infrastructure pieces such as under pkg/system to have helper functions for
   determining whether containerd is being used.

(1) "local" containerd is what the daemon on Windows has used since inception.
It's not really containerd at all - it's simply local invocation of HCS APIs
directly in-process from the daemon through the Microsoft/hcsshim library.

(2) "remote" containerd is what docker on Linux uses for it's runtime. It means
that there is a separate containerd service running, and docker communicates over
GRPC to it.

To try this out, you will need to start with something like the following:

Window 1:
	containerd --log-level debug

Window 2:
	$env:DOCKER_WINDOWS_CONTAINERD=1
	dockerd --experimental -D --containerd \\.\pipe\containerd-containerd

You will need the following binary from github.com/containerd/containerd in your path:
 - containerd.exe

You will need the following binaries from github.com/Microsoft/hcsshim in your path:
 - runhcs.exe
 - containerd-shim-runhcs-v1.exe

For LCOW, it will require and initrd.img and kernel in `C:\Program Files\Linux Containers`.
This is no different to the current requirements. However, you may need updated binaries,
particularly initrd.img built from Microsoft/opengcs as (at the time of writing), Linuxkit
binaries are somewhat out of date.

Note that containerd and hcsshim for HCS v2 APIs do not yet support all the required
functionality needed for docker. This will come in time - this is a baby (although large)
step to migrating Docker on Windows to containerd.

Note that the HCS v2 APIs are only called on RS5+ builds. RS1..RS4 will still use
HCS v1 APIs as the v2 APIs were not fully developed enough on these builds to be usable.
This abstraction is done in HCSShim. (Referring specifically to runtime)

Note the LCOW graphdriver still uses HCS v1 APIs regardless.

Note also that this does not migrate docker to use containerd snapshotters
rather than graphdrivers. This needs to be done in conjunction with Linux also
doing the same switch.

cmd/dockerd/daemon.go

@@ -163,31 +163,9 @@ func (cli *DaemonCli) start(opts *daemonOptions) (err error) {
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
-	if cli.Config.ContainerdAddr == "" && runtime.GOOS != "windows" {
+	if err := cli.initContainerD(ctx); err != nil {
-		systemContainerdAddr, ok, err := systemContainerdRunning(cli.Config.IsRootless())
+		cancel()
-		if err != nil {
+		return err
-			cancel()
-			return errors.Wrap(err, "could not determine whether the system containerd is running")
-		}
-		if !ok {
-			opts, err := cli.getContainerdDaemonOpts()
-			if err != nil {
-				cancel()
-				return errors.Wrap(err, "failed to generate containerd options")
-			}
-
-			r, err := supervisor.Start(ctx, filepath.Join(cli.Config.Root, "containerd"), filepath.Join(cli.Config.ExecRoot, "containerd"), opts...)
-			if err != nil {
-				cancel()
-				return errors.Wrap(err, "failed to start containerd")
-			}
-			cli.Config.ContainerdAddr = r.Address()
-
-			// Try to wait for containerd to shutdown
-			defer r.WaitTimeout(10 * time.Second)
-		} else {
-			cli.Config.ContainerdAddr = systemContainerdAddr
-		}
 	}
 	defer cancel()
 

cmd/dockerd/daemon_unix.go

@@ -3,12 +3,14 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"os"
 	"os/signal"
 	"path/filepath"
 	"strconv"
+	"time"
 
 	"github.com/containerd/containerd/runtime/v1/linux"
 	"github.com/docker/docker/cmd/dockerd/hack"
@@ -18,6 +20,7 @@ import (
 	"github.com/docker/docker/pkg/homedir"
 	"github.com/docker/docker/rootless"
 	"github.com/docker/libnetwork/portallocator"
+	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
 )
 
@@ -145,3 +148,31 @@ func newCgroupParent(config *config.Config) string {
 	}
 	return cgroupParent
 }
+
+func (cli *DaemonCli) initContainerD(ctx context.Context) error {
+	if cli.Config.ContainerdAddr == "" {
+		systemContainerdAddr, ok, err := systemContainerdRunning(cli.Config.IsRootless())
+		if err != nil {
+			return errors.Wrap(err, "could not determine whether the system containerd is running")
+		}
+		if !ok {
+			opts, err := cli.getContainerdDaemonOpts()
+			if err != nil {
+				return errors.Wrap(err, "failed to generate containerd options")
+			}
+
+			r, err := supervisor.Start(ctx, filepath.Join(cli.Config.Root, "containerd"), filepath.Join(cli.Config.ExecRoot, "containerd"), opts...)
+			if err != nil {
+				return errors.Wrap(err, "failed to start containerd")
+			}
+			cli.Config.ContainerdAddr = r.Address()
+
+			// Try to wait for containerd to shutdown
+			defer r.WaitTimeout(10 * time.Second)
+		} else {
+			cli.Config.ContainerdAddr = systemContainerdAddr
+		}
+	}
+
+	return nil
+}

cmd/dockerd/daemon_windows.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"os"
@@ -8,6 +9,7 @@ import (
 
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/libcontainerd/supervisor"
+	"github.com/docker/docker/pkg/system"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows"
 )
@@ -90,3 +92,8 @@ func wrapListeners(proto string, ls []net.Listener) []net.Listener {
 func newCgroupParent(config *config.Config) string {
 	return ""
 }
+
+func (cli *DaemonCli) initContainerD(_ context.Context) error {
+	system.InitContainerdRuntime(cli.Config.Experimental, cli.Config.ContainerdAddr)
+	return nil
+}

daemon/daemon.go

@@ -42,6 +42,7 @@ import (
 	"github.com/moby/buildkit/util/resolver"
 	"github.com/moby/buildkit/util/tracing"
 	"github.com/sirupsen/logrus"
+
 	// register graph drivers
 	_ "github.com/docker/docker/daemon/graphdriver/register"
 	"github.com/docker/docker/daemon/stats"
@@ -50,6 +51,7 @@ import (
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
 	"github.com/docker/docker/libcontainerd"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/locker"
 	"github.com/docker/docker/pkg/plugingetter"
@@ -105,7 +107,7 @@ type Daemon struct {
 	pluginManager         *plugin.Manager
 	linkIndex             *linkIndex
 	containerdCli         *containerd.Client
-	containerd            libcontainerd.Client
+	containerd            libcontainerdtypes.Client
 	defaultIsolation      containertypes.Isolation // Default isolation mode on Windows
 	clusterProvider       cluster.Provider
 	cluster               Cluster
@@ -351,11 +353,11 @@ func (daemon *Daemon) restore() error {
 						logrus.WithField("container", c.ID).WithField("state", s).
 							Info("restored container paused")
 						switch s {
-						case libcontainerd.StatusPaused, libcontainerd.StatusPausing:
+						case libcontainerdtypes.StatusPaused, libcontainerdtypes.StatusPausing:
 							// nothing to do
-						case libcontainerd.StatusStopped:
+						case libcontainerdtypes.StatusStopped:
 							alive = false
-						case libcontainerd.StatusUnknown:
+						case libcontainerdtypes.StatusUnknown:
 							logrus.WithField("container", c.ID).
 								Error("Unknown status for container during restore")
 						default:

daemon/daemon_windows.go

@@ -502,6 +502,7 @@ func (daemon *Daemon) runAsHyperVContainer(hostConfig *containertypes.HostConfig
 // conditionalMountOnStart is a platform specific helper function during the
 // container start to call mount.
 func (daemon *Daemon) conditionalMountOnStart(container *container.Container) error {
+
 	// Bail out now for Linux containers. We cannot mount the containers filesystem on the
 	// host as it is a non-Windows filesystem.
 	if system.LCOWSupported() && container.OS != "windows" {
@@ -519,6 +520,7 @@ func (daemon *Daemon) conditionalMountOnStart(container *container.Container) er
 // conditionalUnmountOnCleanup is a platform specific helper function called
 // during the cleanup of a container to unmount.
 func (daemon *Daemon) conditionalUnmountOnCleanup(container *container.Container) error {
+
 	// Bail out now for Linux containers
 	if system.LCOWSupported() && container.OS != "windows" {
 		return nil

daemon/kill.go

@@ -9,7 +9,7 @@ import (
 
 	containerpkg "github.com/docker/docker/container"
 	"github.com/docker/docker/errdefs"
-	"github.com/docker/docker/libcontainerd"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
 	"github.com/docker/docker/pkg/signal"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -177,5 +177,5 @@ func (daemon *Daemon) killPossiblyDeadProcess(container *containerpkg.Container,
 }
 
 func (daemon *Daemon) kill(c *containerpkg.Container, sig int) error {
-	return daemon.containerd.SignalProcess(context.Background(), c.ID, libcontainerd.InitProcessName, sig)
+	return daemon.containerd.SignalProcess(context.Background(), c.ID, libcontainerdtypes.InitProcessName, sig)
 }

daemon/monitor.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/container"
-	"github.com/docker/docker/libcontainerd"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
 	"github.com/docker/docker/restartmanager"
 	"github.com/sirupsen/logrus"
 )
@@ -27,14 +27,14 @@ func (daemon *Daemon) setStateCounter(c *container.Container) {
 }
 
 // ProcessEvent is called by libcontainerd whenever an event occurs
-func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libcontainerd.EventInfo) error {
+func (daemon *Daemon) ProcessEvent(id string, e libcontainerdtypes.EventType, ei libcontainerdtypes.EventInfo) error {
 	c, err := daemon.GetContainer(id)
 	if c == nil || err != nil {
 		return fmt.Errorf("no such container: %s", id)
 	}
 
 	switch e {
-	case libcontainerd.EventOOM:
+	case libcontainerdtypes.EventOOM:
 		// StateOOM is Linux specific and should never be hit on Windows
 		if runtime.GOOS == "windows" {
 			return errors.New("received StateOOM from libcontainerd on Windows. This should never happen")
@@ -48,7 +48,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libc
 		}
 
 		daemon.LogContainerEvent(c, "oom")
-	case libcontainerd.EventExit:
+	case libcontainerdtypes.EventExit:
 		if int(ei.Pid) == c.Pid {
 			c.Lock()
 			_, _, err := daemon.containerd.DeleteTask(context.Background(), c.ID)
@@ -140,7 +140,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libc
 				"exec-pid":  ei.Pid,
 			}).Warn("Ignoring Exit Event, no such exec command found")
 		}
-	case libcontainerd.EventStart:
+	case libcontainerdtypes.EventStart:
 		c.Lock()
 		defer c.Unlock()
 
@@ -159,7 +159,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libc
 			daemon.LogContainerEvent(c, "start")
 		}
 
-	case libcontainerd.EventPaused:
+	case libcontainerdtypes.EventPaused:
 		c.Lock()
 		defer c.Unlock()
 
@@ -172,7 +172,7 @@ func (daemon *Daemon) ProcessEvent(id string, e libcontainerd.EventType, ei libc
 			}
 			daemon.LogContainerEvent(c, "pause")
 		}
-	case libcontainerd.EventResumed:
+	case libcontainerdtypes.EventResumed:
 		c.Lock()
 		defer c.Unlock()
 

daemon/oci_windows.go

@@ -1,6 +1,7 @@
 package daemon // import "github.com/docker/docker/daemon"
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"path/filepath"
@@ -15,7 +16,7 @@ import (
 	"github.com/docker/docker/pkg/system"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
-	"golang.org/x/sys/windows"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows/registry"
 )
 
@@ -25,6 +26,7 @@ const (
 )
 
 func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
+
 	img, err := daemon.imageService.GetImage(string(c.ImageID))
 	if err != nil {
 		return nil, err
@@ -219,11 +221,18 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
 		return nil, fmt.Errorf("Unsupported platform %q", img.OS)
 	}
 
+	if logrus.IsLevelEnabled(logrus.DebugLevel) {
+		if b, err := json.Marshal(&s); err == nil {
+			logrus.Debugf("Generated spec: %s", string(b))
+		}
+	}
+
 	return (*specs.Spec)(&s), nil
 }
 
 // Sets the Windows-specific fields of the OCI spec
 func (daemon *Daemon) createSpecWindowsFields(c *container.Container, s *specs.Spec, isHyperV bool) error {
+
 	if len(s.Process.Cwd) == 0 {
 		// We default to C:\ to workaround the oddity of the case that the
 		// default directory for cmd running as LocalSystem (or
@@ -396,31 +405,39 @@ func setResourcesInSpec(c *container.Container, s *specs.Spec, isHyperV bool) {
 			}
 		}
 	}
-	memoryLimit := uint64(c.HostConfig.Memory)
+
-	s.Windows.Resources = &specs.WindowsResources{
+	if cpuMaximum != 0 || cpuShares != 0 || cpuCount != 0 {
-		CPU: &specs.WindowsCPUResources{
+		if s.Windows.Resources == nil {
+			s.Windows.Resources = &specs.WindowsResources{}
+		}
+		s.Windows.Resources.CPU = &specs.WindowsCPUResources{
 			Maximum: &cpuMaximum,
 			Shares:  &cpuShares,
 			Count:   &cpuCount,
-		},
+		}
-		Memory: &specs.WindowsMemoryResources{
+	}
+
+	memoryLimit := uint64(c.HostConfig.Memory)
+	if memoryLimit != 0 {
+		if s.Windows.Resources == nil {
+			s.Windows.Resources = &specs.WindowsResources{}
+		}
+		s.Windows.Resources.Memory = &specs.WindowsMemoryResources{
 			Limit: &memoryLimit,
-		},
+		}
-		Storage: &specs.WindowsStorageResources{
+	}
+
+	if c.HostConfig.IOMaximumBandwidth != 0 || c.HostConfig.IOMaximumIOps != 0 {
+		if s.Windows.Resources == nil {
+			s.Windows.Resources = &specs.WindowsResources{}
+		}
+		s.Windows.Resources.Storage = &specs.WindowsStorageResources{
 			Bps:  &c.HostConfig.IOMaximumBandwidth,
 			Iops: &c.HostConfig.IOMaximumIOps,
-		},
+		}
 	}
 }
 
-func escapeArgs(args []string) []string {
-	escapedArgs := make([]string, len(args))
-	for i, a := range args {
-		escapedArgs[i] = windows.EscapeArg(a)
-	}
-	return escapedArgs
-}
-
 // mergeUlimits merge the Ulimits from HostConfig with daemon defaults, and update HostConfig
 // It will do nothing on non-Linux platform
 func (daemon *Daemon) mergeUlimits(c *containertypes.HostConfig) {

daemon/resize.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/docker/docker/libcontainerd"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
 )
 
 // ContainerResize changes the size of the TTY of the process running
@@ -20,7 +20,7 @@ func (daemon *Daemon) ContainerResize(name string, height, width int) error {
 		return errNotRunning(container.ID)
 	}
 
-	if err = daemon.containerd.ResizeTerminal(context.Background(), container.ID, libcontainerd.InitProcessName, width, height); err == nil {
+	if err = daemon.containerd.ResizeTerminal(context.Background(), container.ID, libcontainerdtypes.InitProcessName, width, height); err == nil {
 		attributes := map[string]string{
 			"height": fmt.Sprintf("%d", height),
 			"width":  fmt.Sprintf("%d", width),

daemon/start_windows.go

@@ -1,11 +1,23 @@
 package daemon // import "github.com/docker/docker/daemon"
 
 import (
+	"github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
 	"github.com/Microsoft/opengcs/client"
 	"github.com/docker/docker/container"
+	"github.com/docker/docker/pkg/system"
 )
 
 func (daemon *Daemon) getLibcontainerdCreateOptions(container *container.Container) (interface{}, error) {
+
+	// Set the runtime options to debug regardless of current logging level.
+	if system.ContainerdRuntimeSupported() {
+		opts := &options.Options{Debug: true}
+		return opts, nil
+	}
+
+	// TODO @jhowardmsft (containerd) - Probably need to revisit LCOW options here
+	// rather than blindly ignoring them.
+
 	// LCOW options.
 	if container.OS == "linux" {
 		config := &client.Config{}

daemon/update_linux.go

@@ -4,12 +4,12 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/libcontainerd"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
 	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
-func toContainerdResources(resources container.Resources) *libcontainerd.Resources {
+func toContainerdResources(resources container.Resources) *libcontainerdtypes.Resources {
-	var r libcontainerd.Resources
+	var r libcontainerdtypes.Resources
 
 	r.BlockIO = &specs.LinuxBlockIO{
 		Weight: &resources.BlkioWeight,

daemon/update_windows.go

@@ -2,10 +2,10 @@ package daemon // import "github.com/docker/docker/daemon"
 
 import (
 	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/libcontainerd"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
 )
 
-func toContainerdResources(resources container.Resources) *libcontainerd.Resources {
+func toContainerdResources(resources container.Resources) *libcontainerdtypes.Resources {
 	// We don't support update, so do nothing
 	return nil
 }

daemon/util_test.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/containerd/containerd"
-	"github.com/docker/docker/libcontainerd"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
@@ -18,19 +18,19 @@ type MockContainerdClient struct {
 func (c *MockContainerdClient) Version(ctx context.Context) (containerd.Version, error) {
 	return containerd.Version{}, nil
 }
-func (c *MockContainerdClient) Restore(ctx context.Context, containerID string, attachStdio libcontainerd.StdioCallback) (alive bool, pid int, err error) {
+func (c *MockContainerdClient) Restore(ctx context.Context, containerID string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, err error) {
 	return false, 0, nil
 }
 func (c *MockContainerdClient) Create(ctx context.Context, containerID string, spec *specs.Spec, runtimeOptions interface{}) error {
 	return nil
 }
-func (c *MockContainerdClient) Start(ctx context.Context, containerID, checkpointDir string, withStdin bool, attachStdio libcontainerd.StdioCallback) (pid int, err error) {
+func (c *MockContainerdClient) Start(ctx context.Context, containerID, checkpointDir string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (pid int, err error) {
 	return 0, nil
 }
 func (c *MockContainerdClient) SignalProcess(ctx context.Context, containerID, processID string, signal int) error {
 	return nil
 }
-func (c *MockContainerdClient) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio libcontainerd.StdioCallback) (int, error) {
+func (c *MockContainerdClient) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) {
 	return 0, nil
 }
 func (c *MockContainerdClient) ResizeTerminal(ctx context.Context, containerID, processID string, width, height int) error {
@@ -41,23 +41,23 @@ func (c *MockContainerdClient) CloseStdin(ctx context.Context, containerID, proc
 }
 func (c *MockContainerdClient) Pause(ctx context.Context, containerID string) error  { return nil }
 func (c *MockContainerdClient) Resume(ctx context.Context, containerID string) error { return nil }
-func (c *MockContainerdClient) Stats(ctx context.Context, containerID string) (*libcontainerd.Stats, error) {
+func (c *MockContainerdClient) Stats(ctx context.Context, containerID string) (*libcontainerdtypes.Stats, error) {
 	return nil, nil
 }
 func (c *MockContainerdClient) ListPids(ctx context.Context, containerID string) ([]uint32, error) {
 	return nil, nil
 }
-func (c *MockContainerdClient) Summary(ctx context.Context, containerID string) ([]libcontainerd.Summary, error) {
+func (c *MockContainerdClient) Summary(ctx context.Context, containerID string) ([]libcontainerdtypes.Summary, error) {
 	return nil, nil
 }
 func (c *MockContainerdClient) DeleteTask(ctx context.Context, containerID string) (uint32, time.Time, error) {
 	return 0, time.Time{}, nil
 }
 func (c *MockContainerdClient) Delete(ctx context.Context, containerID string) error { return nil }
-func (c *MockContainerdClient) Status(ctx context.Context, containerID string) (libcontainerd.Status, error) {
+func (c *MockContainerdClient) Status(ctx context.Context, containerID string) (libcontainerdtypes.Status, error) {
 	return "null", nil
 }
-func (c *MockContainerdClient) UpdateResources(ctx context.Context, containerID string, resources *libcontainerd.Resources) error {
+func (c *MockContainerdClient) UpdateResources(ctx context.Context, containerID string, resources *libcontainerdtypes.Resources) error {
 	return nil
 }
 func (c *MockContainerdClient) CreateCheckpoint(ctx context.Context, containerID, checkpointDir string, exit bool) error {

libcontainerd/errors.go

@@ -1,13 +0,0 @@
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
-
-import (
-	"errors"
-
-	"github.com/docker/docker/errdefs"
-)
-
-func newNotFoundError(err string) error { return errdefs.NotFound(errors.New(err)) }
-
-func newInvalidParameterError(err string) error { return errdefs.InvalidParameter(errors.New(err)) }
-
-func newConflictError(err string) error { return errdefs.Conflict(errors.New(err)) }

libcontainerd/libcontainerd_linux.go

@@ -0,0 +1,14 @@
+package libcontainerd // import "github.com/docker/docker/libcontainerd"
+
+import (
+	"context"
+
+	"github.com/containerd/containerd"
+	"github.com/docker/docker/libcontainerd/remote"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
+)
+
+// NewClient creates a new libcontainerd client from a containerd client
+func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) {
+	return remote.NewClient(ctx, cli, stateDir, ns, b)
+}

libcontainerd/libcontainerd_windows.go

@@ -0,0 +1,19 @@
+package libcontainerd // import "github.com/docker/docker/libcontainerd"
+
+import (
+	"context"
+
+	"github.com/containerd/containerd"
+	"github.com/docker/docker/libcontainerd/local"
+	"github.com/docker/docker/libcontainerd/remote"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
+	"github.com/docker/docker/pkg/system"
+)
+
+// NewClient creates a new libcontainerd client from a containerd client
+func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) {
+	if !system.ContainerdRuntimeSupported() {
+		return local.NewClient(ctx, cli, stateDir, ns, b)
+	}
+	return remote.NewClient(ctx, cli, stateDir, ns, b)
+}

libcontainerd/local/process_windows.go

@@ -1,4 +1,4 @@
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
+package local // import "github.com/docker/docker/libcontainerd/local"
 
 import (
 	"io"

libcontainerd/local/utils_windows.go

@@ -1,4 +1,4 @@
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
+package local // import "github.com/docker/docker/libcontainerd/local"
 
 import (
 	"strings"
@@ -19,6 +19,11 @@ func setupEnvironmentVariables(a []string) map[string]string {
 	return r
 }
 
+// LCOWOption is a CreateOption required for LCOW configuration
+type LCOWOption struct {
+	Config *opengcs.Config
+}
+
 // Apply for the LCOW option is a no-op.
 func (s *LCOWOption) Apply(interface{}) error {
 	return nil

libcontainerd/local/utils_windows_test.go

@@ -1,4 +1,4 @@
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
+package local // import "github.com/docker/docker/libcontainerd/local"
 
 import (
 	"testing"

libcontainerd/local/windows.go

@@ -1,4 +1,7 @@
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
+package local // import "github.com/docker/docker/libcontainerd/local"
+
+// This package contains the legacy in-proc calls in HCS using the v1 schema
+// for Windows runtime purposes.
 
 import (
 	"context"
@@ -18,6 +21,10 @@ import (
 	opengcs "github.com/Microsoft/opengcs/client"
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/cio"
+
+	"github.com/docker/docker/errdefs"
+	"github.com/docker/docker/libcontainerd/queue"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
 	"github.com/docker/docker/pkg/sysinfo"
 	"github.com/docker/docker/pkg/system"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -26,8 +33,6 @@ import (
 	"golang.org/x/sys/windows"
 )
 
-const InitProcessName = "init"
-
 type process struct {
 	id         string
 	pid        int
@@ -46,7 +51,7 @@ type container struct {
 	hcsContainer hcsshim.Container
 
 	id               string
-	status           Status
+	status           libcontainerdtypes.Status
 	exitedAt         time.Time
 	exitCode         uint32
 	waitCh           chan struct{}
@@ -74,14 +79,14 @@ type client struct {
 	sync.Mutex
 
 	stateDir   string
-	backend    Backend
+	backend    libcontainerdtypes.Backend
 	logger     *logrus.Entry
-	eventQ     queue
+	eventQ     queue.Queue
 	containers map[string]*container
 }
 
 // NewClient creates a new local executor for windows
-func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b Backend) (Client, error) {
+func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) {
 	c := &client{
 		stateDir:   stateDir,
 		backend:    b,
@@ -149,7 +154,7 @@ func (c *client) Version(ctx context.Context) (containerd.Version, error) {
 //}
 func (c *client) Create(_ context.Context, id string, spec *specs.Spec, runtimeOptions interface{}) error {
 	if ctr := c.getContainer(id); ctr != nil {
-		return errors.WithStack(newConflictError("id already in use"))
+		return errors.WithStack(errdefs.Conflict(errors.New("id already in use")))
 	}
 
 	// spec.Linux must be nil for Windows containers, but spec.Windows
@@ -328,7 +333,7 @@ func (c *client) createWindows(id string, spec *specs.Spec, runtimeOptions inter
 		isWindows:    true,
 		ociSpec:      spec,
 		hcsContainer: hcsContainer,
-		status:       StatusCreated,
+		status:       libcontainerdtypes.StatusCreated,
 		waitCh:       make(chan struct{}),
 	}
 
@@ -532,7 +537,7 @@ func (c *client) createLinux(id string, spec *specs.Spec, runtimeOptions interfa
 		isWindows:    false,
 		ociSpec:      spec,
 		hcsContainer: hcsContainer,
-		status:       StatusCreated,
+		status:       libcontainerdtypes.StatusCreated,
 		waitCh:       make(chan struct{}),
 	}
 
@@ -556,19 +561,19 @@ func (c *client) createLinux(id string, spec *specs.Spec, runtimeOptions interfa
 	c.containers[id] = ctr
 	c.Unlock()
 
-	c.eventQ.append(id, func() {
+	c.eventQ.Append(id, func() {
-		ei := EventInfo{
+		ei := libcontainerdtypes.EventInfo{
 			ContainerID: id,
 		}
 		c.logger.WithFields(logrus.Fields{
 			"container": ctr.id,
-			"event":     EventCreate,
+			"event":     libcontainerdtypes.EventCreate,
 		}).Info("sending event")
-		err := c.backend.ProcessEvent(id, EventCreate, ei)
+		err := c.backend.ProcessEvent(id, libcontainerdtypes.EventCreate, ei)
 		if err != nil {
 			c.logger.WithError(err).WithFields(logrus.Fields{
 				"container": id,
-				"event":     EventCreate,
+				"event":     libcontainerdtypes.EventCreate,
 			}).Error("failed to process event")
 		}
 	})
@@ -607,13 +612,13 @@ func (c *client) extractResourcesFromSpec(spec *specs.Spec, configuration *hcssh
 	}
 }
 
-func (c *client) Start(_ context.Context, id, _ string, withStdin bool, attachStdio StdioCallback) (int, error) {
+func (c *client) Start(_ context.Context, id, _ string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) {
 	ctr := c.getContainer(id)
 	switch {
 	case ctr == nil:
-		return -1, errors.WithStack(newNotFoundError("no such container"))
+		return -1, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
 	case ctr.init != nil:
-		return -1, errors.WithStack(newConflictError("container already started"))
+		return -1, errors.WithStack(errdefs.Conflict(errors.New("container already started")))
 	}
 
 	logger := c.logger.WithField("container", id)
@@ -691,7 +696,7 @@ func (c *client) Start(_ context.Context, id, _ string, withStdin bool, attachSt
 	}()
 	p := &process{
 		hcsProcess: newProcess,
-		id:         InitProcessName,
+		id:         libcontainerdtypes.InitProcessName,
 		pid:        newProcess.Pid(),
 	}
 	logger.WithField("pid", p.pid).Debug("init process started")
@@ -706,29 +711,29 @@ func (c *client) Start(_ context.Context, id, _ string, withStdin bool, attachSt
 		logger.WithError(err).Error("failed to attache stdio")
 		return -1, err
 	}
-	ctr.status = StatusRunning
+	ctr.status = libcontainerdtypes.StatusRunning
 	ctr.init = p
 
 	// Spin up a go routine waiting for exit to handle cleanup
 	go c.reapProcess(ctr, p)
 
 	// Generate the associated event
-	c.eventQ.append(id, func() {
+	c.eventQ.Append(id, func() {
-		ei := EventInfo{
+		ei := libcontainerdtypes.EventInfo{
 			ContainerID: id,
-			ProcessID:   InitProcessName,
+			ProcessID:   libcontainerdtypes.InitProcessName,
 			Pid:         uint32(p.pid),
 		}
 		c.logger.WithFields(logrus.Fields{
 			"container":  ctr.id,
-			"event":      EventStart,
+			"event":      libcontainerdtypes.EventStart,
 			"event-info": ei,
 		}).Info("sending event")
-		err := c.backend.ProcessEvent(ei.ContainerID, EventStart, ei)
+		err := c.backend.ProcessEvent(ei.ContainerID, libcontainerdtypes.EventStart, ei)
 		if err != nil {
 			c.logger.WithError(err).WithFields(logrus.Fields{
 				"container":  id,
-				"event":      EventStart,
+				"event":      libcontainerdtypes.EventStart,
 				"event-info": ei,
 			}).Error("failed to process event")
 		}
@@ -756,15 +761,15 @@ func newIOFromProcess(newProcess hcsshim.Process, terminal bool) (*cio.DirectIO,
 }
 
 // Exec adds a process in an running container
-func (c *client) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio StdioCallback) (int, error) {
+func (c *client) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) {
 	ctr := c.getContainer(containerID)
 	switch {
 	case ctr == nil:
-		return -1, errors.WithStack(newNotFoundError("no such container"))
+		return -1, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
 	case ctr.hcsContainer == nil:
-		return -1, errors.WithStack(newInvalidParameterError("container is not running"))
+		return -1, errors.WithStack(errdefs.InvalidParameter(errors.New("container is not running")))
 	case ctr.execs != nil && ctr.execs[processID] != nil:
-		return -1, errors.WithStack(newConflictError("id already in use"))
+		return -1, errors.WithStack(errdefs.Conflict(errors.New("id already in use")))
 	}
 	logger := c.logger.WithFields(logrus.Fields{
 		"container": containerID,
@@ -856,30 +861,30 @@ func (c *client) Exec(ctx context.Context, containerID, processID string, spec *
 	// Spin up a go routine waiting for exit to handle cleanup
 	go c.reapProcess(ctr, p)
 
-	c.eventQ.append(ctr.id, func() {
+	c.eventQ.Append(ctr.id, func() {
-		ei := EventInfo{
+		ei := libcontainerdtypes.EventInfo{
 			ContainerID: ctr.id,
 			ProcessID:   p.id,
 			Pid:         uint32(p.pid),
 		}
 		c.logger.WithFields(logrus.Fields{
 			"container":  ctr.id,
-			"event":      EventExecAdded,
+			"event":      libcontainerdtypes.EventExecAdded,
 			"event-info": ei,
 		}).Info("sending event")
-		err := c.backend.ProcessEvent(ctr.id, EventExecAdded, ei)
+		err := c.backend.ProcessEvent(ctr.id, libcontainerdtypes.EventExecAdded, ei)
 		if err != nil {
 			c.logger.WithError(err).WithFields(logrus.Fields{
 				"container":  ctr.id,
-				"event":      EventExecAdded,
+				"event":      libcontainerdtypes.EventExecAdded,
 				"event-info": ei,
 			}).Error("failed to process event")
 		}
-		err = c.backend.ProcessEvent(ctr.id, EventExecStarted, ei)
+		err = c.backend.ProcessEvent(ctr.id, libcontainerdtypes.EventExecStarted, ei)
 		if err != nil {
 			c.logger.WithError(err).WithFields(logrus.Fields{
 				"container":  ctr.id,
-				"event":      EventExecStarted,
+				"event":      libcontainerdtypes.EventExecStarted,
 				"event-info": ei,
 			}).Error("failed to process event")
 		}
@@ -905,7 +910,7 @@ func (c *client) SignalProcess(_ context.Context, containerID, processID string,
 	})
 	logger.Debug("Signal()")
 
-	if processID == InitProcessName {
+	if processID == libcontainerdtypes.InitProcessName {
 		if syscall.Signal(signal) == syscall.SIGKILL {
 			// Terminate the compute system
 			ctr.Lock()
@@ -961,7 +966,7 @@ func (c *client) CloseStdin(_ context.Context, containerID, processID string) er
 
 // Pause handles pause requests for containers
 func (c *client) Pause(_ context.Context, containerID string) error {
-	ctr, _, err := c.getProcess(containerID, InitProcessName)
+	ctr, _, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
 	if err != nil {
 		return err
 	}
@@ -977,21 +982,21 @@ func (c *client) Pause(_ context.Context, containerID string) error {
 		return err
 	}
 
-	ctr.status = StatusPaused
+	ctr.status = libcontainerdtypes.StatusPaused
 
-	c.eventQ.append(containerID, func() {
+	c.eventQ.Append(containerID, func() {
-		err := c.backend.ProcessEvent(containerID, EventPaused, EventInfo{
+		err := c.backend.ProcessEvent(containerID, libcontainerdtypes.EventPaused, libcontainerdtypes.EventInfo{
 			ContainerID: containerID,
-			ProcessID:   InitProcessName,
+			ProcessID:   libcontainerdtypes.InitProcessName,
 		})
 		c.logger.WithFields(logrus.Fields{
 			"container": ctr.id,
-			"event":     EventPaused,
+			"event":     libcontainerdtypes.EventPaused,
 		}).Info("sending event")
 		if err != nil {
 			c.logger.WithError(err).WithFields(logrus.Fields{
 				"container": containerID,
-				"event":     EventPaused,
+				"event":     libcontainerdtypes.EventPaused,
 			}).Error("failed to process event")
 		}
 	})
@@ -1001,7 +1006,7 @@ func (c *client) Pause(_ context.Context, containerID string) error {
 
 // Resume handles resume requests for containers
 func (c *client) Resume(_ context.Context, containerID string) error {
-	ctr, _, err := c.getProcess(containerID, InitProcessName)
+	ctr, _, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
 	if err != nil {
 		return err
 	}
@@ -1017,21 +1022,21 @@ func (c *client) Resume(_ context.Context, containerID string) error {
 		return err
 	}
 
-	ctr.status = StatusRunning
+	ctr.status = libcontainerdtypes.StatusRunning
 
-	c.eventQ.append(containerID, func() {
+	c.eventQ.Append(containerID, func() {
-		err := c.backend.ProcessEvent(containerID, EventResumed, EventInfo{
+		err := c.backend.ProcessEvent(containerID, libcontainerdtypes.EventResumed, libcontainerdtypes.EventInfo{
 			ContainerID: containerID,
-			ProcessID:   InitProcessName,
+			ProcessID:   libcontainerdtypes.InitProcessName,
 		})
 		c.logger.WithFields(logrus.Fields{
 			"container": ctr.id,
-			"event":     EventResumed,
+			"event":     libcontainerdtypes.EventResumed,
 		}).Info("sending event")
 		if err != nil {
 			c.logger.WithError(err).WithFields(logrus.Fields{
 				"container": containerID,
-				"event":     EventResumed,
+				"event":     libcontainerdtypes.EventResumed,
 			}).Error("failed to process event")
 		}
 	})
@@ -1040,8 +1045,8 @@ func (c *client) Resume(_ context.Context, containerID string) error {
 }
 
 // Stats handles stats requests for containers
-func (c *client) Stats(_ context.Context, containerID string) (*Stats, error) {
+func (c *client) Stats(_ context.Context, containerID string) (*libcontainerdtypes.Stats, error) {
-	ctr, _, err := c.getProcess(containerID, InitProcessName)
+	ctr, _, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
 	if err != nil {
 		return nil, err
 	}
@@ -1051,14 +1056,14 @@ func (c *client) Stats(_ context.Context, containerID string) (*Stats, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &Stats{
+	return &libcontainerdtypes.Stats{
 		Read:     readAt,
 		HCSStats: &s,
 	}, nil
 }
 
 // Restore is the handler for restoring a container
-func (c *client) Restore(ctx context.Context, id string, attachStdio StdioCallback) (bool, int, error) {
+func (c *client) Restore(ctx context.Context, id string, attachStdio libcontainerdtypes.StdioCallback) (bool, int, error) {
 	c.logger.WithField("container", id).Debug("restore()")
 
 	// TODO Windows: On RS1, a re-attach isn't possible.
@@ -1098,8 +1103,8 @@ func (c *client) ListPids(_ context.Context, _ string) ([]uint32, error) {
 // the containers could be Hyper-V containers, they would not be
 // visible on the container host. However, libcontainerd does have
 // that information.
-func (c *client) Summary(_ context.Context, containerID string) ([]Summary, error) {
+func (c *client) Summary(_ context.Context, containerID string) ([]libcontainerdtypes.Summary, error) {
-	ctr, _, err := c.getProcess(containerID, InitProcessName)
+	ctr, _, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
 	if err != nil {
 		return nil, err
 	}
@@ -1109,9 +1114,9 @@ func (c *client) Summary(_ context.Context, containerID string) ([]Summary, erro
 		return nil, err
 	}
 
-	pl := make([]Summary, len(p))
+	pl := make([]libcontainerdtypes.Summary, len(p))
 	for i := range p {
-		pl[i] = Summary(p[i])
+		pl[i] = libcontainerdtypes.Summary(p[i])
 	}
 	return pl, nil
 }
@@ -1120,7 +1125,7 @@ func (c *client) DeleteTask(ctx context.Context, containerID string) (uint32, ti
 	ec := -1
 	ctr := c.getContainer(containerID)
 	if ctr == nil {
-		return uint32(ec), time.Now(), errors.WithStack(newNotFoundError("no such container"))
+		return uint32(ec), time.Now(), errors.WithStack(errdefs.NotFound(errors.New("no such container")))
 	}
 
 	select {
@@ -1141,32 +1146,32 @@ func (c *client) Delete(_ context.Context, containerID string) error {
 	defer c.Unlock()
 	ctr := c.containers[containerID]
 	if ctr == nil {
-		return errors.WithStack(newNotFoundError("no such container"))
+		return errors.WithStack(errdefs.NotFound(errors.New("no such container")))
 	}
 
 	ctr.Lock()
 	defer ctr.Unlock()
 
 	switch ctr.status {
-	case StatusCreated:
+	case libcontainerdtypes.StatusCreated:
 		if err := c.shutdownContainer(ctr); err != nil {
 			return err
 		}
 		fallthrough
-	case StatusStopped:
+	case libcontainerdtypes.StatusStopped:
 		delete(c.containers, containerID)
 		return nil
 	}
 
-	return errors.WithStack(newInvalidParameterError("container is not stopped"))
+	return errors.WithStack(errdefs.InvalidParameter(errors.New("container is not stopped")))
 }
 
-func (c *client) Status(ctx context.Context, containerID string) (Status, error) {
+func (c *client) Status(ctx context.Context, containerID string) (libcontainerdtypes.Status, error) {
 	c.Lock()
 	defer c.Unlock()
 	ctr := c.containers[containerID]
 	if ctr == nil {
-		return StatusUnknown, errors.WithStack(newNotFoundError("no such container"))
+		return libcontainerdtypes.StatusUnknown, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
 	}
 
 	ctr.Lock()
@@ -1174,7 +1179,7 @@ func (c *client) Status(ctx context.Context, containerID string) (Status, error)
 	return ctr.status, nil
 }
 
-func (c *client) UpdateResources(ctx context.Context, containerID string, resources *Resources) error {
+func (c *client) UpdateResources(ctx context.Context, containerID string, resources *libcontainerdtypes.Resources) error {
 	// Updating resource isn't supported on Windows
 	// but we should return nil for enabling updating container
 	return nil
@@ -1196,22 +1201,22 @@ func (c *client) getProcess(containerID, processID string) (*container, *process
 	ctr := c.getContainer(containerID)
 	switch {
 	case ctr == nil:
-		return nil, nil, errors.WithStack(newNotFoundError("no such container"))
+		return nil, nil, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
 	case ctr.init == nil:
-		return nil, nil, errors.WithStack(newNotFoundError("container is not running"))
+		return nil, nil, errors.WithStack(errdefs.NotFound(errors.New("container is not running")))
-	case processID == InitProcessName:
+	case processID == libcontainerdtypes.InitProcessName:
 		return ctr, ctr.init, nil
 	default:
 		ctr.Lock()
 		defer ctr.Unlock()
 		if ctr.execs == nil {
-			return nil, nil, errors.WithStack(newNotFoundError("no execs"))
+			return nil, nil, errors.WithStack(errdefs.NotFound(errors.New("no execs")))
 		}
 	}
 
 	p := ctr.execs[processID]
 	if p == nil {
-		return nil, nil, errors.WithStack(newNotFoundError("no such exec"))
+		return nil, nil, errors.WithStack(errdefs.NotFound(errors.New("no such exec")))
 	}
 
 	return ctr, p, nil
@@ -1309,10 +1314,10 @@ func (c *client) reapProcess(ctr *container, p *process) int {
 		eventErr = fmt.Errorf("hcsProcess.Close() failed %s", err)
 	}
 
-	if p.id == InitProcessName {
+	if p.id == libcontainerdtypes.InitProcessName {
 		// Update container status
 		ctr.Lock()
-		ctr.status = StatusStopped
+		ctr.status = libcontainerdtypes.StatusStopped
 		ctr.exitedAt = exitedAt
 		ctr.exitCode = uint32(exitCode)
 		close(ctr.waitCh)
@@ -1343,8 +1348,8 @@ func (c *client) reapProcess(ctr *container, p *process) int {
 		}
 	}
 
-	c.eventQ.append(ctr.id, func() {
+	c.eventQ.Append(ctr.id, func() {
-		ei := EventInfo{
+		ei := libcontainerdtypes.EventInfo{
 			ContainerID: ctr.id,
 			ProcessID:   p.id,
 			Pid:         uint32(p.pid),
@@ -1354,18 +1359,18 @@ func (c *client) reapProcess(ctr *container, p *process) int {
 		}
 		c.logger.WithFields(logrus.Fields{
 			"container":  ctr.id,
-			"event":      EventExit,
+			"event":      libcontainerdtypes.EventExit,
 			"event-info": ei,
 		}).Info("sending event")
-		err := c.backend.ProcessEvent(ctr.id, EventExit, ei)
+		err := c.backend.ProcessEvent(ctr.id, libcontainerdtypes.EventExit, ei)
 		if err != nil {
 			c.logger.WithError(err).WithFields(logrus.Fields{
 				"container":  ctr.id,
-				"event":      EventExit,
+				"event":      libcontainerdtypes.EventExit,
 				"event-info": ei,
 			}).Error("failed to process event")
 		}
-		if p.id != InitProcessName {
+		if p.id != libcontainerdtypes.InitProcessName {
 			ctr.Lock()
 			delete(ctr.execs, p.id)
 			ctr.Unlock()

libcontainerd/queue/queue.go

@@ -1,13 +1,15 @@
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
+package queue // import "github.com/docker/docker/libcontainerd/queue"
 
 import "sync"
 
-type queue struct {
+// Queue is the structure used for holding functions in a queue.
+type Queue struct {
 	sync.Mutex
 	fns map[string]chan struct{}
 }
 
-func (q *queue) append(id string, f func()) {
+// Append adds an item to a queue.
+func (q *Queue) Append(id string, f func()) {
 	q.Lock()
 	defer q.Unlock()
 

libcontainerd/queue/queue_test.go

@@ -1,4 +1,4 @@
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
+package queue // import "github.com/docker/docker/libcontainerd/queue"
 
 import (
 	"testing"
@@ -9,21 +9,21 @@ import (
 
 func TestSerialization(t *testing.T) {
 	var (
-		q             queue
+		q             Queue
 		serialization = 1
 	)
 
-	q.append("aaa", func() {
+	q.Append("aaa", func() {
 		//simulate a long time task
 		time.Sleep(10 * time.Millisecond)
 		assert.Equal(t, serialization, 1)
 		serialization = 2
 	})
-	q.append("aaa", func() {
+	q.Append("aaa", func() {
 		assert.Equal(t, serialization, 2)
 		serialization = 3
 	})
-	q.append("aaa", func() {
+	q.Append("aaa", func() {
 		assert.Equal(t, serialization, 3)
 		serialization = 4
 	})

libcontainerd/remote/client.go

@@ -1,11 +1,8 @@
-// +build !windows
+package remote // import "github.com/docker/docker/libcontainerd/remote"
-
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
 
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -16,6 +13,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
 	"github.com/containerd/containerd"
 	apievents "github.com/containerd/containerd/api/events"
 	"github.com/containerd/containerd/api/types"
@@ -28,6 +26,9 @@ import (
 	"github.com/containerd/containerd/runtime/linux/runctypes"
 	"github.com/containerd/typeurl"
 	"github.com/docker/docker/errdefs"
+	"github.com/docker/docker/libcontainerd/queue"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
+
 	"github.com/docker/docker/pkg/ioutils"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -37,10 +38,6 @@ import (
 	"google.golang.org/grpc/status"
 )
 
-// InitProcessName is the name given to the first process of a
-// container
-const InitProcessName = "init"
-
 type container struct {
 	mu sync.Mutex
 
@@ -107,13 +104,13 @@ type client struct {
 	logger   *logrus.Entry
 	ns       string
 
-	backend    Backend
+	backend    libcontainerdtypes.Backend
-	eventQ     queue
+	eventQ     queue.Queue
 	containers map[string]*container
 }
 
 // NewClient creates a new libcontainerd client from a containerd client
-func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b Backend) (Client, error) {
+func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) {
 	c := &client{
 		client:     cli,
 		stateDir:   stateDir,
@@ -134,12 +131,12 @@ func (c *client) Version(ctx context.Context) (containerd.Version, error) {
 
 // Restore loads the containerd container.
 // It should not be called concurrently with any other operation for the given ID.
-func (c *client) Restore(ctx context.Context, id string, attachStdio StdioCallback) (alive bool, pid int, err error) {
+func (c *client) Restore(ctx context.Context, id string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, err error) {
 	c.Lock()
 	_, ok := c.containers[id]
 	if ok {
 		c.Unlock()
-		return false, 0, errors.WithStack(newConflictError("id already in use"))
+		return false, 0, errors.WithStack(errdefs.Conflict(errors.New("id already in use")))
 	}
 
 	cntr := &container{}
@@ -174,7 +171,8 @@ func (c *client) Restore(ctx context.Context, id string, attachStdio StdioCallba
 	attachIO := func(fifos *cio.FIFOSet) (cio.IO, error) {
 		// dio must be assigned to the previously defined dio for the defer above
 		// to handle cleanup
-		dio, err = cio.NewDirectIO(ctx, fifos)
+
+		dio, err = c.newDirectIO(ctx, fifos)
 		if err != nil {
 			return nil, err
 		}
@@ -211,7 +209,7 @@ func (c *client) Restore(ctx context.Context, id string, attachStdio StdioCallba
 
 func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, runtimeOptions interface{}) error {
 	if ctr := c.getContainer(id); ctr != nil {
-		return errors.WithStack(newConflictError("id already in use"))
+		return errors.WithStack(errdefs.Conflict(errors.New("id already in use")))
 	}
 
 	bdir, err := prepareBundleDir(filepath.Join(c.stateDir, id), ociSpec)
@@ -223,8 +221,7 @@ func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, run
 
 	cdCtr, err := c.client.NewContainer(ctx, id,
 		containerd.WithSpec(ociSpec),
-		// TODO(mlaventure): when containerd support lcow, revisit runtime value
+		containerd.WithRuntime(runtimeName, runtimeOptions))
-		containerd.WithRuntime(fmt.Sprintf("io.containerd.runtime.v1.%s", runtime.GOOS), runtimeOptions))
 	if err != nil {
 		return wrapError(err)
 	}
@@ -240,13 +237,13 @@ func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, run
 }
 
 // Start create and start a task for the specified containerd id
-func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio StdioCallback) (int, error) {
+func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) {
 	ctr := c.getContainer(id)
 	if ctr == nil {
-		return -1, errors.WithStack(newNotFoundError("no such container"))
+		return -1, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
 	}
 	if t := ctr.getTask(); t != nil {
-		return -1, errors.WithStack(newConflictError("container already started"))
+		return -1, errors.WithStack(errdefs.Conflict(errors.New("container already started")))
 	}
 
 	var (
@@ -288,17 +285,24 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin
 	uid, gid := getSpecUser(spec)
 	t, err = ctr.ctr.NewTask(ctx,
 		func(id string) (cio.IO, error) {
-			fifos := newFIFOSet(ctr.bundleDir, InitProcessName, withStdin, spec.Process.Terminal)
+			fifos := newFIFOSet(ctr.bundleDir, libcontainerdtypes.InitProcessName, withStdin, spec.Process.Terminal)
 
-			rio, err = c.createIO(fifos, id, InitProcessName, stdinCloseSync, attachStdio)
+			rio, err = c.createIO(fifos, id, libcontainerdtypes.InitProcessName, stdinCloseSync, attachStdio)
 			return rio, err
 		},
 		func(_ context.Context, _ *containerd.Client, info *containerd.TaskInfo) error {
 			info.Checkpoint = cp
-			info.Options = &runctypes.CreateOptions{
+			if runtime.GOOS != "windows" {
-				IoUid:       uint32(uid),
+				info.Options = &runctypes.CreateOptions{
-				IoGid:       uint32(gid),
+					IoUid:       uint32(uid),
-				NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "",
+					IoGid:       uint32(gid),
+					NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "",
+				}
+			} else {
+				// Make sure we set the runhcs options to debug if we are at debug level.
+				if c.logger.Level == logrus.DebugLevel {
+					info.Options = &options.Options{Debug: true}
+				}
 			}
 			return nil
 		})
@@ -335,18 +339,18 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin
 // for the container main process, the stdin fifo will be created in Create not
 // the Start call. stdinCloseSync channel should be closed after Start exec
 // process.
-func (c *client) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio StdioCallback) (int, error) {
+func (c *client) Exec(ctx context.Context, containerID, processID string, spec *specs.Process, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (int, error) {
 	ctr := c.getContainer(containerID)
 	if ctr == nil {
-		return -1, errors.WithStack(newNotFoundError("no such container"))
+		return -1, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
 	}
 	t := ctr.getTask()
 	if t == nil {
-		return -1, errors.WithStack(newInvalidParameterError("container is not running"))
+		return -1, errors.WithStack(errdefs.InvalidParameter(errors.New("container is not running")))
 	}
 
 	if p := ctr.getProcess(processID); p != nil {
-		return -1, errors.WithStack(newConflictError("id already in use"))
+		return -1, errors.WithStack(errdefs.Conflict(errors.New("id already in use")))
 	}
 
 	var (
@@ -424,7 +428,7 @@ func (c *client) CloseStdin(ctx context.Context, containerID, processID string)
 }
 
 func (c *client) Pause(ctx context.Context, containerID string) error {
-	p, err := c.getProcess(containerID, InitProcessName)
+	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
 	if err != nil {
 		return err
 	}
@@ -433,7 +437,7 @@ func (c *client) Pause(ctx context.Context, containerID string) error {
 }
 
 func (c *client) Resume(ctx context.Context, containerID string) error {
-	p, err := c.getProcess(containerID, InitProcessName)
+	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
 	if err != nil {
 		return err
 	}
@@ -441,8 +445,8 @@ func (c *client) Resume(ctx context.Context, containerID string) error {
 	return p.(containerd.Task).Resume(ctx)
 }
 
-func (c *client) Stats(ctx context.Context, containerID string) (*Stats, error) {
+func (c *client) Stats(ctx context.Context, containerID string) (*libcontainerdtypes.Stats, error) {
-	p, err := c.getProcess(containerID, InitProcessName)
+	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
 	if err != nil {
 		return nil, err
 	}
@@ -456,11 +460,11 @@ func (c *client) Stats(ctx context.Context, containerID string) (*Stats, error)
 	if err != nil {
 		return nil, err
 	}
-	return interfaceToStats(m.Timestamp, v), nil
+	return libcontainerdtypes.InterfaceToStats(m.Timestamp, v), nil
 }
 
 func (c *client) ListPids(ctx context.Context, containerID string) ([]uint32, error) {
-	p, err := c.getProcess(containerID, InitProcessName)
+	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
 	if err != nil {
 		return nil, err
 	}
@@ -478,8 +482,8 @@ func (c *client) ListPids(ctx context.Context, containerID string) ([]uint32, er
 	return pids, nil
 }
 
-func (c *client) Summary(ctx context.Context, containerID string) ([]Summary, error) {
+func (c *client) Summary(ctx context.Context, containerID string) ([]libcontainerdtypes.Summary, error) {
-	p, err := c.getProcess(containerID, InitProcessName)
+	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
 	if err != nil {
 		return nil, err
 	}
@@ -489,7 +493,7 @@ func (c *client) Summary(ctx context.Context, containerID string) ([]Summary, er
 		return nil, err
 	}
 
-	var infos []Summary
+	var infos []libcontainerdtypes.Summary
 	for _, pi := range pis {
 		i, err := typeurl.UnmarshalAny(pi.Info)
 		if err != nil {
@@ -506,7 +510,7 @@ func (c *client) Summary(ctx context.Context, containerID string) ([]Summary, er
 }
 
 func (c *client) DeleteTask(ctx context.Context, containerID string) (uint32, time.Time, error) {
-	p, err := c.getProcess(containerID, InitProcessName)
+	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
 	if err != nil {
 		return 255, time.Now(), nil
 	}
@@ -525,7 +529,7 @@ func (c *client) DeleteTask(ctx context.Context, containerID string) (uint32, ti
 func (c *client) Delete(ctx context.Context, containerID string) error {
 	ctr := c.getContainer(containerID)
 	if ctr == nil {
-		return errors.WithStack(newNotFoundError("no such container"))
+		return errors.WithStack(errdefs.NotFound(errors.New("no such container")))
 	}
 
 	if err := ctr.ctr.Delete(ctx); err != nil {
@@ -546,27 +550,27 @@ func (c *client) Delete(ctx context.Context, containerID string) error {
 	return nil
 }
 
-func (c *client) Status(ctx context.Context, containerID string) (Status, error) {
+func (c *client) Status(ctx context.Context, containerID string) (libcontainerdtypes.Status, error) {
 	ctr := c.getContainer(containerID)
 	if ctr == nil {
-		return StatusUnknown, errors.WithStack(newNotFoundError("no such container"))
+		return libcontainerdtypes.StatusUnknown, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
 	}
 
 	t := ctr.getTask()
 	if t == nil {
-		return StatusUnknown, errors.WithStack(newNotFoundError("no such task"))
+		return libcontainerdtypes.StatusUnknown, errors.WithStack(errdefs.NotFound(errors.New("no such task")))
 	}
 
 	s, err := t.Status(ctx)
 	if err != nil {
-		return StatusUnknown, wrapError(err)
+		return libcontainerdtypes.StatusUnknown, wrapError(err)
 	}
 
-	return Status(s.Status), nil
+	return libcontainerdtypes.Status(s.Status), nil
 }
 
 func (c *client) CreateCheckpoint(ctx context.Context, containerID, checkpointDir string, exit bool) error {
-	p, err := c.getProcess(containerID, InitProcessName)
+	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
 	if err != nil {
 		return err
 	}
@@ -648,33 +652,32 @@ func (c *client) removeContainer(id string) {
 func (c *client) getProcess(containerID, processID string) (containerd.Process, error) {
 	ctr := c.getContainer(containerID)
 	if ctr == nil {
-		return nil, errors.WithStack(newNotFoundError("no such container"))
+		return nil, errors.WithStack(errdefs.NotFound(errors.New("no such container")))
 	}
 
 	t := ctr.getTask()
 	if t == nil {
-		return nil, errors.WithStack(newNotFoundError("container is not running"))
+		return nil, errors.WithStack(errdefs.NotFound(errors.New("container is not running")))
 	}
-	if processID == InitProcessName {
+	if processID == libcontainerdtypes.InitProcessName {
 		return t, nil
 	}
 
 	p := ctr.getProcess(processID)
 	if p == nil {
-		return nil, errors.WithStack(newNotFoundError("no such exec"))
+		return nil, errors.WithStack(errdefs.NotFound(errors.New("no such exec")))
 	}
 	return p, nil
 }
 
 // createIO creates the io to be used by a process
 // This needs to get a pointer to interface as upon closure the process may not have yet been registered
-func (c *client) createIO(fifos *cio.FIFOSet, containerID, processID string, stdinCloseSync chan struct{}, attachStdio StdioCallback) (cio.IO, error) {
+func (c *client) createIO(fifos *cio.FIFOSet, containerID, processID string, stdinCloseSync chan struct{}, attachStdio libcontainerdtypes.StdioCallback) (cio.IO, error) {
 	var (
 		io  *cio.DirectIO
 		err error
 	)
-
+	io, err = c.newDirectIO(context.Background(), fifos)
-	io, err = cio.NewDirectIO(context.Background(), fifos)
 	if err != nil {
 		return nil, err
 	}
@@ -713,8 +716,8 @@ func (c *client) createIO(fifos *cio.FIFOSet, containerID, processID string, std
 	return rio, err
 }
 
-func (c *client) processEvent(ctr *container, et EventType, ei EventInfo) {
+func (c *client) processEvent(ctr *container, et libcontainerdtypes.EventType, ei libcontainerdtypes.EventInfo) {
-	c.eventQ.append(ei.ContainerID, func() {
+	c.eventQ.Append(ei.ContainerID, func() {
 		err := c.backend.ProcessEvent(ei.ContainerID, et, ei)
 		if err != nil {
 			c.logger.WithError(err).WithFields(logrus.Fields{
@@ -724,7 +727,7 @@ func (c *client) processEvent(ctr *container, et EventType, ei EventInfo) {
 			}).Error("failed to process event")
 		}
 
-		if et == EventExit && ei.ProcessID != ei.ContainerID {
+		if et == libcontainerdtypes.EventExit && ei.ProcessID != ei.ContainerID {
 			p := ctr.getProcess(ei.ProcessID)
 			if p == nil {
 				c.logger.WithError(errors.New("no such process")).
@@ -759,8 +762,8 @@ func (c *client) processEventStream(ctx context.Context, ns string) {
 	var (
 		err error
 		ev  *events.Envelope
-		et  EventType
+		et  libcontainerdtypes.EventType
-		ei  EventInfo
+		ei  libcontainerdtypes.EventInfo
 		ctr *container
 	)
 
@@ -800,22 +803,22 @@ func (c *client) processEventStream(ctx context.Context, ns string) {
 
 			switch t := v.(type) {
 			case *apievents.TaskCreate:
-				et = EventCreate
+				et = libcontainerdtypes.EventCreate
-				ei = EventInfo{
+				ei = libcontainerdtypes.EventInfo{
 					ContainerID: t.ContainerID,
 					ProcessID:   t.ContainerID,
 					Pid:         t.Pid,
 				}
 			case *apievents.TaskStart:
-				et = EventStart
+				et = libcontainerdtypes.EventStart
-				ei = EventInfo{
+				ei = libcontainerdtypes.EventInfo{
 					ContainerID: t.ContainerID,
 					ProcessID:   t.ContainerID,
 					Pid:         t.Pid,
 				}
 			case *apievents.TaskExit:
-				et = EventExit
+				et = libcontainerdtypes.EventExit
-				ei = EventInfo{
+				ei = libcontainerdtypes.EventInfo{
 					ContainerID: t.ContainerID,
 					ProcessID:   t.ID,
 					Pid:         t.Pid,
@@ -823,33 +826,33 @@ func (c *client) processEventStream(ctx context.Context, ns string) {
 					ExitedAt:    t.ExitedAt,
 				}
 			case *apievents.TaskOOM:
-				et = EventOOM
+				et = libcontainerdtypes.EventOOM
-				ei = EventInfo{
+				ei = libcontainerdtypes.EventInfo{
 					ContainerID: t.ContainerID,
 					OOMKilled:   true,
 				}
 				oomKilled = true
 			case *apievents.TaskExecAdded:
-				et = EventExecAdded
+				et = libcontainerdtypes.EventExecAdded
-				ei = EventInfo{
+				ei = libcontainerdtypes.EventInfo{
 					ContainerID: t.ContainerID,
 					ProcessID:   t.ExecID,
 				}
 			case *apievents.TaskExecStarted:
-				et = EventExecStarted
+				et = libcontainerdtypes.EventExecStarted
-				ei = EventInfo{
+				ei = libcontainerdtypes.EventInfo{
 					ContainerID: t.ContainerID,
 					ProcessID:   t.ExecID,
 					Pid:         t.Pid,
 				}
 			case *apievents.TaskPaused:
-				et = EventPaused
+				et = libcontainerdtypes.EventPaused
-				ei = EventInfo{
+				ei = libcontainerdtypes.EventInfo{
 					ContainerID: t.ContainerID,
 				}
 			case *apievents.TaskResumed:
-				et = EventResumed
+				et = libcontainerdtypes.EventResumed
-				ei = EventInfo{
+				ei = libcontainerdtypes.EventInfo{
 					ContainerID: t.ContainerID,
 				}
 			default:

libcontainerd/remote/client_io_windows.go

@@ -0,0 +1,161 @@
+package remote // import "github.com/docker/docker/libcontainerd/remote"
+
+import (
+	"io"
+	"net"
+	"sync"
+
+	winio "github.com/Microsoft/go-winio"
+	"github.com/containerd/containerd/cio"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	//	"golang.org/x/net/context"
+)
+
+type delayedConnection struct {
+	l    net.Listener
+	con  net.Conn
+	wg   sync.WaitGroup
+	once sync.Once
+}
+
+func (dc *delayedConnection) Write(p []byte) (int, error) {
+	dc.wg.Wait()
+	if dc.con != nil {
+		return dc.con.Write(p)
+	}
+	return 0, errors.New("use of closed network connection")
+}
+
+func (dc *delayedConnection) Read(p []byte) (int, error) {
+	dc.wg.Wait()
+	if dc.con != nil {
+		return dc.con.Read(p)
+	}
+	return 0, errors.New("use of closed network connection")
+}
+
+func (dc *delayedConnection) unblockConnectionWaiters() {
+	defer dc.once.Do(func() {
+		dc.wg.Done()
+	})
+}
+
+func (dc *delayedConnection) Close() error {
+	dc.l.Close()
+	if dc.con != nil {
+		return dc.con.Close()
+	}
+	dc.unblockConnectionWaiters()
+	return nil
+}
+
+type stdioPipes struct {
+	stdin  io.WriteCloser
+	stdout io.ReadCloser
+	stderr io.ReadCloser
+}
+
+// newStdioPipes creates actual fifos for stdio.
+func (c *client) newStdioPipes(fifos *cio.FIFOSet) (_ *stdioPipes, err error) {
+	p := &stdioPipes{}
+	if fifos.Stdin != "" {
+		c.logger.WithFields(logrus.Fields{"stdin": fifos.Stdin}).Debug("listen")
+		l, err := winio.ListenPipe(fifos.Stdin, nil)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create stdin pipe %s", fifos.Stdin)
+		}
+		dc := &delayedConnection{
+			l: l,
+		}
+		dc.wg.Add(1)
+		defer func() {
+			if err != nil {
+				dc.Close()
+			}
+		}()
+		p.stdin = dc
+
+		go func() {
+			c.logger.WithFields(logrus.Fields{"stdin": fifos.Stdin}).Debug("accept")
+			conn, err := l.Accept()
+			if err != nil {
+				dc.Close()
+				if err != winio.ErrPipeListenerClosed {
+					c.logger.WithError(err).Errorf("failed to accept stdin connection on %s", fifos.Stdin)
+				}
+				return
+			}
+			c.logger.WithFields(logrus.Fields{"stdin": fifos.Stdin}).Debug("connected")
+			dc.con = conn
+			dc.unblockConnectionWaiters()
+		}()
+	}
+
+	if fifos.Stdout != "" {
+		c.logger.WithFields(logrus.Fields{"stdout": fifos.Stdout}).Debug("listen")
+		l, err := winio.ListenPipe(fifos.Stdout, nil)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create stdout pipe %s", fifos.Stdout)
+		}
+		dc := &delayedConnection{
+			l: l,
+		}
+		dc.wg.Add(1)
+		defer func() {
+			if err != nil {
+				dc.Close()
+			}
+		}()
+		p.stdout = dc
+
+		go func() {
+			c.logger.WithFields(logrus.Fields{"stdout": fifos.Stdout}).Debug("accept")
+			conn, err := l.Accept()
+			if err != nil {
+				dc.Close()
+				if err != winio.ErrPipeListenerClosed {
+					c.logger.WithError(err).Errorf("failed to accept stdout connection on %s", fifos.Stdout)
+				}
+				return
+			}
+			c.logger.WithFields(logrus.Fields{"stdout": fifos.Stdout}).Debug("connected")
+			dc.con = conn
+			dc.unblockConnectionWaiters()
+		}()
+	}
+
+	if fifos.Stderr != "" {
+		c.logger.WithFields(logrus.Fields{"stderr": fifos.Stderr}).Debug("listen")
+		l, err := winio.ListenPipe(fifos.Stderr, nil)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to create stderr pipe %s", fifos.Stderr)
+		}
+		dc := &delayedConnection{
+			l: l,
+		}
+		dc.wg.Add(1)
+		defer func() {
+			if err != nil {
+				dc.Close()
+			}
+		}()
+		p.stderr = dc
+
+		go func() {
+			c.logger.WithFields(logrus.Fields{"stderr": fifos.Stderr}).Debug("accept")
+			conn, err := l.Accept()
+			if err != nil {
+				dc.Close()
+				if err != winio.ErrPipeListenerClosed {
+					c.logger.WithError(err).Errorf("failed to accept stderr connection on %s", fifos.Stderr)
+				}
+				return
+			}
+			c.logger.WithFields(logrus.Fields{"stderr": fifos.Stderr}).Debug("connected")
+			dc.con = conn
+			dc.unblockConnectionWaiters()
+		}()
+	}
+	return p, nil
+}

libcontainerd/remote/client_linux.go

@@ -1,4 +1,4 @@
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
+package remote // import "github.com/docker/docker/libcontainerd/remote"
 
 import (
 	"context"
@@ -9,17 +9,20 @@ import (
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/cio"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 )
 
-func summaryFromInterface(i interface{}) (*Summary, error) {
+const runtimeName = "io.containerd.runtime.v1.linux"
-	return &Summary{}, nil
+
+func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) {
+	return &libcontainerdtypes.Summary{}, nil
 }
 
-func (c *client) UpdateResources(ctx context.Context, containerID string, resources *Resources) error {
+func (c *client) UpdateResources(ctx context.Context, containerID string, resources *libcontainerdtypes.Resources) error {
-	p, err := c.getProcess(containerID, InitProcessName)
+	p, err := c.getProcess(containerID, libcontainerdtypes.InitProcessName)
 	if err != nil {
 		return err
 	}
@@ -106,3 +109,7 @@ func newFIFOSet(bundleDir, processID string, withStdin, withTerminal bool) *cio.
 
 	return cio.NewFIFOSet(config, closer)
 }
+
+func (c *client) newDirectIO(ctx context.Context, fifos *cio.FIFOSet) (*cio.DirectIO, error) {
+	return cio.NewDirectIO(ctx, fifos)
+}

libcontainerd/remote/client_windows.go

@@ -1,19 +1,24 @@
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
+package remote // import "github.com/docker/docker/libcontainerd/remote"
 
 import (
+	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 
 	"github.com/containerd/containerd/cio"
 	"github.com/containerd/containerd/windows/hcsshimtypes"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 )
 
-func summaryFromInterface(i interface{}) (*Summary, error) {
+const runtimeName = "io.containerd.runhcs.v1"
+
+func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) {
 	switch pd := i.(type) {
 	case *hcsshimtypes.ProcessDetails:
-		return &Summary{
+		return &libcontainerdtypes.Summary{
 			CreateTimestamp:              pd.CreatedAt,
 			ImageName:                    pd.ImageName,
 			KernelTime100ns:              pd.KernelTime_100Ns,
@@ -29,7 +34,8 @@ func summaryFromInterface(i interface{}) (*Summary, error) {
 }
 
 func prepareBundleDir(bundleDir string, ociSpec *specs.Spec) (string, error) {
-	return bundleDir, nil
+	// TODO: (containerd) Determine if we need to use system.MkdirAllWithACL here
+	return bundleDir, os.MkdirAll(bundleDir, 0755)
 }
 
 func pipeName(containerID, processID, name string) string {
@@ -53,3 +59,22 @@ func newFIFOSet(bundleDir, processID string, withStdin, withTerminal bool) *cio.
 
 	return cio.NewFIFOSet(config, nil)
 }
+
+func (c *client) newDirectIO(ctx context.Context, fifos *cio.FIFOSet) (*cio.DirectIO, error) {
+	pipes, err := c.newStdioPipes(fifos)
+	if err != nil {
+		return nil, err
+	}
+	return cio.NewDirectIOFromFIFOSet(ctx, pipes.stdin, pipes.stdout, pipes.stderr, fifos), nil
+}
+
+func (c *client) UpdateResources(ctx context.Context, containerID string, resources *libcontainerdtypes.Resources) error {
+	// TODO: (containerd): Not implemented, but don't error.
+	return nil
+}
+
+func getSpecUser(ociSpec *specs.Spec) (int, int) {
+	// TODO: (containerd): Not implemented, but don't error.
+	// Not clear if we can even do this for LCOW.
+	return 0, 0
+}

libcontainerd/types/types.go

@@ -1,4 +1,4 @@
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
+package types // import "github.com/docker/docker/libcontainerd/types"
 
 import (
 	"context"
@@ -89,3 +89,6 @@ type Client interface {
 
 // StdioCallback is called to connect a container or process stdio.
 type StdioCallback func(io *cio.DirectIO) (cio.IO, error)
+
+// InitProcessName is the name given to the first process of a container
+const InitProcessName = "init"

libcontainerd/types/types_linux.go

@@ -1,4 +1,4 @@
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
+package types // import "github.com/docker/docker/libcontainerd/types"
 
 import (
 	"time"
@@ -16,7 +16,8 @@ type Stats struct {
 	Metrics *cgroups.Metrics
 }
 
-func interfaceToStats(read time.Time, v interface{}) *Stats {
+// InterfaceToStats returns a stats object from the platform-specific interface.
+func InterfaceToStats(read time.Time, v interface{}) *Stats {
 	return &Stats{
 		Metrics: v.(*cgroups.Metrics),
 		Read:    read,

libcontainerd/types/types_windows.go

@@ -1,10 +1,9 @@
-package libcontainerd // import "github.com/docker/docker/libcontainerd"
+package types // import "github.com/docker/docker/libcontainerd/types"
 
 import (
 	"time"
 
 	"github.com/Microsoft/hcsshim"
-	opengcs "github.com/Microsoft/opengcs/client"
 )
 
 // Summary contains a ProcessList item from HCS to support `top`
@@ -16,7 +15,8 @@ type Stats struct {
 	HCSStats *hcsshim.Statistics
 }
 
-func interfaceToStats(read time.Time, v interface{}) *Stats {
+// InterfaceToStats returns a stats object from the platform-specific interface.
+func InterfaceToStats(read time.Time, v interface{}) *Stats {
 	return &Stats{
 		HCSStats: v.(*hcsshim.Statistics),
 		Read:     read,
@@ -26,11 +26,6 @@ func interfaceToStats(read time.Time, v interface{}) *Stats {
 // Resources defines updatable container resource values.
 type Resources struct{}
 
-// LCOWOption is a CreateOption required for LCOW configuration
-type LCOWOption struct {
-	Config *opengcs.Config
-}
-
 // Checkpoint holds the details of a checkpoint (not supported in windows)
 type Checkpoint struct {
 	Name string

pkg/system/init_unix.go

@@ -5,3 +5,8 @@ package system // import "github.com/docker/docker/pkg/system"
 // InitLCOW does nothing since LCOW is a windows only feature
 func InitLCOW(experimental bool) {
 }
+
+// ContainerdRuntimeSupported returns true if the use of ContainerD runtime is supported.
+func ContainerdRuntimeSupported(_ bool, _ string) bool {
+	return true
+}

pkg/system/init_windows.go

@@ -1,7 +1,19 @@
 package system // import "github.com/docker/docker/pkg/system"
 
-// lcowSupported determines if Linux Containers on Windows are supported.
+import (
-var lcowSupported = false
+	"os"
+
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	// lcowSupported determines if Linux Containers on Windows are supported.
+	lcowSupported = false
+
+	// containerdRuntimeSupported determines if ContainerD should be the runtime.
+	// As of March 2019, this is an experimental feature.
+	containerdRuntimeSupported = false
+)
 
 // InitLCOW sets whether LCOW is supported or not
 func InitLCOW(experimental bool) {
@@ -10,3 +22,19 @@ func InitLCOW(experimental bool) {
 		lcowSupported = true
 	}
 }
+
+// InitContainerdRuntime sets whether to use ContainerD for runtime
+// on Windows. This is an experimental feature still in development, and
+// also requires an environment variable to be set (so as not to turn the
+// feature on from simply experimental which would also mean LCOW.
+func InitContainerdRuntime(experimental bool, cdPath string) {
+	if experimental && len(cdPath) > 0 && len(os.Getenv("DOCKER_WINDOWS_CONTAINERD_RUNTIME")) > 0 {
+		logrus.Warnf("Using ContainerD runtime. This feature is experimental")
+		containerdRuntimeSupported = true
+	}
+}
+
+// ContainerdRuntimeSupported returns true if the use of ContainerD runtime is supported.
+func ContainerdRuntimeSupported() bool {
+	return containerdRuntimeSupported
+}

plugin/executor/containerd/containerd.go

@@ -12,6 +12,7 @@ import (
 	"github.com/containerd/containerd/runtime/linux/runctypes"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/libcontainerd"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -30,11 +31,11 @@ type ExitHandler interface {
 // However right now this whole package is tied to github.com/docker/docker/libcontainerd
 type Client interface {
 	Create(ctx context.Context, containerID string, spec *specs.Spec, runtimeOptions interface{}) error
-	Restore(ctx context.Context, containerID string, attachStdio libcontainerd.StdioCallback) (alive bool, pid int, err error)
+	Restore(ctx context.Context, containerID string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, err error)
-	Status(ctx context.Context, containerID string) (libcontainerd.Status, error)
+	Status(ctx context.Context, containerID string) (libcontainerdtypes.Status, error)
 	Delete(ctx context.Context, containerID string) error
 	DeleteTask(ctx context.Context, containerID string) (uint32, time.Time, error)
-	Start(ctx context.Context, containerID, checkpointDir string, withStdin bool, attachStdio libcontainerd.StdioCallback) (pid int, err error)
+	Start(ctx context.Context, containerID, checkpointDir string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (pid int, err error)
 	SignalProcess(ctx context.Context, containerID, processID string, signal int) error
 }
 
@@ -87,7 +88,7 @@ func (e *Executor) Create(id string, spec specs.Spec, stdout, stderr io.WriteClo
 				logrus.WithError(err2).WithField("id", id).Warn("Received an error while attempting to read plugin status")
 			}
 		} else {
-			if status != libcontainerd.StatusRunning && status != libcontainerd.StatusUnknown {
+			if status != libcontainerdtypes.StatusRunning && status != libcontainerdtypes.StatusUnknown {
 				if err2 := e.client.Delete(ctx, id); err2 != nil && !errdefs.IsNotFound(err2) {
 					logrus.WithError(err2).WithField("plugin", id).Error("Error cleaning up containerd container")
 				}
@@ -122,19 +123,19 @@ func (e *Executor) Restore(id string, stdout, stderr io.WriteCloser) (bool, erro
 // IsRunning returns if the container with the given id is running
 func (e *Executor) IsRunning(id string) (bool, error) {
 	status, err := e.client.Status(context.Background(), id)
-	return status == libcontainerd.StatusRunning, err
+	return status == libcontainerdtypes.StatusRunning, err
 }
 
 // Signal sends the specified signal to the container
 func (e *Executor) Signal(id string, signal int) error {
-	return e.client.SignalProcess(context.Background(), id, libcontainerd.InitProcessName, signal)
+	return e.client.SignalProcess(context.Background(), id, libcontainerdtypes.InitProcessName, signal)
 }
 
 // ProcessEvent handles events from containerd
 // All events are ignored except the exit event, which is sent of to the stored handler
-func (e *Executor) ProcessEvent(id string, et libcontainerd.EventType, ei libcontainerd.EventInfo) error {
+func (e *Executor) ProcessEvent(id string, et libcontainerdtypes.EventType, ei libcontainerdtypes.EventInfo) error {
 	switch et {
-	case libcontainerd.EventExit:
+	case libcontainerdtypes.EventExit:
 		deleteTaskAndContainer(context.Background(), e.client, id)
 		return e.exitHandler.HandleExitEvent(ei.ContainerID)
 	}
@@ -152,7 +153,7 @@ func (c *rio) Wait() {
 	c.IO.Wait()
 }
 
-func attachStreamsFunc(stdout, stderr io.WriteCloser) libcontainerd.StdioCallback {
+func attachStreamsFunc(stdout, stderr io.WriteCloser) libcontainerdtypes.StdioCallback {
 	return func(iop *cio.DirectIO) (cio.IO, error) {
 		if iop.Stdin != nil {
 			iop.Stdin.Close()

plugin/executor/containerd/containerd_test.go

@@ -8,7 +8,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/docker/docker/libcontainerd"
+	libcontainerdtypes "github.com/docker/docker/libcontainerd/types"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"gotest.tools/assert"
@@ -82,22 +82,22 @@ func (c *mockClient) Create(ctx context.Context, id string, _ *specs.Spec, _ int
 	return nil
 }
 
-func (c *mockClient) Restore(ctx context.Context, id string, attachStdio libcontainerd.StdioCallback) (alive bool, pid int, err error) {
+func (c *mockClient) Restore(ctx context.Context, id string, attachStdio libcontainerdtypes.StdioCallback) (alive bool, pid int, err error) {
 	return false, 0, nil
 }
 
-func (c *mockClient) Status(ctx context.Context, id string) (libcontainerd.Status, error) {
+func (c *mockClient) Status(ctx context.Context, id string) (libcontainerdtypes.Status, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
 	running, ok := c.containers[id]
 	if !ok {
-		return libcontainerd.StatusUnknown, errors.New("not found")
+		return libcontainerdtypes.StatusUnknown, errors.New("not found")
 	}
 	if running {
-		return libcontainerd.StatusRunning, nil
+		return libcontainerdtypes.StatusRunning, nil
 	}
-	return libcontainerd.StatusStopped, nil
+	return libcontainerdtypes.StatusStopped, nil
 }
 
 func (c *mockClient) Delete(ctx context.Context, id string) error {
@@ -111,7 +111,7 @@ func (c *mockClient) DeleteTask(ctx context.Context, id string) (uint32, time.Ti
 	return 0, time.Time{}, nil
 }
 
-func (c *mockClient) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio libcontainerd.StdioCallback) (pid int, err error) {
+func (c *mockClient) Start(ctx context.Context, id, checkpointDir string, withStdin bool, attachStdio libcontainerdtypes.StdioCallback) (pid int, err error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()