LCOW: Builder plumbing

Signed-off-by: John Howard <jhoward@microsoft.com>

api/types/backend/build.go

@@ -28,4 +28,5 @@ type GetImageAndLayerOptions struct {
 	ForcePull  bool
 	AuthConfig map[string]types.AuthConfig
 	Output     io.Writer
+	Platform   string
 }

api/types/client.go

@@ -178,6 +178,10 @@ type ImageBuildOptions struct {
 	SecurityOpt []string
 	ExtraHosts  []string // List of extra hosts
 	Target      string
+
+	// TODO @jhowardmsft LCOW Support: This will require extending to include
+	// `Platform string`, but is ommited for now as it's hard-coded temporarily
+	// to avoid API changes.
 }
 
 // ImageBuildResponse holds information

api/types/configs.go

@@ -16,6 +16,7 @@ type ContainerCreateConfig struct {
 	HostConfig       *container.HostConfig
 	NetworkingConfig *network.NetworkingConfig
 	AdjustCPUShares  bool
+	Platform         string
 }
 
 // ContainerRmConfig holds arguments for the container remove

builder/dockerfile/builder.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"runtime"
 	"strings"
 
 	"github.com/Sirupsen/logrus"
@@ -20,6 +21,7 @@ import (
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/stringid"
+	"github.com/docker/docker/pkg/system"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	"golang.org/x/sync/syncmap"
@@ -73,13 +75,24 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
 		}()
 	}
 
+	// TODO @jhowardmsft LCOW support - this will require rework to allow both linux and Windows simultaneously.
+	// This is an interim solution to hardcode to linux if LCOW is turned on.
+	if dockerfile.Platform == "" {
+		dockerfile.Platform = runtime.GOOS
+		if dockerfile.Platform == "windows" && system.LCOWSupported() {
+			dockerfile.Platform = "linux"
+		}
+	}
+
 	builderOptions := builderOptions{
 		Options:        config.Options,
 		ProgressWriter: config.ProgressWriter,
 		Backend:        bm.backend,
 		PathCache:      bm.pathCache,
 		Archiver:       bm.archiver,
+		Platform:       dockerfile.Platform,
 	}
+
 	return newBuilder(ctx, builderOptions).build(source, dockerfile)
 }
 
@@ -90,6 +103,7 @@ type builderOptions struct {
 	ProgressWriter backend.ProgressWriter
 	PathCache      pathCache
 	Archiver       *archive.Archiver
+	Platform       string
 }
 
 // Builder is a Dockerfile builder
@@ -113,14 +127,32 @@ type Builder struct {
 	pathCache        pathCache
 	containerManager *containerManager
 	imageProber      ImageProber
+
+	// TODO @jhowardmft LCOW Support. This will be moved to options at a later
+	// stage, however that cannot be done now as it affects the public API
+	// if it were.
+	platform string
 }
 
 // newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options.
+// TODO @jhowardmsft LCOW support: Eventually platform can be moved into the builder
+// options, however, that would be an API change as it shares types.ImageBuildOptions.
 func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
 	config := options.Options
 	if config == nil {
 		config = new(types.ImageBuildOptions)
 	}
+
+	// @jhowardmsft LCOW Support. For the time being, this is interim. Eventually
+	// will be moved to types.ImageBuildOptions, but it can't for now as that would
+	// be an API change.
+	if options.Platform == "" {
+		options.Platform = runtime.GOOS
+	}
+	if options.Platform == "windows" && system.LCOWSupported() {
+		options.Platform = "linux"
+	}
+
 	b := &Builder{
 		clientCtx:        clientCtx,
 		options:          config,
@@ -136,7 +168,9 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
 		pathCache:        options.PathCache,
 		imageProber:      newImageProber(options.Backend, config.CacheFrom, config.NoCache),
 		containerManager: newContainerManager(options.Backend),
+		platform:         options.Platform,
 	}
+
 	return b
 }
 
@@ -267,6 +301,17 @@ func BuildFromConfig(config *container.Config, changes []string) (*container.Con
 		return nil, err
 	}
 
+	// TODO @jhowardmsft LCOW support. For now, if LCOW enabled, switch to linux.
+	// Also explicitly set the platform. Ultimately this will be in the builder
+	// options, but we can't do that yet as it would change the API.
+	if dockerfile.Platform == "" {
+		dockerfile.Platform = runtime.GOOS
+	}
+	if dockerfile.Platform == "windows" && system.LCOWSupported() {
+		dockerfile.Platform = "linux"
+	}
+	b.platform = dockerfile.Platform
+
 	// ensure that the commands are valid
 	for _, n := range dockerfile.AST.Children {
 		if !validCommitCommands[n.Value] {

builder/dockerfile/containerbackend.go

@@ -28,10 +28,11 @@ func newContainerManager(docker builder.ExecBackend) *containerManager {
 }
 
 // Create a container
-func (c *containerManager) Create(runConfig *container.Config, hostConfig *container.HostConfig) (container.ContainerCreateCreatedBody, error) {
+func (c *containerManager) Create(runConfig *container.Config, hostConfig *container.HostConfig, platform string) (container.ContainerCreateCreatedBody, error) {
 	container, err := c.backend.ContainerCreate(types.ContainerCreateConfig{
 		Config:     runConfig,
 		HostConfig: hostConfig,
+		Platform:   platform,
 	})
 	if err != nil {
 		return container, err

builder/dockerfile/dispatchers.go

@@ -26,6 +26,7 @@ import (
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/docker/docker/pkg/signal"
+	"github.com/docker/docker/pkg/system"
 	"github.com/docker/go-connections/nat"
 	"github.com/pkg/errors"
 )
@@ -270,10 +271,12 @@ func (b *Builder) getFromImage(shlex *ShellLex, name string) (builder.Image, err
 		name = stage.ImageID()
 	}
 
-	// Windows cannot support a container with no base image.
+	// Windows cannot support a container with no base image unless it is LCOW.
 	if name == api.NoBaseImageSpecifier {
 		if runtime.GOOS == "windows" {
-			return nil, errors.New("Windows does not support FROM scratch")
+			if b.platform == "windows" || (b.platform != "windows" && !system.LCOWSupported()) {
+				return nil, errors.New("Windows does not support FROM scratch")
+			}
 		}
 		return scratchImage, nil
 	}

builder/dockerfile/dispatchers_test.go

@@ -194,7 +194,7 @@ func TestFromScratch(t *testing.T) {
 	req := defaultDispatchReq(b, "scratch")
 	err := from(req)
 
-	if runtime.GOOS == "windows" {
+	if runtime.GOOS == "windows" && !system.LCOWSupported() {
 		assert.EqualError(t, err, "Windows does not support FROM scratch")
 		return
 	}
@@ -202,7 +202,12 @@ func TestFromScratch(t *testing.T) {
 	require.NoError(t, err)
 	assert.True(t, req.state.hasFromImage())
 	assert.Equal(t, "", req.state.imageID)
-	assert.Equal(t, []string{"PATH=" + system.DefaultPathEnv}, req.state.runConfig.Env)
+	// Windows does not set the default path. TODO @jhowardmsft LCOW support. This will need revisiting as we get further into the implementation
+	expected := "PATH=" + system.DefaultPathEnv
+	if runtime.GOOS == "windows" {
+		expected = ""
+	}
+	assert.Equal(t, []string{expected}, req.state.runConfig.Env)
 }
 
 func TestFromWithArg(t *testing.T) {

builder/dockerfile/imagecontext.go

@@ -97,12 +97,15 @@ type imageSources struct {
 	cache     pathCache // TODO: remove
 }
 
+// TODO @jhowardmsft LCOW Support: Eventually, platform can be moved to options.Options.Platform,
+// and removed from builderOptions, but that can't be done yet as it would affect the API.
 func newImageSources(ctx context.Context, options builderOptions) *imageSources {
 	getAndMount := func(idOrRef string) (builder.Image, builder.ReleaseableLayer, error) {
 		return options.Backend.GetImageAndReleasableLayer(ctx, idOrRef, backend.GetImageAndLayerOptions{
 			ForcePull:  options.Options.PullParent,
 			AuthConfig: options.Options.AuthConfigs,
 			Output:     options.ProgressWriter.Output,
+			Platform:   options.Platform,
 		})
 	}
 

builder/dockerfile/internals.go

@@ -256,13 +256,13 @@ func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *contai
 	}
 	// Set a log config to override any default value set on the daemon
 	hostConfig := &container.HostConfig{LogConfig: defaultLogConfig}
-	container, err := b.containerManager.Create(runConfig, hostConfig)
+	container, err := b.containerManager.Create(runConfig, hostConfig, b.platform)
 	return container.ID, err
 }
 
 func (b *Builder) create(runConfig *container.Config) (string, error) {
 	hostConfig := hostConfigFromOptions(b.options)
-	container, err := b.containerManager.Create(runConfig, hostConfig)
+	container, err := b.containerManager.Create(runConfig, hostConfig, b.platform)
 	if err != nil {
 		return "", err
 	}

builder/dockerfile/parser/parser.go

@@ -7,11 +7,13 @@ import (
 	"fmt"
 	"io"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 	"unicode"
 
 	"github.com/docker/docker/builder/dockerfile/command"
+	"github.com/docker/docker/pkg/system"
 	"github.com/pkg/errors"
 )
 
@@ -79,22 +81,28 @@ func (node *Node) AddChild(child *Node, startLine, endLine int) {
 }
 
 var (
-	dispatch           map[string]func(string, *Directive) (*Node, map[string]bool, error)
-	tokenWhitespace    = regexp.MustCompile(`[\t\v\f\r ]+`)
-	tokenEscapeCommand = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
-	tokenComment       = regexp.MustCompile(`^#.*$`)
+	dispatch             map[string]func(string, *Directive) (*Node, map[string]bool, error)
+	tokenWhitespace      = regexp.MustCompile(`[\t\v\f\r ]+`)
+	tokenEscapeCommand   = regexp.MustCompile(`^#[ \t]*escape[ \t]*=[ \t]*(?P<escapechar>.).*$`)
+	tokenPlatformCommand = regexp.MustCompile(`^#[ \t]*platform[ \t]*=[ \t]*(?P<platform>.*)$`)
+	tokenComment         = regexp.MustCompile(`^#.*$`)
 )
 
 // DefaultEscapeToken is the default escape token
 const DefaultEscapeToken = '\\'
 
+// DefaultPlatformToken is the platform assumed for the build if not explicitly provided
+var DefaultPlatformToken = runtime.GOOS
+
 // Directive is the structure used during a build run to hold the state of
 // parsing directives.
 type Directive struct {
 	escapeToken           rune           // Current escape token
+	platformToken         string         // Current platform token
 	lineContinuationRegex *regexp.Regexp // Current line continuation regex
 	processingComplete    bool           // Whether we are done looking for directives
 	escapeSeen            bool           // Whether the escape directive has been seen
+	platformSeen          bool           // Whether the platform directive has been seen
 }
 
 // setEscapeToken sets the default token for escaping characters in a Dockerfile.
@@ -107,6 +115,22 @@ func (d *Directive) setEscapeToken(s string) error {
 	return nil
 }
 
+// setPlatformToken sets the default platform for pulling images in a Dockerfile.
+func (d *Directive) setPlatformToken(s string) error {
+	s = strings.ToLower(s)
+	valid := []string{runtime.GOOS}
+	if runtime.GOOS == "windows" && system.LCOWSupported() {
+		valid = append(valid, "linux")
+	}
+	for _, item := range valid {
+		if s == item {
+			d.platformToken = s
+			return nil
+		}
+	}
+	return fmt.Errorf("invalid PLATFORM '%s'. Must be one of %v", s, valid)
+}
+
 // possibleParserDirective looks for one or more parser directives '# escapeToken=<char>' and
 // '# platform=<string>'. Parser directives must precede any builder instruction
 // or other comments, and cannot be repeated.
@@ -128,6 +152,23 @@ func (d *Directive) possibleParserDirective(line string) error {
 		}
 	}
 
+	// TODO @jhowardmsft LCOW Support: Eventually this check can be removed,
+	// but only recognise a platform token if running in LCOW mode.
+	if runtime.GOOS == "windows" && system.LCOWSupported() {
+		tpcMatch := tokenPlatformCommand.FindStringSubmatch(strings.ToLower(line))
+		if len(tpcMatch) != 0 {
+			for i, n := range tokenPlatformCommand.SubexpNames() {
+				if n == "platform" {
+					if d.platformSeen == true {
+						return errors.New("only one platform parser directive can be used")
+					}
+					d.platformSeen = true
+					return d.setPlatformToken(tpcMatch[i])
+				}
+			}
+		}
+	}
+
 	d.processingComplete = true
 	return nil
 }
@@ -136,6 +177,7 @@ func (d *Directive) possibleParserDirective(line string) error {
 func NewDefaultDirective() *Directive {
 	directive := Directive{}
 	directive.setEscapeToken(string(DefaultEscapeToken))
+	directive.setPlatformToken(runtime.GOOS)
 	return &directive
 }
 
@@ -200,6 +242,7 @@ func newNodeFromLine(line string, directive *Directive) (*Node, error) {
 type Result struct {
 	AST         *Node
 	EscapeToken rune
+	Platform    string
 }
 
 // Parse reads lines from a Reader, parses the lines into an AST and returns
@@ -252,7 +295,7 @@ func Parse(rwc io.Reader) (*Result, error) {
 		}
 		root.AddChild(child, startLine, currentLine)
 	}
-	return &Result{AST: root, EscapeToken: d.escapeToken}, nil
+	return &Result{AST: root, EscapeToken: d.escapeToken, Platform: d.platformToken}, nil
 }
 
 func trimComments(src []byte) []byte {

daemon/build.go

@@ -2,7 +2,6 @@ package daemon
 
 import (
 	"io"
-	"runtime"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/distribution/reference"
@@ -110,7 +109,7 @@ func newReleasableLayerForImage(img *image.Image, layerStore layer.Store) (build
 }
 
 // TODO: could this use the regular daemon PullImage ?
-func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer) (*image.Image, error) {
+func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer, platform string) (*image.Image, error) {
 	ref, err := reference.ParseNormalizedNamed(name)
 	if err != nil {
 		return nil, err
@@ -129,12 +128,7 @@ func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfi
 		pullRegistryAuth = &resolvedConfig
 	}
 
-	// TODO @jhowardmsft LCOW Support: For now, use the runtime operating system of the host.
-	// When it gets to the builder part, this will need revisiting. There would have to be
-	// some indication from the user either through CLI flag to build, or through an explicit
-	// mechanism in a dockerfile such as a parser directive extension or an addition to
-	// the FROM statement syntax.
-	if err := daemon.pullImageWithReference(ctx, ref, runtime.GOOS, nil, pullRegistryAuth, output); err != nil {
+	if err := daemon.pullImageWithReference(ctx, ref, platform, nil, pullRegistryAuth, output); err != nil {
 		return nil, err
 	}
 	return daemon.GetImage(name)
@@ -153,18 +147,16 @@ func (daemon *Daemon) GetImageAndReleasableLayer(ctx context.Context, refOrID st
 		image, _ := daemon.GetImage(refOrID)
 		// TODO: shouldn't we error out if error is different from "not found" ?
 		if image != nil {
-			// TODO LCOW @jhowardmsft. For now using runtime.GOOS for this, will need enhancing for platform when porting the builder
-			layer, err := newReleasableLayerForImage(image, daemon.stores[runtime.GOOS].layerStore)
+			layer, err := newReleasableLayerForImage(image, daemon.stores[opts.Platform].layerStore)
 			return image, layer, err
 		}
 	}
 
-	image, err := daemon.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output)
+	image, err := daemon.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output, opts.Platform)
 	if err != nil {
 		return nil, nil, err
 	}
-	// TODO LCOW @jhowardmsft. For now using runtime.GOOS for this, will need enhancing for platform when porting the builder
-	layer, err := newReleasableLayerForImage(image, daemon.stores[runtime.GOOS].layerStore)
+	layer, err := newReleasableLayerForImage(image, daemon.stores[opts.Platform].layerStore)
 	return image, layer, err
 }
 

daemon/container.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"runtime"
 	"time"
 
 	"github.com/docker/docker/api/errors"
@@ -112,7 +111,7 @@ func (daemon *Daemon) Register(c *container.Container) {
 	daemon.idIndex.Add(c.ID)
 }
 
-func (daemon *Daemon) newContainer(name string, config *containertypes.Config, hostConfig *containertypes.HostConfig, imgID image.ID, managed bool) (*container.Container, error) {
+func (daemon *Daemon) newContainer(name string, platform string, config *containertypes.Config, hostConfig *containertypes.HostConfig, imgID image.ID, managed bool) (*container.Container, error) {
 	var (
 		id             string
 		err            error
@@ -145,10 +144,8 @@ func (daemon *Daemon) newContainer(name string, config *containertypes.Config, h
 	base.ImageID = imgID
 	base.NetworkSettings = &network.Settings{IsAnonymousEndpoint: noExplicitName}
 	base.Name = name
-	// TODO @jhowardmsft LCOW - Get it from the platform of the container. For now, assume it is the OS of the host
-	base.Driver = daemon.GraphDriverName(runtime.GOOS)
-	// TODO @jhowardmsft LCOW - Similarly on this field. To solve this it will need a CLI/REST change in a subsequent PR during LCOW development
-	base.Platform = runtime.GOOS
+	base.Driver = daemon.GraphDriverName(platform)
+	base.Platform = platform
 	return base, err
 }
 

daemon/create.go

@@ -19,6 +19,7 @@ import (
 	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/stringid"
+	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/runconfig"
 	"github.com/opencontainers/selinux/go-selinux/label"
 )
@@ -75,6 +76,16 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) (
 		err       error
 	)
 
+	// TODO: @jhowardmsft LCOW support - at a later point, can remove the hard-coding
+	// to force the platform to be linux.
+	// Default the platform if not supplied
+	if params.Platform == "" {
+		params.Platform = runtime.GOOS
+	}
+	if params.Platform == "windows" && system.LCOWSupported() {
+		params.Platform = "linux"
+	}
+
 	if params.Config.Image != "" {
 		img, err = daemon.GetImage(params.Config.Image)
 		if err != nil {
@@ -82,9 +93,23 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) (
 		}
 
 		if runtime.GOOS == "solaris" && img.OS != "solaris " {
-			return nil, errors.New("Platform on which parent image was created is not Solaris")
+			return nil, errors.New("platform on which parent image was created is not Solaris")
 		}
 		imgID = img.ID()
+
+		if runtime.GOOS == "windows" && img.OS == "linux" && !system.LCOWSupported() {
+			return nil, errors.New("platform on which parent image was created is not Windows")
+		}
+	}
+
+	// Make sure the platform requested matches the image
+	if img != nil {
+		if params.Platform != img.Platform() {
+			// Ignore this in LCOW mode. @jhowardmsft TODO - This will need revisiting later.
+			if !(runtime.GOOS == "windows" && system.LCOWSupported()) {
+				return nil, fmt.Errorf("cannot create a %s container from a %s image", params.Platform, img.Platform())
+			}
+		}
 	}
 
 	if err := daemon.mergeAndVerifyConfig(params.Config, img); err != nil {
@@ -95,7 +120,7 @@ func (daemon *Daemon) create(params types.ContainerCreateConfig, managed bool) (
 		return nil, err
 	}
 
-	if container, err = daemon.newContainer(params.Name, params.Config, params.HostConfig, imgID, managed); err != nil {
+	if container, err = daemon.newContainer(params.Name, params.Platform, params.Config, params.HostConfig, imgID, managed); err != nil {
 		return nil, err
 	}
 	defer func() {