vendor: update buildkit to 9acf51e491

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

vendor.conf

@@ -26,7 +26,7 @@ github.com/imdario/mergo v0.3.5
 golang.org/x/sync fd80eb99c8f653c847d294a001bdf2a3a6f768f5
 
 # buildkit
-github.com/moby/buildkit cce2080ddbe4698912f2290892b247c83627efa8
+github.com/moby/buildkit 9acf51e49185b348608e0096b2903dd72907adcb
 github.com/tonistiigi/fsutil 8abad97ee3969cdf5e9c367f46adba2c212b3ddb
 github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746
 github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7

vendor/github.com/moby/buildkit/README.md

@@ -138,11 +138,11 @@ docker inspect myimage
 
 ##### Building a Dockerfile using [external frontend](https://hub.docker.com/r/tonistiigi/dockerfile/tags/):
 
-During development, an external version of the Dockerfile frontend is pushed to https://hub.docker.com/r/tonistiigi/dockerfile that can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)).
+During development, an external version of the Dockerfile frontend is pushed to https://hub.docker.com/r/tonistiigi/dockerfile that can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)). For automatic build from master branch of this repository `tonistiigi/dockerfile:master` image can be used.
 
 ```
-buildctl build --frontend=gateway.v0 --frontend-opt=source=tonistiigi/dockerfile:v0 --local context=. --local dockerfile=.
+buildctl build --frontend=gateway.v0 --frontend-opt=source=tonistiigi/dockerfile --local context=. --local dockerfile=.
-buildctl build --frontend gateway.v0 --frontend-opt=source=tonistiigi/dockerfile:v0 --frontend-opt=context=git://github.com/moby/moby --frontend-opt build-arg:APT_MIRROR=cdn-fastly.deb.debian.org
+buildctl build --frontend gateway.v0 --frontend-opt=source=tonistiigi/dockerfile --frontend-opt=context=git://github.com/moby/moby --frontend-opt build-arg:APT_MIRROR=cdn-fastly.deb.debian.org
 ````
 
 ### Exporters

vendor/github.com/moby/buildkit/cache/remotecache/export.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images"
-	"github.com/docker/distribution/manifest"
 	v1 "github.com/moby/buildkit/cache/remotecache/v1"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver"
@@ -17,6 +16,7 @@ import (
 	"github.com/moby/buildkit/util/progress"
 	"github.com/moby/buildkit/util/push"
 	digest "github.com/opencontainers/go-digest"
+	specs "github.com/opencontainers/image-spec/specs-go"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
@@ -46,7 +46,9 @@ func (ce *CacheExporter) Finalize(ctx context.Context, cc *v1.CacheChains, targe
 
 	// own type because oci type can't be pushed and docker type doesn't have annotations
 	type manifestList struct {
-		manifest.Versioned
+		specs.Versioned
+
+		MediaType string `json:"mediaType,omitempty"`
 
 		// Manifests references platform specific manifests.
 		Manifests []ocispec.Descriptor `json:"manifests"`

vendor/github.com/moby/buildkit/executor/oci/user.go

@@ -2,27 +2,31 @@ package oci
 
 import (
 	"context"
+	"errors"
 	"os"
 	"strconv"
 	"strings"
 
+	"github.com/containerd/containerd/containers"
+	containerdoci "github.com/containerd/containerd/oci"
 	"github.com/containerd/continuity/fs"
 	"github.com/opencontainers/runc/libcontainer/user"
+	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
-func GetUser(ctx context.Context, root, username string) (uint32, uint32, error) {
+func GetUser(ctx context.Context, root, username string) (uint32, uint32, []uint32, error) {
 	// fast path from uid/gid
-	if uid, gid, err := ParseUser(username); err == nil {
+	if uid, gid, err := ParseUIDGID(username); err == nil {
-		return uid, gid, nil
+		return uid, gid, nil, nil
 	}
 
 	passwdPath, err := user.GetPasswdPath()
 	if err != nil {
-		return 0, 0, err
+		return 0, 0, nil, err
 	}
 	groupPath, err := user.GetGroupPath()
 	if err != nil {
-		return 0, 0, err
+		return 0, 0, nil, err
 	}
 	passwdFile, err := openUserFile(root, passwdPath)
 	if err == nil {
@@ -35,33 +39,29 @@ func GetUser(ctx context.Context, root, username string) (uint32, uint32, error)
 
 	execUser, err := user.GetExecUser(username, nil, passwdFile, groupFile)
 	if err != nil {
-		return 0, 0, err
+		return 0, 0, nil, err
 	}
-
+	var sgids []uint32
-	return uint32(execUser.Uid), uint32(execUser.Gid), nil
+	for _, g := range execUser.Sgids {
+		sgids = append(sgids, uint32(g))
+	}
+	return uint32(execUser.Uid), uint32(execUser.Gid), sgids, nil
 }
 
-func ParseUser(str string) (uid uint32, gid uint32, err error) {
+// ParseUIDGID takes the fast path to parse UID and GID if and only if they are both provided
+func ParseUIDGID(str string) (uid uint32, gid uint32, err error) {
 	if str == "" {
 		return 0, 0, nil
 	}
 	parts := strings.SplitN(str, ":", 2)
-	for i, v := range parts {
+	if len(parts) == 1 {
-		switch i {
+		return 0, 0, errors.New("groups ID is not provided")
-		case 0:
+	}
-			uid, err = parseUID(v)
+	if uid, err = parseUID(parts[0]); err != nil {
-			if err != nil {
+		return 0, 0, err
-				return 0, 0, err
+	}
-			}
+	if gid, err = parseUID(parts[1]); err != nil {
-			if len(parts) == 1 {
+		return 0, 0, err
-				gid = uid
-			}
-		case 1:
-			gid, err = parseUID(v)
-			if err != nil {
-				return 0, 0, err
-			}
-		}
 	}
 	return
 }
@@ -84,3 +84,24 @@ func parseUID(str string) (uint32, error) {
 	}
 	return uint32(uid), nil
 }
+
+// WithUIDGID allows the UID and GID for the Process to be set
+// FIXME: This is a temporeray fix for the missing supplementary GIDs from containerd
+// once the PR in containerd is merged we should remove this function.
+func WithUIDGID(uid, gid uint32, sgids []uint32) containerdoci.SpecOpts {
+	return func(_ context.Context, _ containerdoci.Client, _ *containers.Container, s *containerdoci.Spec) error {
+		setProcess(s)
+		s.Process.User.UID = uid
+		s.Process.User.GID = gid
+		s.Process.User.AdditionalGids = sgids
+		return nil
+	}
+}
+
+// setProcess sets Process to empty if unset
+// FIXME: Same on this one. Need to be removed after containerd fix merged
+func setProcess(s *containerdoci.Spec) {
+	if s.Process == nil {
+		s.Process = &specs.Process{}
+	}
+}

vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go

@@ -133,7 +133,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
 	}
 	defer mount.Unmount(rootFSPath, 0)
 
-	uid, gid, err := oci.GetUser(ctx, rootFSPath, meta.User)
+	uid, gid, sgids, err := oci.GetUser(ctx, rootFSPath, meta.User)
 	if err != nil {
 		return err
 	}
@@ -143,7 +143,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
 		return err
 	}
 	defer f.Close()
-	opts := []containerdoci.SpecOpts{containerdoci.WithUIDGID(uid, gid)}
+	opts := []containerdoci.SpecOpts{oci.WithUIDGID(uid, gid, sgids)}
 	if system.SeccompSupported() {
 		opts = append(opts, seccomp.WithDefaultProfile())
 	}
@@ -170,9 +170,7 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
 	}
 
 	if w.rootless {
-		specconv.ToRootless(spec, &specconv.RootlessOpts{
+		specconv.ToRootless(spec, nil)
-			MapSubUIDGID: true,
-		})
 		// TODO(AkihiroSuda): keep Cgroups enabled if /sys/fs/cgroup/cpuset/buildkit exists and writable
 		spec.Linux.CgroupsPath = ""
 		// TODO(AkihiroSuda): ToRootless removes netns, but we should readd netns here

vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go

@@ -91,8 +91,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 		metaResolver = imagemetaresolver.Default()
 	}
 
-	var allDispatchStates []*dispatchState
+	allDispatchStates := newDispatchStates()
-	dispatchStatesByName := map[string]*dispatchState{}
 
 	// set base state for every image
 	for _, st := range stages {
@@ -100,6 +99,9 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 		if err != nil {
 			return nil, nil, err
 		}
+		if name == "" {
+			return nil, nil, errors.Errorf("base name (%s) should not be blank", st.BaseName)
+		}
 		st.BaseName = name
 
 		ds := &dispatchState{
@@ -121,13 +123,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 			ds.platform = &p
 		}
 
-		if d, ok := dispatchStatesByName[st.BaseName]; ok {
+		allDispatchStates.addState(ds)
-			ds.base = d
-		}
-		allDispatchStates = append(allDispatchStates, ds)
-		if st.Name != "" {
-			dispatchStatesByName[strings.ToLower(st.Name)] = ds
-		}
 		if opt.IgnoreCache != nil {
 			if len(opt.IgnoreCache) == 0 {
 				ds.ignoreCache = true
@@ -143,20 +139,20 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 
 	var target *dispatchState
 	if opt.Target == "" {
-		target = allDispatchStates[len(allDispatchStates)-1]
+		target = allDispatchStates.lastTarget()
 	} else {
 		var ok bool
-		target, ok = dispatchStatesByName[strings.ToLower(opt.Target)]
+		target, ok = allDispatchStates.findStateByName(opt.Target)
 		if !ok {
 			return nil, nil, errors.Errorf("target stage %s could not be found", opt.Target)
 		}
 	}
 
 	// fill dependencies to stages so unreachable ones can avoid loading image configs
-	for _, d := range allDispatchStates {
+	for _, d := range allDispatchStates.states {
 		d.commands = make([]command, len(d.stage.Commands))
 		for i, cmd := range d.stage.Commands {
-			newCmd, err := toCommand(cmd, dispatchStatesByName, allDispatchStates)
+			newCmd, err := toCommand(cmd, allDispatchStates)
 			if err != nil {
 				return nil, nil, err
 			}
@@ -165,7 +161,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 				if src != nil {
 					d.deps[src] = struct{}{}
 					if src.unregistered {
-						allDispatchStates = append(allDispatchStates, src)
+						allDispatchStates.addState(src)
 					}
 				}
 			}
@@ -173,7 +169,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 	}
 
 	eg, ctx := errgroup.WithContext(ctx)
-	for i, d := range allDispatchStates {
+	for i, d := range allDispatchStates.states {
 		reachable := isReachable(target, d)
 		// resolve image config for every stage
 		if d.base == nil {
@@ -239,7 +235,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 	buildContext := &mutableOutput{}
 	ctxPaths := map[string]struct{}{}
 
-	for _, d := range allDispatchStates {
+	for _, d := range allDispatchStates.states {
 		if !isReachable(target, d) {
 			continue
 		}
@@ -271,17 +267,16 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 		}
 
 		opt := dispatchOpt{
-			allDispatchStates:    allDispatchStates,
+			allDispatchStates: allDispatchStates,
-			dispatchStatesByName: dispatchStatesByName,
+			metaArgs:          metaArgs,
-			metaArgs:             metaArgs,
+			buildArgValues:    opt.BuildArgs,
-			buildArgValues:       opt.BuildArgs,
+			shlex:             shlex,
-			shlex:                shlex,
+			sessionID:         opt.SessionID,
-			sessionID:            opt.SessionID,
+			buildContext:      llb.NewState(buildContext),
-			buildContext:         llb.NewState(buildContext),
+			proxyEnv:          proxyEnv,
-			proxyEnv:             proxyEnv,
+			cacheIDNamespace:  opt.CacheIDNamespace,
-			cacheIDNamespace:     opt.CacheIDNamespace,
+			buildPlatforms:    opt.BuildPlatforms,
-			buildPlatforms:       opt.BuildPlatforms,
+			targetPlatform:    *opt.TargetPlatform,
-			targetPlatform:       *opt.TargetPlatform,
 		}
 
 		if err = dispatchOnBuild(d, d.image.Config.OnBuild, opt); err != nil {
@@ -330,14 +325,14 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 	return &st, &target.image, nil
 }
 
-func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) (command, error) {
+func toCommand(ic instructions.Command, allDispatchStates *dispatchStates) (command, error) {
 	cmd := command{Command: ic}
 	if c, ok := ic.(*instructions.CopyCommand); ok {
 		if c.From != "" {
 			var stn *dispatchState
 			index, err := strconv.Atoi(c.From)
 			if err != nil {
-				stn, ok = dispatchStatesByName[strings.ToLower(c.From)]
+				stn, ok = allDispatchStates.findStateByName(c.From)
 				if !ok {
 					stn = &dispatchState{
 						stage:        instructions.Stage{BaseName: c.From},
@@ -346,16 +341,16 @@ func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatc
 					}
 				}
 			} else {
-				if index < 0 || index >= len(allDispatchStates) {
+				stn, err = allDispatchStates.findStateByIndex(index)
-					return command{}, errors.Errorf("invalid stage index %d", index)
+				if err != nil {
+					return command{}, err
 				}
-				stn = allDispatchStates[index]
 			}
 			cmd.sources = []*dispatchState{stn}
 		}
 	}
 
-	if ok := detectRunMount(&cmd, dispatchStatesByName, allDispatchStates); ok {
+	if ok := detectRunMount(&cmd, allDispatchStates); ok {
 		return cmd, nil
 	}
 
@@ -363,17 +358,16 @@ func toCommand(ic instructions.Command, dispatchStatesByName map[string]*dispatc
 }
 
 type dispatchOpt struct {
-	allDispatchStates    []*dispatchState
+	allDispatchStates *dispatchStates
-	dispatchStatesByName map[string]*dispatchState
+	metaArgs          []instructions.ArgCommand
-	metaArgs             []instructions.ArgCommand
+	buildArgValues    map[string]string
-	buildArgValues       map[string]string
+	shlex             *shell.Lex
-	shlex                *shell.Lex
+	sessionID         string
-	sessionID            string
+	buildContext      llb.State
-	buildContext         llb.State
+	proxyEnv          *llb.ProxyEnv
-	proxyEnv             *llb.ProxyEnv
+	cacheIDNamespace  string
-	cacheIDNamespace     string
+	targetPlatform    specs.Platform
-	targetPlatform       specs.Platform
+	buildPlatforms    []specs.Platform
-	buildPlatforms       []specs.Platform
 }
 
 func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
@@ -456,6 +450,43 @@ type dispatchState struct {
 	unregistered bool
 }
 
+type dispatchStates struct {
+	states       []*dispatchState
+	statesByName map[string]*dispatchState
+}
+
+func newDispatchStates() *dispatchStates {
+	return &dispatchStates{statesByName: map[string]*dispatchState{}}
+}
+
+func (dss *dispatchStates) addState(ds *dispatchState) {
+	dss.states = append(dss.states, ds)
+
+	if d, ok := dss.statesByName[ds.stage.BaseName]; ok {
+		ds.base = d
+	}
+	if ds.stage.Name != "" {
+		dss.statesByName[strings.ToLower(ds.stage.Name)] = ds
+	}
+}
+
+func (dss *dispatchStates) findStateByName(name string) (*dispatchState, bool) {
+	ds, ok := dss.statesByName[strings.ToLower(name)]
+	return ds, ok
+}
+
+func (dss *dispatchStates) findStateByIndex(index int) (*dispatchState, error) {
+	if index < 0 || index >= len(dss.states) {
+		return nil, errors.Errorf("invalid stage index %d", index)
+	}
+
+	return dss.states[index], nil
+}
+
+func (dss *dispatchStates) lastTarget() *dispatchState {
+	return dss.states[len(dss.states)-1]
+}
+
 type command struct {
 	instructions.Command
 	sources []*dispatchState
@@ -474,7 +505,7 @@ func dispatchOnBuild(d *dispatchState, triggers []string, opt dispatchOpt) error
 		if err != nil {
 			return err
 		}
-		cmd, err := toCommand(ic, opt.dispatchStatesByName, opt.allDispatchStates)
+		cmd, err := toCommand(ic, opt.allDispatchStates)
 		if err != nil {
 			return err
 		}
@@ -570,7 +601,11 @@ func dispatchCopy(d *dispatchState, c instructions.SourcesAndDest, sourceState l
 
 	for i, src := range c.Sources() {
 		commitMessage.WriteString(" " + src)
-		if isAddCommand && (strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://")) {
+		if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
+			if !isAddCommand {
+				return errors.New("source can't be a URL for COPY")
+			}
+
 			// Resources from remote URLs are not decompressed.
 			// https://docs.docker.com/engine/reference/builder/#add
 			//

vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunmount.go

@@ -7,7 +7,7 @@ import (
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 )
 
-func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) bool {
+func detectRunMount(cmd *command, allDispatchStates *dispatchStates) bool {
 	return false
 }
 

vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runmount.go

@@ -5,14 +5,13 @@ package dockerfile2llb
 import (
 	"path"
 	"path/filepath"
-	"strings"
 
 	"github.com/moby/buildkit/client/llb"
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
 	"github.com/pkg/errors"
 )
 
-func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState, allDispatchStates []*dispatchState) bool {
+func detectRunMount(cmd *command, allDispatchStates *dispatchStates) bool {
 	if c, ok := cmd.Command.(*instructions.RunCommand); ok {
 		mounts := instructions.GetMounts(c)
 		sources := make([]*dispatchState, len(mounts))
@@ -24,7 +23,7 @@ func detectRunMount(cmd *command, dispatchStatesByName map[string]*dispatchState
 			if from == "" || mount.Type == instructions.MountTypeTmpfs {
 				continue
 			}
-			stn, ok := dispatchStatesByName[strings.ToLower(from)]
+			stn, ok := allDispatchStates.findStateByName(from)
 			if !ok {
 				stn = &dispatchState{
 					stage:        instructions.Stage{BaseName: from},

vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/bflag.go

@@ -72,7 +72,7 @@ func (bf *BFlags) AddString(name string, def string) *Flag {
 	return flag
 }
 
-// AddString adds a string flag to BFlags that can match multiple values
+// AddStrings adds a string flag to BFlags that can match multiple values
 func (bf *BFlags) AddStrings(name string) *Flag {
 	flag := bf.addFlag(name, stringsType)
 	if flag == nil {

vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands.go

@@ -159,7 +159,7 @@ func (s SourcesAndDest) Dest() string {
 
 // AddCommand : ADD foo /path
 //
-// Add the file 'foo' to '/path'. Tarball and Remote URL (git, http) handling
+// Add the file 'foo' to '/path'. Tarball and Remote URL (http, https) handling
 // exist here. If you do not wish to have this automatic handling, use COPY.
 //
 type AddCommand struct {

vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_unix.go

@@ -2,8 +2,9 @@
 
 package shell
 
-// EqualEnvKeys compare two strings and returns true if they are equal. On
+// EqualEnvKeys compare two strings and returns true if they are equal.
-// Windows this comparison is case insensitive.
+// On Unix this comparison is case sensitive.
+// On Windows this comparison is case insensitive.
 func EqualEnvKeys(from, to string) bool {
 	return from == to
 }

vendor/github.com/moby/buildkit/frontend/dockerfile/shell/equal_env_windows.go

@@ -2,8 +2,9 @@ package shell
 
 import "strings"
 
-// EqualEnvKeys compare two strings and returns true if they are equal. On
+// EqualEnvKeys compare two strings and returns true if they are equal.
-// Windows this comparison is case insensitive.
+// On Unix this comparison is case sensitive.
+// On Windows this comparison is case insensitive.
 func EqualEnvKeys(from, to string) bool {
 	return strings.ToUpper(from) == strings.ToUpper(to)
 }

vendor/github.com/moby/buildkit/util/imageutil/config.go

@@ -141,13 +141,17 @@ func DetectManifestMediaType(ra content.ReaderAt) (string, error) {
 	}
 
 	var mfst struct {
-		Config json.RawMessage `json:"config"`
+		MediaType string          `json:"mediaType"`
+		Config    json.RawMessage `json:"config"`
 	}
 
 	if err := json.Unmarshal(p, &mfst); err != nil {
 		return "", err
 	}
 
+	if mfst.MediaType != "" {
+		return mfst.MediaType, nil
+	}
 	if mfst.Config != nil {
 		return images.MediaTypeDockerSchema2Manifest, nil
 	}