From a02553670ec7b08c293beb701d0c7c7a2922a85a Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Tue, 15 Dec 2020 02:06:55 +0000 Subject: [PATCH] vendor buildkit to v0.8.1 Signed-off-by: Tibor Vass --- vendor.conf | 2 +- vendor/github.com/moby/buildkit/README.md | 8 +-- .../moby/buildkit/client/llb/fileop.go | 8 +++ .../moby/buildkit/client/llb/source.go | 8 +-- .../github.com/moby/buildkit/solver/edge.go | 7 ++- .../github.com/moby/buildkit/solver/jobs.go | 8 ++- .../moby/buildkit/solver/llbsolver/solver.go | 4 ++ .../moby/buildkit/source/gitidentifier.go | 3 +- .../moby/buildkit/source/identifier.go | 3 + .../buildkit/util/compression/compression.go | 20 ++++--- .../util/flightcontrol/flightcontrol.go | 58 +++++++++++-------- .../util/sshutil/transport_validation.go | 11 ++++ 12 files changed, 93 insertions(+), 47 deletions(-) create mode 100644 vendor/github.com/moby/buildkit/util/sshutil/transport_validation.go diff --git a/vendor.conf b/vendor.conf index 5338a2e22a..34b377b1ac 100644 --- a/vendor.conf +++ b/vendor.conf @@ -33,7 +33,7 @@ github.com/imdario/mergo 1afb36080aec31e0d1528973ebe6 golang.org/x/sync cd5d95a43a6e21273425c7ae415d3df9ea832eeb # buildkit -github.com/moby/buildkit 950603da215ae03b843f3f66fbe86c4876a6f5a1 +github.com/moby/buildkit 8142d66b5ebde79846b869fba30d9d30633e74aa # v0.8.1 github.com/tonistiigi/fsutil 0834f99b7b85462efb69b4f571a4fa3ca7da5ac9 github.com/tonistiigi/units 6950e57a87eaf136bbe44ef2ec8e75b9e3569de2 github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 diff --git a/vendor/github.com/moby/buildkit/README.md b/vendor/github.com/moby/buildkit/README.md index 4532dd7cde..2ea64c78ba 100644 --- a/vendor/github.com/moby/buildkit/README.md +++ b/vendor/github.com/moby/buildkit/README.md @@ -3,7 +3,7 @@ # BuildKit [![GoDoc](https://godoc.org/github.com/moby/buildkit?status.svg)](https://godoc.org/github.com/moby/buildkit/client/llb) -[![Build Status](https://travis-ci.com/moby/buildkit.svg?branch=master)](https://travis-ci.com/moby/buildkit) +[![Build Status](https://github.com/moby/buildkit/workflows/build/badge.svg)](https://github.com/moby/buildkit/actions?query=workflow%3Abuild) [![Go Report Card](https://goreportcard.com/badge/github.com/moby/buildkit)](https://goreportcard.com/report/github.com/moby/buildkit) [![codecov](https://codecov.io/gh/moby/buildkit/branch/master/graph/badge.svg)](https://codecov.io/gh/moby/buildkit) @@ -28,7 +28,7 @@ Introductory blog post https://blog.mobyproject.org/introducing-buildkit-17e056c Join `#buildkit` channel on [Docker Community Slack](http://dockr.ly/slack) -:information_source: If you are visiting this repo for the usage of experimental Dockerfile features like `RUN --mount=type=(bind|cache|tmpfs|secret|ssh)`, please refer to [`frontend/dockerfile/docs/experimental.md`](frontend/dockerfile/docs/experimental.md). +:information_source: If you are visiting this repo for the usage of BuildKit-only Dockerfile features like `RUN --mount=type=(bind|cache|tmpfs|secret|ssh)`, please refer to [`frontend/dockerfile/docs/syntax.md`](frontend/dockerfile/docs/syntax.md). :information_source: [BuildKit has been integrated to `docker build` since Docker 18.06 .](https://docs.docker.com/develop/develop-images/build_enhancements/) You don't need to read this document unless you want to use the full-featured standalone version of BuildKit. @@ -178,7 +178,7 @@ buildctl build \ #### Building a Dockerfile using external frontend: -External versions of the Dockerfile frontend are pushed to https://hub.docker.com/r/docker/dockerfile-upstream and https://hub.docker.com/r/docker/dockerfile and can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)). For automatic build from master branch of this repository `docker/dockerfile-upsteam:master` or `docker/dockerfile-upstream:master-experimental` image can be used. +External versions of the Dockerfile frontend are pushed to https://hub.docker.com/r/docker/dockerfile-upstream and https://hub.docker.com/r/docker/dockerfile and can be used with the gateway frontend. The source for the external frontend is currently located in `./frontend/dockerfile/cmd/dockerfile-frontend` but will move out of this repository in the future ([#163](https://github.com/moby/buildkit/issues/163)). For automatic build from master branch of this repository `docker/dockerfile-upstream:master` or `docker/dockerfile-upstream:master-labs` image can be used. ```bash buildctl build \ @@ -435,7 +435,7 @@ For Kubernetes deployments, see [`examples/kubernetes`](./examples/kubernetes). ### Daemonless -To run client and an ephemeral daemon in a single container ("daemonless mode"): +To run the client and an ephemeral daemon in a single container ("daemonless mode"): ```bash docker run \ diff --git a/vendor/github.com/moby/buildkit/client/llb/fileop.go b/vendor/github.com/moby/buildkit/client/llb/fileop.go index fdcf9929a8..a2342db693 100644 --- a/vendor/github.com/moby/buildkit/client/llb/fileop.go +++ b/vendor/github.com/moby/buildkit/client/llb/fileop.go @@ -661,6 +661,14 @@ func (f *FileOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, [] pfo := &pb.FileOp{} + if f.constraints.Platform == nil { + p, err := getPlatform(*f.action.state)(ctx) + if err != nil { + return "", nil, nil, nil, err + } + f.constraints.Platform = p + } + pop, md := MarshalConstraints(c, &f.constraints) pop.Op = &pb.Op_File{ File: pfo, diff --git a/vendor/github.com/moby/buildkit/client/llb/source.go b/vendor/github.com/moby/buildkit/client/llb/source.go index af48a1d81c..ddd6097ad4 100644 --- a/vendor/github.com/moby/buildkit/client/llb/source.go +++ b/vendor/github.com/moby/buildkit/client/llb/source.go @@ -5,7 +5,6 @@ import ( _ "crypto/sha256" // for opencontainers/go-digest "encoding/json" "os" - "regexp" "strconv" "strings" @@ -207,8 +206,6 @@ const ( gitProtocolUnknown ) -var gitSSHRegex = regexp.MustCompile("^([a-z0-9]+@)?[^:]+:.*$") - func getGitProtocol(remote string) (string, int) { prefixes := map[string]int{ "http://": gitProtocolHTTP, @@ -224,7 +221,7 @@ func getGitProtocol(remote string) (string, int) { } } - if protocolType == gitProtocolUnknown && gitSSHRegex.MatchString(remote) { + if protocolType == gitProtocolUnknown && sshutil.IsSSHTransport(remote) { protocolType = gitProtocolSSH } @@ -254,6 +251,9 @@ func Git(remote, ref string, opts ...GitOption) State { remote = parts[0] + "/" + parts[1] } } + if protocolType == gitProtocolUnknown { + url = "https://" + url + } id := remote diff --git a/vendor/github.com/moby/buildkit/solver/edge.go b/vendor/github.com/moby/buildkit/solver/edge.go index 125c55fcfe..c35a5ef48b 100644 --- a/vendor/github.com/moby/buildkit/solver/edge.go +++ b/vendor/github.com/moby/buildkit/solver/edge.go @@ -595,6 +595,7 @@ func (e *edge) recalcCurrentState() { stHigh := edgeStatusCacheSlow // maximum possible state if e.cacheMap != nil { for _, dep := range e.deps { + isSlowCacheIncomplete := e.slowCacheFunc(dep) != nil && (dep.state == edgeStatusCacheSlow || (dep.state == edgeStatusComplete && !dep.slowCacheComplete)) isSlowIncomplete := (e.slowCacheFunc(dep) != nil || e.preprocessFunc(dep) != nil) && (dep.state == edgeStatusCacheSlow || (dep.state == edgeStatusComplete && !dep.slowCacheComplete)) if dep.state > stLow && len(dep.keyMap) == 0 && !isSlowIncomplete { @@ -604,10 +605,10 @@ func (e *edge) recalcCurrentState() { } } effectiveState := dep.state - if dep.state == edgeStatusCacheSlow && isSlowIncomplete { + if dep.state == edgeStatusCacheSlow && isSlowCacheIncomplete { effectiveState = edgeStatusCacheFast } - if dep.state == edgeStatusComplete && isSlowIncomplete { + if dep.state == edgeStatusComplete && isSlowCacheIncomplete { effectiveState = edgeStatusCacheFast } if effectiveState < stHigh { @@ -619,7 +620,7 @@ func (e *edge) recalcCurrentState() { if dep.state < edgeStatusCacheFast { allDepsCompletedCacheFast = false } - if isSlowIncomplete || dep.state < edgeStatusCacheSlow { + if isSlowCacheIncomplete || dep.state < edgeStatusCacheSlow { allDepsCompletedCacheSlow = false } if dep.state < edgeStatusCacheSlow && len(dep.keyMap) == 0 { diff --git a/vendor/github.com/moby/buildkit/solver/jobs.go b/vendor/github.com/moby/buildkit/solver/jobs.go index 67f1a30f94..806cec2577 100644 --- a/vendor/github.com/moby/buildkit/solver/jobs.go +++ b/vendor/github.com/moby/buildkit/solver/jobs.go @@ -527,7 +527,13 @@ func (j *Job) Discard() error { st.mu.Unlock() } - delete(j.list.jobs, j.id) + go func() { + // don't clear job right away. there might still be a status request coming to read progress + time.Sleep(10 * time.Second) + j.list.mu.Lock() + defer j.list.mu.Unlock() + delete(j.list.jobs, j.id) + }() return nil } diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go b/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go index d3ef585284..31e8afca7a 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/solver.go @@ -131,6 +131,10 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro } } + if res == nil { + res = &frontend.Result{} + } + defer func() { res.EachRef(func(ref solver.ResultProxy) error { go ref.Release(context.TODO()) diff --git a/vendor/github.com/moby/buildkit/source/gitidentifier.go b/vendor/github.com/moby/buildkit/source/gitidentifier.go index 680af6ab1f..e64b064508 100644 --- a/vendor/github.com/moby/buildkit/source/gitidentifier.go +++ b/vendor/github.com/moby/buildkit/source/gitidentifier.go @@ -4,6 +4,7 @@ import ( "net/url" "strings" + "github.com/moby/buildkit/util/sshutil" "github.com/pkg/errors" ) @@ -58,7 +59,7 @@ func (i *GitIdentifier) ID() string { // isGitTransport returns true if the provided str is a git transport by inspecting // the prefix of the string for known protocols used in git. func isGitTransport(str string) bool { - return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://") || strings.HasPrefix(str, "git://") || strings.HasPrefix(str, "git@") + return strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://") || strings.HasPrefix(str, "git://") || sshutil.IsSSHTransport(str) } func getRefAndSubdir(fragment string) (ref string, subdir string) { diff --git a/vendor/github.com/moby/buildkit/source/identifier.go b/vendor/github.com/moby/buildkit/source/identifier.go index 1120bd7649..4c1251fc70 100644 --- a/vendor/github.com/moby/buildkit/source/identifier.go +++ b/vendor/github.com/moby/buildkit/source/identifier.go @@ -102,6 +102,9 @@ func FromLLB(op *pb.Op_Source, platform *pb.Platform) (Identifier, error) { id.KeepGitDir = true } case pb.AttrFullRemoteURL: + if !isGitTransport(v) { + v = "https://" + v + } id.Remote = v case pb.AttrAuthHeaderSecret: id.AuthHeaderSecret = v diff --git a/vendor/github.com/moby/buildkit/util/compression/compression.go b/vendor/github.com/moby/buildkit/util/compression/compression.go index f63d2934bc..654b056335 100644 --- a/vendor/github.com/moby/buildkit/util/compression/compression.go +++ b/vendor/github.com/moby/buildkit/util/compression/compression.go @@ -99,17 +99,21 @@ func detectCompressionType(cr io.Reader) (Type, error) { } var toDockerLayerType = map[string]string{ - ocispec.MediaTypeImageLayer: images.MediaTypeDockerSchema2Layer, - images.MediaTypeDockerSchema2Layer: images.MediaTypeDockerSchema2Layer, - ocispec.MediaTypeImageLayerGzip: images.MediaTypeDockerSchema2LayerGzip, - images.MediaTypeDockerSchema2LayerGzip: images.MediaTypeDockerSchema2LayerGzip, + ocispec.MediaTypeImageLayer: images.MediaTypeDockerSchema2Layer, + images.MediaTypeDockerSchema2Layer: images.MediaTypeDockerSchema2Layer, + ocispec.MediaTypeImageLayerGzip: images.MediaTypeDockerSchema2LayerGzip, + images.MediaTypeDockerSchema2LayerGzip: images.MediaTypeDockerSchema2LayerGzip, + images.MediaTypeDockerSchema2LayerForeign: images.MediaTypeDockerSchema2Layer, + images.MediaTypeDockerSchema2LayerForeignGzip: images.MediaTypeDockerSchema2LayerGzip, } var toOCILayerType = map[string]string{ - ocispec.MediaTypeImageLayer: ocispec.MediaTypeImageLayer, - images.MediaTypeDockerSchema2Layer: ocispec.MediaTypeImageLayer, - ocispec.MediaTypeImageLayerGzip: ocispec.MediaTypeImageLayerGzip, - images.MediaTypeDockerSchema2LayerGzip: ocispec.MediaTypeImageLayerGzip, + ocispec.MediaTypeImageLayer: ocispec.MediaTypeImageLayer, + images.MediaTypeDockerSchema2Layer: ocispec.MediaTypeImageLayer, + ocispec.MediaTypeImageLayerGzip: ocispec.MediaTypeImageLayerGzip, + images.MediaTypeDockerSchema2LayerGzip: ocispec.MediaTypeImageLayerGzip, + images.MediaTypeDockerSchema2LayerForeign: ocispec.MediaTypeImageLayer, + images.MediaTypeDockerSchema2LayerForeignGzip: ocispec.MediaTypeImageLayerGzip, } func convertLayerMediaType(mediaType string, oci bool) string { diff --git a/vendor/github.com/moby/buildkit/util/flightcontrol/flightcontrol.go b/vendor/github.com/moby/buildkit/util/flightcontrol/flightcontrol.go index a543364c75..fc9f7272a4 100644 --- a/vendor/github.com/moby/buildkit/util/flightcontrol/flightcontrol.go +++ b/vendor/github.com/moby/buildkit/util/flightcontrol/flightcontrol.go @@ -130,7 +130,11 @@ func (c *call) wait(ctx context.Context) (v interface{}, err error) { c.mu.Lock() // detect case where caller has just returned, let it clean up before select { - case <-c.ready: // could return if no error + case <-c.ready: + c.mu.Unlock() + <-c.cleaned + return nil, errRetry + case <-c.ctx.done: // could return if no error c.mu.Unlock() <-c.cleaned return nil, errRetry @@ -141,6 +145,10 @@ func (c *call) wait(ctx context.Context) (v interface{}, err error) { if ok { c.progressState.add(pw) } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + c.ctxs = append(c.ctxs, ctx) c.mu.Unlock() @@ -149,18 +157,16 @@ func (c *call) wait(ctx context.Context) (v interface{}, err error) { select { case <-ctx.Done(): - select { - case <-c.ctx.Done(): + if c.ctx.checkDone() { // if this cancelled the last context, then wait for function to shut down // and don't accept any more callers <-c.ready return c.result, c.err - default: - if ok { - c.progressState.close(pw) - } - return nil, ctx.Err() } + if ok { + c.progressState.close(pw) + } + return nil, ctx.Err() case <-c.ready: return c.result, c.err // shared not implemented yet } @@ -183,9 +189,6 @@ func (c *call) Deadline() (deadline time.Time, ok bool) { } func (c *call) Done() <-chan struct{} { - c.mu.Lock() - c.ctx.signal() - c.mu.Unlock() return c.ctx.done } @@ -238,23 +241,28 @@ func newContext(c *call) *sharedContext { return &sharedContext{call: c, done: make(chan struct{})} } -// call with lock -func (c *sharedContext) signal() { +func (sc *sharedContext) checkDone() bool { + sc.mu.Lock() select { - case <-c.done: + case <-sc.done: + sc.mu.Unlock() + return true default: - var err error - for _, ctx := range c.ctxs { - select { - case <-ctx.Done(): - err = ctx.Err() - default: - return - } - } - c.err = err - close(c.done) } + var err error + for _, ctx := range sc.ctxs { + select { + case <-ctx.Done(): + err = ctx.Err() + default: + sc.mu.Unlock() + return false + } + } + sc.err = err + close(sc.done) + sc.mu.Unlock() + return true } type rawProgressWriter interface { diff --git a/vendor/github.com/moby/buildkit/util/sshutil/transport_validation.go b/vendor/github.com/moby/buildkit/util/sshutil/transport_validation.go new file mode 100644 index 0000000000..f00a1df223 --- /dev/null +++ b/vendor/github.com/moby/buildkit/util/sshutil/transport_validation.go @@ -0,0 +1,11 @@ +package sshutil + +import ( + "regexp" +) + +var gitSSHRegex = regexp.MustCompile("^[a-zA-Z0-9-_]+@[a-zA-Z0-9-.]+:.*$") + +func IsSSHTransport(s string) bool { + return gitSSHRegex.MatchString(s) +}