From bc5484d2dd5039e4fcb6774ae568002fd04efc7a Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 2 Oct 2019 17:16:29 -0700 Subject: [PATCH] bump moby/buildkit f7042823e340d38d1746aa675b83d1aca431cee3 full diff: https://github.com/moby/buildkit/compare/588c73e1e4f0f3d7d3738abaaa7cf8026064b33e...f7042823e340d38d1746aa675b83d1aca431cee3 Signed-off-by: Sebastiaan van Stijn fix daemon for changes in containerd registry configuration Signed-off-by: Evan Hazlett Update buildernext and daemon for buildkit update Signed-off-by: Derek McGowan --- .../adapters/localinlinecache/inlinecache.go | 4 +- builder/builder-next/controller.go | 2 +- builder/builder-next/worker/worker.go | 13 ++-- daemon/daemon.go | 9 ++- vendor.conf | 2 +- vendor/github.com/moby/buildkit/README.md | 2 +- .../moby/buildkit/cache/remotecache/import.go | 22 +++++++ .../cache/remotecache/registry/registry.go | 37 ++++++++++- .../executor/runcexecutor/executor.go | 9 ++- .../dockerfile/dockerfile2llb/convert.go | 13 +++- .../dockerfile2llb/convert_norunnetwork.go | 12 ++++ .../dockerfile2llb/convert_norunsecurity.go | 5 +- .../dockerfile2llb/convert_runnetwork.go | 26 ++++++++ .../dockerfile2llb/convert_runsecurity.go | 21 +++---- .../instructions/commands_runnetwork.go | 63 +++++++++++++++++++ .../instructions/commands_runsecurity.go | 44 ++++--------- vendor/github.com/moby/buildkit/go.mod | 42 +++++++------ .../moby/buildkit/session/sshforward/copy.go | 7 ++- .../moby/buildkit/session/sshforward/ssh.go | 2 +- .../github.com/moby/buildkit/solver/edge.go | 3 + .../buildkit/solver/internal/pipe/pipe.go | 14 +++-- .../moby/buildkit/solver/llbsolver/bridge.go | 4 +- .../moby/buildkit/source/http/httpsource.go | 22 +++++++ .../moby/buildkit/util/leaseutil/manager.go | 52 --------------- .../moby/buildkit/util/resolver/resolver.go | 15 ++++- .../github.com/moby/buildkit/worker/worker.go | 2 + 26 files changed, 296 insertions(+), 151 deletions(-) create mode 100644 vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunnetwork.go create mode 100644 vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runnetwork.go create mode 100644 vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_runnetwork.go diff --git a/builder/builder-next/adapters/localinlinecache/inlinecache.go b/builder/builder-next/adapters/localinlinecache/inlinecache.go index 9ed2d53382..d14296028f 100644 --- a/builder/builder-next/adapters/localinlinecache/inlinecache.go +++ b/builder/builder-next/adapters/localinlinecache/inlinecache.go @@ -23,9 +23,9 @@ import ( ) // ResolveCacheImporterFunc returns a resolver function for local inline cache -func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc, rs reference.Store, is imagestore.Store) remotecache.ResolveCacheImporterFunc { +func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc, cs content.Store, rs reference.Store, is imagestore.Store) remotecache.ResolveCacheImporterFunc { - upstream := registryremotecache.ResolveCacheImporterFunc(sm, resolverOpt) + upstream := registryremotecache.ResolveCacheImporterFunc(sm, cs, resolverOpt) return func(ctx context.Context, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) { if dt, err := tryImportLocal(rs, is, attrs["ref"]); err == nil { diff --git a/builder/builder-next/controller.go b/builder/builder-next/controller.go index 62a6f09767..1650dde79e 100644 --- a/builder/builder-next/controller.go +++ b/builder/builder-next/controller.go @@ -189,7 +189,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) { Frontends: frontends, CacheKeyStorage: cacheStorage, ResolveCacheImporterFuncs: map[string]remotecache.ResolveCacheImporterFunc{ - "registry": localinlinecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt, dist.ReferenceStore, dist.ImageStore), + "registry": localinlinecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt, store, dist.ReferenceStore, dist.ImageStore), "local": localremotecache.ResolveCacheImporterFunc(opt.SessionManager), }, ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{ diff --git a/builder/builder-next/worker/worker.go b/builder/builder-next/worker/worker.go index a47ded37f4..b823c9abf2 100644 --- a/builder/builder-next/worker/worker.go +++ b/builder/builder-next/worker/worker.go @@ -151,6 +151,11 @@ func (w *Worker) GCPolicy() []client.PruneInfo { return w.Opt.GCPolicy } +// ContentStore returns content store +func (w *Worker) ContentStore() content.Store { + return w.Opt.ContentStore +} + // LoadRef loads a reference by ID func (w *Worker) LoadRef(id string, hidden bool) (cache.ImmutableRef, error) { var opts []cache.RefOption @@ -322,7 +327,7 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I defer func() { for _, l := range rootfs { - w.ContentStore.Delete(context.TODO(), l.Blob.Digest) + w.ContentStore().Delete(context.TODO(), l.Blob.Digest) } }() @@ -391,12 +396,12 @@ func (ld *layerDescriptor) DiffID() (layer.DiffID, error) { func (ld *layerDescriptor) Download(ctx context.Context, progressOutput pkgprogress.Output) (io.ReadCloser, int64, error) { done := oneOffProgress(ld.pctx, fmt.Sprintf("pulling %s", ld.desc.Digest)) - if err := contentutil.Copy(ctx, ld.w.ContentStore, ld.provider, ld.desc); err != nil { + if err := contentutil.Copy(ctx, ld.w.ContentStore(), ld.provider, ld.desc); err != nil { return nil, 0, done(err) } _ = done(nil) - ra, err := ld.w.ContentStore.ReaderAt(ctx, ld.desc) + ra, err := ld.w.ContentStore().ReaderAt(ctx, ld.desc) if err != nil { return nil, 0, err } @@ -405,7 +410,7 @@ func (ld *layerDescriptor) Download(ctx context.Context, progressOutput pkgprogr } func (ld *layerDescriptor) Close() { - // ld.is.ContentStore.Delete(context.TODO(), ld.desc.Digest) + // ld.is.ContentStore().Delete(context.TODO(), ld.desc.Digest) } func (ld *layerDescriptor) Registered(diffID layer.DiffID) { diff --git a/daemon/daemon.go b/daemon/daemon.go index 6edcb1b5f8..6b3ab259ce 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -173,8 +173,9 @@ func (daemon *Daemon) NewResolveOptionsFunc() resolver.ResolveOptionsFunc { if uri, err := url.Parse(v); err == nil { v = uri.Host } + plainHTTP := true m[v] = resolver.RegistryConf{ - PlainHTTP: true, + PlainHTTP: &plainHTTP, } } def := docker.ResolverOptions{ @@ -193,12 +194,16 @@ func (daemon *Daemon) NewResolveOptionsFunc() resolver.ResolveOptionsFunc { } if len(c.Mirrors) > 0 { + // TODO ResolverOptions.Host is deprecated; ResolverOptions.Hosts should be used def.Host = func(string) (string, error) { return c.Mirrors[rand.Intn(len(c.Mirrors))], nil } } - def.PlainHTTP = c.PlainHTTP + // TODO ResolverOptions.PlainHTTP is deprecated; ResolverOptions.Hosts should be used + if c.PlainHTTP != nil { + def.PlainHTTP = *c.PlainHTTP + } return def } diff --git a/vendor.conf b/vendor.conf index c8fc505e9b..2b019699d9 100644 --- a/vendor.conf +++ b/vendor.conf @@ -26,7 +26,7 @@ github.com/imdario/mergo 7c29201646fa3de8506f70121347 golang.org/x/sync e225da77a7e68af35c70ccbf71af2b83e6acac3c # buildkit -github.com/moby/buildkit 588c73e1e4f0f3d7d3738abaaa7cf8026064b33e +github.com/moby/buildkit f7042823e340d38d1746aa675b83d1aca431cee3 github.com/tonistiigi/fsutil 3bbb99cdbd76619ab717299830c60f6f2a533a6b github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746 github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7 diff --git a/vendor/github.com/moby/buildkit/README.md b/vendor/github.com/moby/buildkit/README.md index 90dd0bee35..b75d4c7f80 100644 --- a/vendor/github.com/moby/buildkit/README.md +++ b/vendor/github.com/moby/buildkit/README.md @@ -38,7 +38,7 @@ BuildKit is used by the following projects: - [Knative Build Templates](https://github.com/knative/build-templates) - [the Sanic build tool](https://github.com/distributed-containers-inc/sanic) - [vab](https://github.com/stellarproject/vab) -- [Rio](https://github.com/rancher/rio) (on roadmap) +- [Rio](https://github.com/rancher/rio) ### Quick start diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/import.go b/vendor/github.com/moby/buildkit/cache/remotecache/import.go index 88223ff920..b19ad88092 100644 --- a/vendor/github.com/moby/buildkit/cache/remotecache/import.go +++ b/vendor/github.com/moby/buildkit/cache/remotecache/import.go @@ -27,6 +27,11 @@ type Importer interface { Resolve(ctx context.Context, desc ocispec.Descriptor, id string, w worker.Worker) (solver.CacheManager, error) } +type DistributionSourceLabelSetter interface { + SetDistributionSourceLabel(context.Context, digest.Digest) error + SetDistributionSourceAnnotation(desc ocispec.Descriptor) ocispec.Descriptor +} + func NewImporter(provider content.Provider) Importer { return &contentCacheImporter{provider: provider} } @@ -61,6 +66,15 @@ func (ci *contentCacheImporter) Resolve(ctx context.Context, desc ocispec.Descri } } + if dsls, ok := ci.provider.(DistributionSourceLabelSetter); ok { + for dgst, l := range allLayers { + err := dsls.SetDistributionSourceLabel(ctx, dgst) + _ = err // error ignored because layer may not exist + l.Descriptor = dsls.SetDistributionSourceAnnotation(l.Descriptor) + allLayers[dgst] = l + } + } + if configDesc.Digest == "" { return ci.importInlineCache(ctx, dt, id, w) } @@ -127,6 +141,14 @@ func (ci *contentCacheImporter) importInlineCache(ctx context.Context, dt []byte return nil } + if dsls, ok := ci.provider.(DistributionSourceLabelSetter); ok { + for i, l := range m.Layers { + err := dsls.SetDistributionSourceLabel(ctx, l.Digest) + _ = err // error ignored because layer may not exist + m.Layers[i] = dsls.SetDistributionSourceAnnotation(l) + } + } + p, err := content.ReadBlob(ctx, ci.provider, m.Config) if err != nil { return errors.WithStack(err) diff --git a/vendor/github.com/moby/buildkit/cache/remotecache/registry/registry.go b/vendor/github.com/moby/buildkit/cache/remotecache/registry/registry.go index 824d2f8173..51e97c7839 100644 --- a/vendor/github.com/moby/buildkit/cache/remotecache/registry/registry.go +++ b/vendor/github.com/moby/buildkit/cache/remotecache/registry/registry.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/containerd/containerd/content" "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" "github.com/docker/distribution/reference" @@ -12,6 +13,8 @@ import ( "github.com/moby/buildkit/session/auth" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/resolver" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -46,7 +49,7 @@ func ResolveCacheExporterFunc(sm *session.Manager, resolverOpt resolver.ResolveO } } -func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheImporterFunc { +func ResolveCacheImporterFunc(sm *session.Manager, cs content.Store, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheImporterFunc { return func(ctx context.Context, attrs map[string]string) (remotecache.Importer, specs.Descriptor, error) { ref, err := canonicalizeRef(attrs[attrRef]) if err != nil { @@ -61,10 +64,40 @@ func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveO if err != nil { return nil, specs.Descriptor{}, err } - return remotecache.NewImporter(contentutil.FromFetcher(fetcher)), desc, nil + src := &withDistributionSourceLabel{ + Provider: contentutil.FromFetcher(fetcher), + ref: ref, + source: cs, + } + return remotecache.NewImporter(src), desc, nil } } +type withDistributionSourceLabel struct { + content.Provider + ref string + source content.Manager +} + +var _ remotecache.DistributionSourceLabelSetter = &withDistributionSourceLabel{} + +func (dsl *withDistributionSourceLabel) SetDistributionSourceLabel(ctx context.Context, dgst digest.Digest) error { + hf, err := docker.AppendDistributionSourceLabel(dsl.source, dsl.ref) + if err != nil { + return err + } + _, err = hf(ctx, ocispec.Descriptor{Digest: dgst}) + return err +} + +func (dsl *withDistributionSourceLabel) SetDistributionSourceAnnotation(desc ocispec.Descriptor) ocispec.Descriptor { + if desc.Annotations == nil { + desc.Annotations = map[string]string{} + } + desc.Annotations["containerd.io/distribution.source.ref"] = dsl.ref + return desc +} + func newRemoteResolver(ctx context.Context, resolverOpt resolver.ResolveOptionsFunc, sm *session.Manager, ref string) remotes.Resolver { opt := resolverOpt(ref) opt.Credentials = getCredentialsFunc(ctx, sm) diff --git a/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go b/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go index 15712fc1c4..26e432e61f 100644 --- a/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go +++ b/vendor/github.com/moby/buildkit/executor/runcexecutor/executor.go @@ -293,12 +293,11 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache. NoPivot: w.noPivot, }) close(done) - if err != nil { - return err - } - if status != 0 { - err := errors.Errorf("exit code: %d", status) + if status != 0 || err != nil { + if err == nil { + err = errors.Errorf("exit code: %d", status) + } select { case <-ctx.Done(): return errors.Wrapf(ctx.Err(), err.Error()) diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go index 8cec181ed5..840520e8cc 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert.go @@ -640,10 +640,21 @@ func dispatchRun(d *dispatchState, c *instructions.RunCommand, proxy *llb.ProxyE } opt = append(opt, runMounts...) - err = dispatchRunSecurity(d, c) + securityOpt, err := dispatchRunSecurity(c) if err != nil { return err } + if securityOpt != nil { + opt = append(opt, securityOpt) + } + + networkOpt, err := dispatchRunNetwork(c) + if err != nil { + return err + } + if networkOpt != nil { + opt = append(opt, networkOpt) + } shlex := *dopt.shlex shlex.RawQuotes = true diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunnetwork.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunnetwork.go new file mode 100644 index 0000000000..300b9d85f1 --- /dev/null +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunnetwork.go @@ -0,0 +1,12 @@ +// +build !dfrunnetwork + +package dockerfile2llb + +import ( + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/frontend/dockerfile/instructions" +) + +func dispatchRunNetwork(c *instructions.RunCommand) (llb.RunOption, error) { + return nil, nil +} diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunsecurity.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunsecurity.go index bc37ff43c8..10184b42e0 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunsecurity.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_norunsecurity.go @@ -3,9 +3,10 @@ package dockerfile2llb import ( + "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend/dockerfile/instructions" ) -func dispatchRunSecurity(d *dispatchState, c *instructions.RunCommand) error { - return nil +func dispatchRunSecurity(c *instructions.RunCommand) (llb.RunOption, error) { + return nil, nil } diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runnetwork.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runnetwork.go new file mode 100644 index 0000000000..01313e23be --- /dev/null +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runnetwork.go @@ -0,0 +1,26 @@ +// +build dfrunnetwork + +package dockerfile2llb + +import ( + "github.com/pkg/errors" + + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/moby/buildkit/solver/pb" +) + +func dispatchRunNetwork(c *instructions.RunCommand) (llb.RunOption, error) { + network := instructions.GetNetwork(c) + + switch network { + case instructions.NetworkDefault: + return nil, nil + case instructions.NetworkNone: + return llb.Network(pb.NetMode_NONE), nil + case instructions.NetworkHost: + return llb.Network(pb.NetMode_HOST), nil + default: + return nil, errors.Errorf("unsupported network mode %q", network) + } +} diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runsecurity.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runsecurity.go index 7b1f099464..764424a2c1 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runsecurity.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb/convert_runsecurity.go @@ -5,23 +5,20 @@ package dockerfile2llb import ( "github.com/pkg/errors" + "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/solver/pb" ) -func dispatchRunSecurity(d *dispatchState, c *instructions.RunCommand) error { +func dispatchRunSecurity(c *instructions.RunCommand) (llb.RunOption, error) { security := instructions.GetSecurity(c) - for _, sec := range security { - switch sec { - case instructions.SecurityInsecure: - d.state = d.state.Security(pb.SecurityMode_INSECURE) - case instructions.SecuritySandbox: - d.state = d.state.Security(pb.SecurityMode_SANDBOX) - default: - return errors.Errorf("unsupported security mode %q", sec) - } + switch security { + case instructions.SecurityInsecure: + return llb.Security(pb.SecurityMode_INSECURE), nil + case instructions.SecuritySandbox: + return llb.Security(pb.SecurityMode_SANDBOX), nil + default: + return nil, errors.Errorf("unsupported security mode %q", security) } - - return nil } diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_runnetwork.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_runnetwork.go new file mode 100644 index 0000000000..adef3fd7aa --- /dev/null +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_runnetwork.go @@ -0,0 +1,63 @@ +// +build dfrunnetwork + +package instructions + +import ( + "github.com/pkg/errors" +) + +const ( + NetworkDefault = "default" + NetworkNone = "none" + NetworkHost = "host" +) + +var allowedNetwork = map[string]struct{}{ + NetworkDefault: {}, + NetworkNone: {}, + NetworkHost: {}, +} + +func isValidNetwork(value string) bool { + _, ok := allowedNetwork[value] + return ok +} + +var networkKey = "dockerfile/run/network" + +func init() { + parseRunPreHooks = append(parseRunPreHooks, runNetworkPreHook) + parseRunPostHooks = append(parseRunPostHooks, runNetworkPostHook) +} + +func runNetworkPreHook(cmd *RunCommand, req parseRequest) error { + st := &networkState{} + st.flag = req.flags.AddString("network", NetworkDefault) + cmd.setExternalValue(networkKey, st) + return nil +} + +func runNetworkPostHook(cmd *RunCommand, req parseRequest) error { + st := cmd.getExternalValue(networkKey).(*networkState) + if st == nil { + return errors.Errorf("no network state") + } + + value := st.flag.Value + if !isValidNetwork(value) { + return errors.Errorf("invalid network mode %q", value) + } + + st.networkMode = value + + return nil +} + +func GetNetwork(cmd *RunCommand) string { + return cmd.getExternalValue(networkKey).(*networkState).networkMode +} + +type networkState struct { + flag *Flag + networkMode string +} diff --git a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_runsecurity.go b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_runsecurity.go index b83b6f2f85..0c0be80664 100644 --- a/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_runsecurity.go +++ b/vendor/github.com/moby/buildkit/frontend/dockerfile/instructions/commands_runsecurity.go @@ -3,9 +3,6 @@ package instructions import ( - "encoding/csv" - "strings" - "github.com/pkg/errors" ) @@ -24,9 +21,7 @@ func isValidSecurity(value string) bool { return ok } -type securityKeyT string - -var securityKey = securityKeyT("dockerfile/run/security") +var securityKey = "dockerfile/run/security" func init() { parseRunPreHooks = append(parseRunPreHooks, runSecurityPreHook) @@ -35,49 +30,32 @@ func init() { func runSecurityPreHook(cmd *RunCommand, req parseRequest) error { st := &securityState{} - st.flag = req.flags.AddStrings("security") + st.flag = req.flags.AddString("security", SecuritySandbox) cmd.setExternalValue(securityKey, st) return nil } func runSecurityPostHook(cmd *RunCommand, req parseRequest) error { - st := getSecurityState(cmd) + st := cmd.getExternalValue(securityKey).(*securityState) if st == nil { return errors.Errorf("no security state") } - for _, value := range st.flag.StringValues { - csvReader := csv.NewReader(strings.NewReader(value)) - fields, err := csvReader.Read() - if err != nil { - return errors.Wrap(err, "failed to parse csv security") - } - - for _, field := range fields { - if !isValidSecurity(field) { - return errors.Errorf("security %q is not valid", field) - } - - st.security = append(st.security, field) - } + value := st.flag.Value + if !isValidSecurity(value) { + return errors.Errorf("security %q is not valid", value) } + st.security = value + return nil } -func getSecurityState(cmd *RunCommand) *securityState { - v := cmd.getExternalValue(securityKey) - if v == nil { - return nil - } - return v.(*securityState) -} - -func GetSecurity(cmd *RunCommand) []string { - return getSecurityState(cmd).security +func GetSecurity(cmd *RunCommand) string { + return cmd.getExternalValue(securityKey).(*securityState).security } type securityState struct { flag *Flag - security []string + security string } diff --git a/vendor/github.com/moby/buildkit/go.mod b/vendor/github.com/moby/buildkit/go.mod index 7f76ec2558..c2b385b359 100644 --- a/vendor/github.com/moby/buildkit/go.mod +++ b/vendor/github.com/moby/buildkit/go.mod @@ -4,20 +4,20 @@ go 1.11 require ( github.com/BurntSushi/toml v0.3.1 - github.com/Microsoft/go-winio v0.4.13-0.20190408173621-84b4ab48a507 + github.com/Microsoft/go-winio v0.4.14 github.com/apache/thrift v0.0.0-20161221203622-b2a4d4ae21c7 // indirect github.com/codahale/hdrhistogram v0.0.0-20160425231609-f8ad88b59a58 // indirect - github.com/containerd/cgroups v0.0.0-20190226200435-dbea6f2bd416 // indirect + github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601 // indirect github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50 - github.com/containerd/containerd v1.3.0-0.20190507210959-7c1e88399ec0 - github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc - github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260 // indirect - github.com/containerd/go-cni v0.0.0-20190610170741-5a4663dad645 - github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3 - github.com/containerd/ttrpc v0.0.0-20190411181408-699c4e40d1e7 // indirect + github.com/containerd/containerd v1.3.0 + github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 + github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c // indirect + github.com/containerd/go-cni v0.0.0-20190813230227-49fbd9b210f3 + github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda + github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8 // indirect github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd // indirect - github.com/containernetworking/cni v0.6.1-0.20180218032124-142cde0c766c // indirect - github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e + github.com/containernetworking/cni v0.7.1 // indirect + github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a github.com/docker/cli v0.0.0-20190321234815-f40f9c240ab0 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible github.com/docker/docker v1.14.0-0.20190319215453-e7b5f7dbe98c @@ -25,17 +25,18 @@ require ( github.com/docker/go-connections v0.3.0 github.com/docker/go-events v0.0.0-20170721190031-9461782956ad // indirect github.com/docker/libnetwork v0.8.0-dev.2.0.20190604151032-3c26b4e7495e - github.com/godbus/dbus v4.1.0+incompatible // indirect + github.com/godbus/dbus v0.0.0-20181101234600-2ff6f7ffd60f // indirect github.com/gofrs/flock v0.7.0 github.com/gogo/googleapis v1.1.0 github.com/gogo/protobuf v1.2.0 github.com/golang/protobuf v1.2.0 - github.com/google/go-cmp v0.2.0 + github.com/google/go-cmp v0.3.0 github.com/google/shlex v0.0.0-20150127133951-6f45313302b9 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 github.com/hashicorp/go-immutable-radix v1.0.0 github.com/hashicorp/golang-lru v0.0.0-20160207214719-a0d98a5f2880 github.com/hashicorp/uuid v0.0.0-20160311170451-ebb0a03e909c // indirect + github.com/imdario/mergo v0.3.7 // indirect github.com/ishidawataru/sctp v0.0.0-20180213033435-07191f837fed // indirect github.com/jaguilar/vt100 v0.0.0-20150826170717-2703a27b14ea github.com/kr/pretty v0.1.0 // indirect @@ -43,31 +44,32 @@ require ( github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 - github.com/opencontainers/runc v1.0.0-rc8 + github.com/opencontainers/runc v1.0.0-rc8.0.20190621203724-f4982d86f7fd github.com/opencontainers/runtime-spec v0.0.0-20180909173843-eba862dc2470 github.com/opentracing-contrib/go-stdlib v0.0.0-20171029140428-b1a47cfbdd75 github.com/opentracing/opentracing-go v0.0.0-20171003133519-1361b9cd60be github.com/pkg/errors v0.8.1 github.com/pkg/profile v1.2.1 + github.com/prometheus/procfs v0.0.3 // indirect github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002 - github.com/sirupsen/logrus v1.3.0 + github.com/sirupsen/logrus v1.4.1 github.com/stretchr/testify v1.3.0 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect - github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76 + github.com/tonistiigi/fsutil v0.0.0-20190819224149-3d2716dd0a4d github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea github.com/uber/jaeger-client-go v0.0.0-20180103221425-e02c85f9069e github.com/uber/jaeger-lib v1.2.1 // indirect github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5 github.com/vishvananda/netlink v1.0.0 // indirect github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect - go.etcd.io/bbolt v1.3.2 + go.etcd.io/bbolt v1.3.3 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 - golang.org/x/net v0.0.0-20190311183353-d8887717615a - golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f - golang.org/x/sys v0.0.0-20190303122642-d455e41777fc + golang.org/x/net v0.0.0-20190522155817-f3200d17e092 + golang.org/x/sync v0.0.0-20190423024810-112230192c58 + golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e golang.org/x/time v0.0.0-20161028155119-f51c12702a4d google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 - google.golang.org/grpc v1.20.1 + google.golang.org/grpc v1.23.0 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gotest.tools v2.2.0+incompatible ) diff --git a/vendor/github.com/moby/buildkit/session/sshforward/copy.go b/vendor/github.com/moby/buildkit/session/sshforward/copy.go index c2763fa452..85366f19af 100644 --- a/vendor/github.com/moby/buildkit/session/sshforward/copy.go +++ b/vendor/github.com/moby/buildkit/session/sshforward/copy.go @@ -9,17 +9,17 @@ import ( "google.golang.org/grpc" ) -func Copy(ctx context.Context, conn io.ReadWriteCloser, stream grpc.Stream) error { +func Copy(ctx context.Context, conn io.ReadWriteCloser, stream grpc.Stream, closeStream func() error) error { g, ctx := errgroup.WithContext(ctx) g.Go(func() (retErr error) { p := &BytesMessage{} for { if err := stream.RecvMsg(p); err != nil { + conn.Close() if err == io.EOF { return nil } - conn.Close() return errors.WithStack(err) } select { @@ -42,6 +42,9 @@ func Copy(ctx context.Context, conn io.ReadWriteCloser, stream grpc.Stream) erro n, err := conn.Read(buf) switch { case err == io.EOF: + if closeStream != nil { + closeStream() + } return nil case err != nil: return errors.WithStack(err) diff --git a/vendor/github.com/moby/buildkit/session/sshforward/ssh.go b/vendor/github.com/moby/buildkit/session/sshforward/ssh.go index 660e89f7f1..0001f59b5f 100644 --- a/vendor/github.com/moby/buildkit/session/sshforward/ssh.go +++ b/vendor/github.com/moby/buildkit/session/sshforward/ssh.go @@ -49,7 +49,7 @@ func (s *server) run(ctx context.Context, l net.Listener, id string) error { return err } - go Copy(ctx, conn, stream) + go Copy(ctx, conn, stream, stream.CloseSend) } }) diff --git a/vendor/github.com/moby/buildkit/solver/edge.go b/vendor/github.com/moby/buildkit/solver/edge.go index b809652c47..0a076a2168 100644 --- a/vendor/github.com/moby/buildkit/solver/edge.go +++ b/vendor/github.com/moby/buildkit/solver/edge.go @@ -177,6 +177,9 @@ func (e *edge) finishIncoming(req pipe.Sender) { // updateIncoming updates the current value of incoming pipe request func (e *edge) updateIncoming(req pipe.Sender) { + if debugScheduler { + logrus.Debugf("updateIncoming %s %#v desired=%s", e.edge.Vertex.Name(), e.edgeState, req.Request().Payload.(*edgeRequest).desiredState) + } req.Update(&e.edgeState) } diff --git a/vendor/github.com/moby/buildkit/solver/internal/pipe/pipe.go b/vendor/github.com/moby/buildkit/solver/internal/pipe/pipe.go index a302e3a137..81702f5236 100644 --- a/vendor/github.com/moby/buildkit/solver/internal/pipe/pipe.go +++ b/vendor/github.com/moby/buildkit/solver/internal/pipe/pipe.go @@ -11,11 +11,15 @@ import ( type channel struct { OnSendCompletion func() value atomic.Value - lastValue interface{} + lastValue *wrappedValue +} + +type wrappedValue struct { + value interface{} } func (c *channel) Send(v interface{}) { - c.value.Store(v) + c.value.Store(&wrappedValue{value: v}) if c.OnSendCompletion != nil { c.OnSendCompletion() } @@ -23,11 +27,11 @@ func (c *channel) Send(v interface{}) { func (c *channel) Receive() (interface{}, bool) { v := c.value.Load() - if c.lastValue == v { + if v == nil || v.(*wrappedValue) == c.lastValue { return nil, false } - c.lastValue = v - return v, true + c.lastValue = v.(*wrappedValue) + return v.(*wrappedValue).value, true } type Pipe struct { diff --git a/vendor/github.com/moby/buildkit/solver/llbsolver/bridge.go b/vendor/github.com/moby/buildkit/solver/llbsolver/bridge.go index 42f2a8ce4f..0905f98325 100644 --- a/vendor/github.com/moby/buildkit/solver/llbsolver/bridge.go +++ b/vendor/github.com/moby/buildkit/solver/llbsolver/bridge.go @@ -54,7 +54,7 @@ func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest) (res * } } if prevCm, ok := b.cms[cmId]; !ok { - func(cmId string) { + func(cmId string, im gw.CacheOptionsEntry) { cm = newLazyCacheManager(cmId, func() (solver.CacheManager, error) { var cmNew solver.CacheManager if err := inVertexContext(b.builder.Context(ctx), "importing cache manifest from "+cmId, "", func(ctx context.Context) error { @@ -74,7 +74,7 @@ func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest) (res * } return cmNew, nil }) - }(cmId) + }(cmId, im) b.cms[cmId] = cm } else { cm = prevCm diff --git a/vendor/github.com/moby/buildkit/source/http/httpsource.go b/vendor/github.com/moby/buildkit/source/http/httpsource.go index 7394a03e0a..9bde6195cb 100644 --- a/vendor/github.com/moby/buildkit/source/http/httpsource.go +++ b/vendor/github.com/moby/buildkit/source/http/httpsource.go @@ -144,6 +144,11 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, b req = req.WithContext(ctx) m := map[string]*metadata.StorageItem{} + // If we request a single ETag in 'If-None-Match', some servers omit the + // unambiguous ETag in their response. + // See: https://github.com/moby/buildkit/issues/905 + var onlyETag string + if len(sis) > 0 { for _, si := range sis { // if metaDigest := getMetaDigest(si); metaDigest == hs.formatCacheKey("") { @@ -160,6 +165,10 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, b etags = append(etags, t) } req.Header.Add("If-None-Match", strings.Join(etags, ", ")) + + if len(etags) == 1 { + onlyETag = etags[0] + } } } @@ -172,6 +181,12 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, b if err == nil { if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNotModified { respETag := resp.Header.Get("ETag") + + // If a 304 is returned without an ETag and we had only sent one ETag, + // the response refers to the ETag we asked about. + if respETag == "" && onlyETag != "" && resp.StatusCode == http.StatusNotModified { + respETag = onlyETag + } si, ok := m[respETag] if ok { hs.refID = si.ID() @@ -197,6 +212,13 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, index int) (string, b } if resp.StatusCode == http.StatusNotModified { respETag := resp.Header.Get("ETag") + if respETag == "" && onlyETag != "" { + respETag = onlyETag + + // Set the missing ETag header on the response so that it's available + // to .save() + resp.Header.Set("ETag", onlyETag) + } si, ok := m[respETag] if !ok { return "", false, errors.Errorf("invalid not-modified ETag: %v", respETag) diff --git a/vendor/github.com/moby/buildkit/util/leaseutil/manager.go b/vendor/github.com/moby/buildkit/util/leaseutil/manager.go index ced6b9bf07..fe363e0a02 100644 --- a/vendor/github.com/moby/buildkit/util/leaseutil/manager.go +++ b/vendor/github.com/moby/buildkit/util/leaseutil/manager.go @@ -5,9 +5,7 @@ import ( "time" "github.com/containerd/containerd/leases" - "github.com/containerd/containerd/metadata" "github.com/containerd/containerd/namespaces" - bolt "go.etcd.io/bbolt" ) func WithLease(ctx context.Context, ls leases.Manager, opts ...leases.Opt) (context.Context, func(context.Context) error, error) { @@ -29,56 +27,6 @@ func WithLease(ctx context.Context, ls leases.Manager, opts ...leases.Opt) (cont }, nil } -func NewManager(mdb *metadata.DB) leases.Manager { - return &local{db: mdb} -} - -type local struct { - db *metadata.DB -} - -func (l *local) Create(ctx context.Context, opts ...leases.Opt) (leases.Lease, error) { - var lease leases.Lease - if err := l.db.Update(func(tx *bolt.Tx) error { - var err error - lease, err = metadata.NewLeaseManager(tx).Create(ctx, opts...) - return err - }); err != nil { - return leases.Lease{}, err - } - return lease, nil -} - -func (l *local) Delete(ctx context.Context, lease leases.Lease, opts ...leases.DeleteOpt) error { - var do leases.DeleteOptions - for _, opt := range opts { - if err := opt(ctx, &do); err != nil { - return err - } - } - - if err := l.db.Update(func(tx *bolt.Tx) error { - return metadata.NewLeaseManager(tx).Delete(ctx, lease) - }); err != nil { - return err - } - - return nil - -} - -func (l *local) List(ctx context.Context, filters ...string) ([]leases.Lease, error) { - var ll []leases.Lease - if err := l.db.View(func(tx *bolt.Tx) error { - var err error - ll, err = metadata.NewLeaseManager(tx).List(ctx, filters...) - return err - }); err != nil { - return nil, err - } - return ll, nil -} - func WithNamespace(lm leases.Manager, ns string) leases.Manager { return &nsLM{Manager: lm, ns: ns} } diff --git a/vendor/github.com/moby/buildkit/util/resolver/resolver.go b/vendor/github.com/moby/buildkit/util/resolver/resolver.go index da28923564..096bfc330b 100644 --- a/vendor/github.com/moby/buildkit/util/resolver/resolver.go +++ b/vendor/github.com/moby/buildkit/util/resolver/resolver.go @@ -2,6 +2,7 @@ package resolver import ( "math/rand" + "strings" "github.com/containerd/containerd/remotes/docker" "github.com/docker/distribution/reference" @@ -10,7 +11,7 @@ import ( type RegistryConf struct { Mirrors []string - PlainHTTP bool + PlainHTTP *bool } type ResolveOptionsFunc func(string) docker.ResolverOptions @@ -32,13 +33,21 @@ func NewResolveOptionsFunc(m map[string]RegistryConf) ResolveOptionsFunc { return def } + var mirrorHost string if len(c.Mirrors) > 0 { + mirrorHost = c.Mirrors[rand.Intn(len(c.Mirrors))] def.Host = func(string) (string, error) { - return c.Mirrors[rand.Intn(len(c.Mirrors))], nil + return mirrorHost, nil } } - def.PlainHTTP = c.PlainHTTP + if c.PlainHTTP != nil { + def.PlainHTTP = *c.PlainHTTP + } else { + if mirrorHost == "localhost" || strings.HasPrefix(mirrorHost, "localhost:") { + def.PlainHTTP = true + } + } return def } diff --git a/vendor/github.com/moby/buildkit/worker/worker.go b/vendor/github.com/moby/buildkit/worker/worker.go index 38cc7db039..2c3c616ec2 100644 --- a/vendor/github.com/moby/buildkit/worker/worker.go +++ b/vendor/github.com/moby/buildkit/worker/worker.go @@ -4,6 +4,7 @@ import ( "context" "io" + "github.com/containerd/containerd/content" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/client" "github.com/moby/buildkit/executor" @@ -34,6 +35,7 @@ type Worker interface { GetRemote(ctx context.Context, ref cache.ImmutableRef, createIfNeeded bool) (*solver.Remote, error) FromRemote(ctx context.Context, remote *solver.Remote) (cache.ImmutableRef, error) PruneCacheMounts(ctx context.Context, ids []string) error + ContentStore() content.Store } // Pre-defined label keys