kotovalexarian-likes-github/moby--moby
1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00
Code Releases Activity

builder-next: fixes for updated buildkit

Browse source 
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi 2019-02-27 22:35:26 -08:00
parent 4337e97f25
commit 7ac4e33e63
6 changed files with 62 additions and 38 deletions
builder/builder-next
adapters/containerimage
pull.go
builder.gocontroller.go
exporter
export.gowriter.go
worker
worker.go

29
builder/builder-next/adapters/containerimage/pull.go
View file

@ -45,7 +45,6 @@ import (
// SourceOpt is options for creating the image source
type SourceOpt struct {
SessionManager *session.Manager
ContentStore content.Store
CacheAccessor cache.Accessor
ReferenceStore reference.Store
@ -73,19 +72,19 @@ func (is *imageSource) ID() string {
return source.DockerImageScheme
}
func (is *imageSource) getResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, ref string) remotes.Resolver {
func (is *imageSource) getResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, ref string, sm *session.Manager) remotes.Resolver {
opt := docker.ResolverOptions{
Client: tracing.DefaultClient,
}
if rfn != nil {
opt = rfn(ref)
}
opt.Credentials = is.getCredentialsFromSession(ctx)
opt.Credentials = is.getCredentialsFromSession(ctx, sm)
r := docker.NewResolver(opt)
return r
}
func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(string) (string, string, error) {
func (is *imageSource) getCredentialsFromSession(ctx context.Context, sm *session.Manager) func(string) (string, string, error) {
id := session.FromContext(ctx)
if id == "" {
// can be removed after containerd/containerd#2812
@ -97,7 +96,7 @@ func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(strin
timeoutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
caller, err := is.SessionManager.Get(timeoutCtx, id)
caller, err := sm.Get(timeoutCtx, id)
if err != nil {
return "", "", err
}
@ -122,13 +121,13 @@ func (is *imageSource) resolveLocal(refStr string) ([]byte, error) {
return img.RawJSON(), nil
}
func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *ocispec.Platform) (digest.Digest, []byte, error) {
func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *ocispec.Platform, sm *session.Manager) (digest.Digest, []byte, error) {
type t struct {
dgst digest.Digest
dt []byte
}
res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref), is.ContentStore, platform)
dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref, sm), is.ContentStore, platform)
if err != nil {
return nil, err
}
@ -142,14 +141,14 @@ func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *
return typed.dgst, typed.dt, nil
}
func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt) (digest.Digest, []byte, error) {
func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt, sm *session.Manager) (digest.Digest, []byte, error) {
resolveMode, err := source.ParseImageResolveMode(opt.ResolveMode)
if err != nil {
return "", nil, err
}
switch resolveMode {
case source.ResolveModeForcePull:
dgst, dt, err := is.resolveRemote(ctx, ref, opt.Platform)
dgst, dt, err := is.resolveRemote(ctx, ref, opt.Platform, sm)
// TODO: pull should fallback to local in case of failure to allow offline behavior
// the fallback doesn't work currently
return dgst, dt, err
@ -171,13 +170,13 @@ func (is *imageSource) ResolveImageConfig(ctx context.Context, ref string, opt g
return "", dt, err
}
// fallback to remote
return is.resolveRemote(ctx, ref, opt.Platform)
return is.resolveRemote(ctx, ref, opt.Platform, sm)
}
// should never happen
return "", nil, fmt.Errorf("builder cannot resolve image %s: invalid mode %q", ref, opt.ResolveMode)
}
func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (source.SourceInstance, error) {
func (is *imageSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager) (source.SourceInstance, error) {
imageIdentifier, ok := id.(*source.ImageIdentifier)
if !ok {
return nil, errors.Errorf("invalid image identifier %v", id)
@ -191,8 +190,9 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (sourc
p := &puller{
src: imageIdentifier,
is: is,
resolver: is.getResolver(ctx, is.ResolverOpt, imageIdentifier.Reference.String()),
resolver: is.getResolver(ctx, is.ResolverOpt, imageIdentifier.Reference.String(), sm),
platform: platform,
sm: sm,
}
return p, nil
}
@ -208,6 +208,7 @@ type puller struct {
resolver remotes.Resolver
config []byte
platform ocispec.Platform
sm *session.Manager
}
func (p *puller) mainManifestKey(dgst digest.Digest, platform ocispec.Platform) (digest.Digest, error) {
@ -294,7 +295,7 @@ func (p *puller) resolve(ctx context.Context) error {
resolveProgressDone(err)
return
}
_, dt, err := p.is.ResolveImageConfig(ctx, ref.String(), gw.ResolveImageConfigOpt{Platform: &p.platform, ResolveMode: resolveModeToString(p.src.ResolveMode)})
_, dt, err := p.is.ResolveImageConfig(ctx, ref.String(), gw.ResolveImageConfigOpt{Platform: &p.platform, ResolveMode: resolveModeToString(p.src.ResolveMode)}, p.sm)
if err != nil {
p.resolveErr = err
resolveProgressDone(err)
@ -420,7 +421,7 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
)
}
if err := images.Dispatch(ctx, images.Handlers(handlers...), p.desc); err != nil {
if err := images.Dispatch(ctx, images.Handlers(handlers...), nil, p.desc); err != nil {
stopProgress()
return nil, err
}

12
builder/builder-next/builder.go
View file

@ -5,6 +5,7 @@ import (
"fmt"
"io"
"net"
"strconv"
"strings"
"sync"
"time"
@ -318,6 +319,16 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
exporterAttrs["name"] = strings.Join(opt.Options.Tags, ",")
}
cache := controlapi.CacheOptions{}
if inlineCache := opt.Options.BuildArgs["BUILDKIT_INLINE_CACHE"]; inlineCache != nil {
if b, err := strconv.ParseBool(*inlineCache); err == nil && b {
cache.Exports = append(cache.Exports, &controlapi.CacheOptionsEntry{
Type: "inline",
})
}
}
req := &controlapi.SolveRequest{
Ref: id,
Exporter: "moby",
@ -325,6 +336,7 @@ func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.
Frontend: "dockerfile.v0",
FrontendAttrs: frontendAttrs,
Session: opt.Options.SessionID,
Cache: cache,
}
if opt.Options.NetworkMode == "host" {

13
builder/builder-next/controller.go
View file

@ -18,10 +18,10 @@ import (
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/cache/remotecache"
inlineremotecache "github.com/moby/buildkit/cache/remotecache/inline"
registryremotecache "github.com/moby/buildkit/cache/remotecache/registry"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/control"
"github.com/moby/buildkit/exporter"
"github.com/moby/buildkit/frontend"
dockerfile "github.com/moby/buildkit/frontend/dockerfile/builder"
"github.com/moby/buildkit/frontend/gateway"
@ -95,7 +95,6 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
}
src, err := containerimage.NewSource(containerimage.SourceOpt{
SessionManager: opt.SessionManager,
CacheAccessor: cm,
ContentStore: store,
DownloadManager: dist.DownloadManager,
@ -139,7 +138,6 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
wopt := mobyworker.Opt{
ID: "moby",
SessionManager: opt.SessionManager,
MetadataStore: md,
ContentStore: store,
CacheManager: cm,
@ -149,10 +147,8 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
ImageSource: src,
DownloadManager: dist.DownloadManager,
V2MetadataService: dist.V2MetadataService,
Exporters: map[string]exporter.Exporter{
"moby": exp,
},
Transport: rt,
Exporter: exp,
Transport: rt,
}
wc := &worker.Controller{}
@ -175,6 +171,9 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
ResolveCacheImporterFuncs: map[string]remotecache.ResolveCacheImporterFunc{
"registry": registryremotecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt),
},
ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{
"inline": inlineremotecache.ResolveCacheExporterFunc(),
},
// TODO: set ResolveCacheExporterFunc for exporting cache
})
}

2
builder/builder-next/exporter/export.go
View file

@ -148,7 +148,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, inp exporter.Source)
diffs, history = normalizeLayersAndHistory(diffs, history, ref)
config, err = patchImageConfig(config, diffs, history)
config, err = patchImageConfig(config, diffs, history, inp.Metadata[exptypes.ExporterInlineCache])
if err != nil {
return nil, err
}

10
builder/builder-next/exporter/writer.go
View file

@ -41,7 +41,7 @@ func parseHistoryFromConfig(dt []byte) ([]ocispec.History, error) {
return config.History, nil
}
func patchImageConfig(dt []byte, dps []digest.Digest, history []ocispec.History) ([]byte, error) {
func patchImageConfig(dt []byte, dps []digest.Digest, history []ocispec.History, cache []byte) ([]byte, error) {
m := map[string]json.RawMessage{}
if err := json.Unmarshal(dt, &m); err != nil {
return nil, errors.Wrap(err, "failed to parse image config for patch")
@ -77,6 +77,14 @@ func patchImageConfig(dt []byte, dps []digest.Digest, history []ocispec.History)
m["created"] = dt
}
if cache != nil {
dt, err := json.Marshal(cache)
if err != nil {
return nil, err
}
m["moby.buildkit.cache.v0"] = dt
}
dt, err = json.Marshal(m)
return dt, errors.Wrap(err, "failed to marshal config after patch")
}

34
builder/builder-next/worker/worker.go
View file

@ -23,6 +23,7 @@ import (
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/executor"
"github.com/moby/buildkit/exporter"
localexporter "github.com/moby/buildkit/exporter/local"
"github.com/moby/buildkit/frontend"
gw "github.com/moby/buildkit/frontend/gateway/client"
"github.com/moby/buildkit/session"
@ -47,17 +48,16 @@ type Opt struct {
ID string
Labels map[string]string
GCPolicy []client.PruneInfo
SessionManager *session.Manager
MetadataStore *metadata.Store
Executor executor.Executor
Snapshotter snapshot.Snapshotter
ContentStore content.Store
CacheManager cache.Manager
ImageSource source.Source
Exporters map[string]exporter.Exporter
DownloadManager distribution.RootFSDownloadManager
V2MetadataService distmetadata.V2MetadataService
Transport nethttp.RoundTripper
Exporter exporter.Exporter
}
// Worker is a local worker instance with dedicated snapshotter, cache, and so on.
@ -99,9 +99,8 @@ func NewWorker(opt Opt) (*Worker, error) {
}
ss, err := local.NewSource(local.Opt{
SessionManager: opt.SessionManager,
CacheAccessor: cm,
MetadataStore: opt.MetadataStore,
CacheAccessor: cm,
MetadataStore: opt.MetadataStore,
})
if err == nil {
sm.Register(ss)
@ -146,13 +145,13 @@ func (w *Worker) LoadRef(id string, hidden bool) (cache.ImmutableRef, error) {
}
// ResolveOp converts a LLB vertex into a LLB operation
func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge) (solver.Op, error) {
func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge, sm *session.Manager) (solver.Op, error) {
if baseOp, ok := v.Sys().(*pb.Op); ok {
switch op := baseOp.Op.(type) {
case *pb.Op_Source:
return ops.NewSourceOp(v, op, baseOp.Platform, w.SourceManager, w)
return ops.NewSourceOp(v, op, baseOp.Platform, w.SourceManager, sm, w)
case *pb.Op_Exec:
return ops.NewExecOp(v, op, w.CacheManager, w.Opt.SessionManager, w.MetadataStore, w.Executor, w)
return ops.NewExecOp(v, op, baseOp.Platform, w.CacheManager, sm, w.MetadataStore, w.Executor, w)
case *pb.Op_Build:
return ops.NewBuildOp(v, op, s, w)
}
@ -161,13 +160,13 @@ func (w *Worker) ResolveOp(v solver.Vertex, s frontend.FrontendLLBBridge) (solve
}
// ResolveImageConfig returns image config for an image
func (w *Worker) ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt) (digest.Digest, []byte, error) {
func (w *Worker) ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt, sm *session.Manager) (digest.Digest, []byte, error) {
// ImageSource is typically source/containerimage
resolveImageConfig, ok := w.ImageSource.(resolveImageConfig)
if !ok {
return "", nil, errors.Errorf("worker %q does not implement ResolveImageConfig", w.ID())
}
return resolveImageConfig.ResolveImageConfig(ctx, ref, opt)
return resolveImageConfig.ResolveImageConfig(ctx, ref, opt, sm)
}
// Exec executes a process directly on a worker
@ -191,12 +190,17 @@ func (w *Worker) Prune(ctx context.Context, ch chan client.UsageInfo, info ...cl
}
// Exporter returns exporter by name
func (w *Worker) Exporter(name string) (exporter.Exporter, error) {
exp, ok := w.Exporters[name]
if !ok {
func (w *Worker) Exporter(name string, sm *session.Manager) (exporter.Exporter, error) {
switch name {
case "moby":
return w.Opt.Exporter, nil
case client.ExporterLocal:
return localexporter.New(localexporter.Opt{
SessionManager: sm,
})
default:
return nil, errors.Errorf("exporter %q could not be found", name)
}
return exp, nil
}
// GetRemote returns a remote snapshot reference for a local one
@ -338,5 +342,5 @@ func oneOffProgress(ctx context.Context, id string) func(err error) error {
}
type resolveImageConfig interface {
ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt) (digest.Digest, []byte, error)
ResolveImageConfig(ctx context.Context, ref string, opt gw.ResolveImageConfigOpt, sm *session.Manager) (digest.Digest, []byte, error)
}