Merge pull request #37852 from AntaresS/patch-buildkit

add support for "registry-mirrors" and "insecure-registries" to buildkit

builder/builder-next/adapters/containerimage/pull.go

@@ -34,6 +34,7 @@ import (
 	"github.com/moby/buildkit/util/flightcontrol"
 	"github.com/moby/buildkit/util/imageutil"
 	"github.com/moby/buildkit/util/progress"
+	"github.com/moby/buildkit/util/resolver"
 	"github.com/moby/buildkit/util/tracing"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"
@@ -51,6 +52,7 @@ type SourceOpt struct {
 	DownloadManager distribution.RootFSDownloadManager
 	MetadataStore   metadata.V2MetadataService
 	ImageStore      image.Store
+	ResolverOpt     resolver.ResolveOptionsFunc
 }
 
 type imageSource struct {
@@ -71,11 +73,16 @@ func (is *imageSource) ID() string {
 	return source.DockerImageScheme
 }
 
-func (is *imageSource) getResolver(ctx context.Context) remotes.Resolver {
+func (is *imageSource) getResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, ref string) remotes.Resolver {
-	return docker.NewResolver(docker.ResolverOptions{
+	opt := docker.ResolverOptions{
 		Client:      tracing.DefaultClient,
 		Credentials: is.getCredentialsFromSession(ctx),
-	})
+	}
+	if rfn != nil {
+		opt = rfn(ref)
+	}
+	r := docker.NewResolver(opt)
+	return r
 }
 
 func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(string) (string, string, error) {
@@ -118,7 +125,7 @@ func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *
 		dt   []byte
 	}
 	res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
-		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx), is.ContentStore, platform)
+		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref), is.ContentStore, platform)
 		if err != nil {
 			return nil, err
 		}
@@ -181,7 +188,7 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (sourc
 	p := &puller{
 		src:      imageIdentifier,
 		is:       is,
-		resolver: is.getResolver(ctx),
+		resolver: is.getResolver(ctx, is.ResolverOpt, imageIdentifier.Reference.String()),
 		platform: platform,
 	}
 	return p, nil

builder/builder-next/builder.go

@@ -23,6 +23,7 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver/llbsolver"
 	"github.com/moby/buildkit/util/entitlements"
+	"github.com/moby/buildkit/util/resolver"
 	"github.com/moby/buildkit/util/tracing"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -55,6 +56,7 @@ type Opt struct {
 	Dist                images.DistributionServices
 	NetworkController   libnetwork.NetworkController
 	DefaultCgroupParent string
+	ResolverOpt         resolver.ResolveOptionsFunc
 }
 
 // Builder can build using BuildKit backend

builder/builder-next/controller.go

@@ -97,6 +97,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		MetadataStore:   dist.V2MetadataService,
 		ImageStore:      dist.ImageStore,
 		ReferenceStore:  dist.ReferenceStore,
+		ResolverOpt:     opt.ResolverOpt,
 	})
 	if err != nil {
 		return nil, err
@@ -160,7 +161,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		WorkerController:         wc,
 		Frontends:                frontends,
 		CacheKeyStorage:          cacheStorage,
-		ResolveCacheImporterFunc: registryremotecache.ResolveCacheImporterFunc(opt.SessionManager),
+		ResolveCacheImporterFunc: registryremotecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt),
 		// TODO: set ResolveCacheExporterFunc for exporting cache
 	})
 }

cmd/dockerd/daemon.go

@@ -291,6 +291,7 @@ func newRouterOptions(config *config.Config, d *daemon.Daemon) (routerOptions, e
 		Dist:                d.DistributionServices(),
 		NetworkController:   d.NetworkController(),
 		DefaultCgroupParent: cgroupParent,
+		ResolverOpt:         d.NewResolveOptionsFunc(),
 	})
 	if err != nil {
 		return opts, err

daemon/daemon.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net"
 	"os"
 	"path"
@@ -23,6 +24,8 @@ import (
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/defaults"
 	"github.com/containerd/containerd/pkg/dialer"
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	containertypes "github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/swarm"
@@ -36,6 +39,8 @@ import (
 	"github.com/docker/docker/daemon/logger"
 	"github.com/docker/docker/daemon/network"
 	"github.com/docker/docker/errdefs"
+	"github.com/moby/buildkit/util/resolver"
+	"github.com/moby/buildkit/util/tracing"
 	"github.com/sirupsen/logrus"
 	// register graph drivers
 	_ "github.com/docker/docker/daemon/graphdriver/register"
@@ -141,6 +146,57 @@ func (daemon *Daemon) Features() *map[string]bool {
 	return &daemon.configStore.Features
 }
 
+// NewResolveOptionsFunc returns a call back function to resolve "registry-mirrors" and
+// "insecure-registries" for buildkit
+func (daemon *Daemon) NewResolveOptionsFunc() resolver.ResolveOptionsFunc {
+	return func(ref string) docker.ResolverOptions {
+		var (
+			registryKey = "docker.io"
+			mirrors     = make([]string, len(daemon.configStore.Mirrors))
+			m           = map[string]resolver.RegistryConf{}
+		)
+		// must trim "https://" or "http://" prefix
+		for i, v := range daemon.configStore.Mirrors {
+			v = strings.TrimPrefix(v, "https://")
+			v = strings.TrimPrefix(v, "http://")
+			mirrors[i] = v
+		}
+		// set "registry-mirrors"
+		m[registryKey] = resolver.RegistryConf{Mirrors: mirrors}
+		// set "insecure-registries"
+		for _, v := range daemon.configStore.InsecureRegistries {
+			v = strings.TrimPrefix(v, "http://")
+			m[v] = resolver.RegistryConf{
+				PlainHTTP: true,
+			}
+		}
+		def := docker.ResolverOptions{
+			Client: tracing.DefaultClient,
+		}
+
+		parsed, err := reference.ParseNormalizedNamed(ref)
+		if err != nil {
+			return def
+		}
+		host := reference.Domain(parsed)
+
+		c, ok := m[host]
+		if !ok {
+			return def
+		}
+
+		if len(c.Mirrors) > 0 {
+			def.Host = func(string) (string, error) {
+				return c.Mirrors[rand.Intn(len(c.Mirrors))], nil
+			}
+		}
+
+		def.PlainHTTP = c.PlainHTTP
+
+		return def
+	}
+}
+
 func (daemon *Daemon) restore() error {
 	containers := make(map[string]*container.Container)
 

vendor.conf

@@ -26,7 +26,7 @@ github.com/imdario/mergo v0.3.6
 golang.org/x/sync 1d60e4601c6fd243af51cc01ddf169918a5407ca
 
 # buildkit
-github.com/moby/buildkit a9fe50acf16dd05d1f9877b27068884543ad7a1f
+github.com/moby/buildkit d88354f7856a1fafef6f23bc9c5a538c246f4023
 github.com/tonistiigi/fsutil b19464cd1b6a00773b4f2eb7acf9c30426f9df42
 github.com/grpc-ecosystem/grpc-opentracing 8e809c8a86450a29b90dcc9efbf062d0fe6d9746
 github.com/opentracing/opentracing-go 1361b9cd60be79c4c3a7fa9841b3c132e40066a7

vendor/github.com/moby/buildkit/cache/remotecache/registry/registry.go

@@ -10,17 +10,17 @@ import (
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/auth"
 	"github.com/moby/buildkit/util/contentutil"
-	"github.com/moby/buildkit/util/tracing"
+	"github.com/moby/buildkit/util/resolver"
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
-func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExporterFunc {
+func ResolveCacheExporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheExporterFunc {
 	return func(ctx context.Context, typ, ref string) (remotecache.Exporter, error) {
 		if typ != "" {
 			return nil, errors.Errorf("unsupported cache exporter type: %s", typ)
 		}
-		remote := newRemoteResolver(ctx, sm)
+		remote := newRemoteResolver(ctx, resolverOpt, sm, ref)
 		pusher, err := remote.Pusher(ctx, ref)
 		if err != nil {
 			return nil, err
@@ -29,12 +29,12 @@ func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExpor
 	}
 }
 
-func ResolveCacheImporterFunc(sm *session.Manager) remotecache.ResolveCacheImporterFunc {
+func ResolveCacheImporterFunc(sm *session.Manager, resolverOpt resolver.ResolveOptionsFunc) remotecache.ResolveCacheImporterFunc {
 	return func(ctx context.Context, typ, ref string) (remotecache.Importer, specs.Descriptor, error) {
 		if typ != "" {
 			return nil, specs.Descriptor{}, errors.Errorf("unsupported cache importer type: %s", typ)
 		}
-		remote := newRemoteResolver(ctx, sm)
+		remote := newRemoteResolver(ctx, resolverOpt, sm, ref)
 		xref, desc, err := remote.Resolve(ctx, ref)
 		if err != nil {
 			return nil, specs.Descriptor{}, err
@@ -47,11 +47,10 @@ func ResolveCacheImporterFunc(sm *session.Manager) remotecache.ResolveCacheImpor
 	}
 }
 
-func newRemoteResolver(ctx context.Context, sm *session.Manager) remotes.Resolver {
+func newRemoteResolver(ctx context.Context, resolverOpt resolver.ResolveOptionsFunc, sm *session.Manager, ref string) remotes.Resolver {
-	return docker.NewResolver(docker.ResolverOptions{
+	opt := resolverOpt(ref)
-		Client:      tracing.DefaultClient,
+	opt.Credentials = getCredentialsFunc(ctx, sm)
-		Credentials: getCredentialsFunc(ctx, sm),
+	return docker.NewResolver(opt)
-	})
 }
 
 func getCredentialsFunc(ctx context.Context, sm *session.Manager) func(string) (string, string, error) {

vendor/github.com/moby/buildkit/solver/jobs.go

@@ -444,6 +444,7 @@ func (j *Job) Discard() error {
 	j.pw.Close()
 
 	for k, st := range j.list.actives {
+		st.mu.Lock()
 		if _, ok := st.jobs[j]; ok {
 			delete(st.jobs, j)
 			j.list.deleteIfUnreferenced(k, st)
@@ -451,6 +452,7 @@ func (j *Job) Discard() error {
 		if _, ok := st.allPw[j.pw]; ok {
 			delete(st.allPw, j.pw)
 		}
+		st.mu.Unlock()
 	}
 	return nil
 }

vendor/github.com/moby/buildkit/util/resolver/resolver.go

@@ -0,0 +1,45 @@
+package resolver
+
+import (
+	"math/rand"
+
+	"github.com/containerd/containerd/remotes/docker"
+	"github.com/docker/distribution/reference"
+	"github.com/moby/buildkit/util/tracing"
+)
+
+type RegistryConf struct {
+	Mirrors   []string
+	PlainHTTP bool
+}
+
+type ResolveOptionsFunc func(string) docker.ResolverOptions
+
+func NewResolveOptionsFunc(m map[string]RegistryConf) ResolveOptionsFunc {
+	return func(ref string) docker.ResolverOptions {
+		def := docker.ResolverOptions{
+			Client: tracing.DefaultClient,
+		}
+
+		parsed, err := reference.ParseNormalizedNamed(ref)
+		if err != nil {
+			return def
+		}
+		host := reference.Domain(parsed)
+
+		c, ok := m[host]
+		if !ok {
+			return def
+		}
+
+		if len(c.Mirrors) > 0 {
+			def.Host = func(string) (string, error) {
+				return c.Mirrors[rand.Intn(len(c.Mirrors))], nil
+			}
+		}
+
+		def.PlainHTTP = c.PlainHTTP
+
+		return def
+	}
+}

vendor/github.com/moby/buildkit/vendor.conf

@@ -39,7 +39,7 @@ golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631
 github.com/docker/docker 71cd53e4a197b303c6ba086bd584ffd67a884281
 github.com/pkg/profile 5b67d428864e92711fcbd2f8629456121a56d91f
 
-github.com/tonistiigi/fsutil b19464cd1b6a00773b4f2eb7acf9c30426f9df42
+github.com/tonistiigi/fsutil 7e391b0e788f9b925f22bd3cf88e0210d1643673
 github.com/hashicorp/go-immutable-radix 826af9ccf0feeee615d546d69b11f8e98da8c8f1 git://github.com/tonistiigi/go-immutable-radix.git
 github.com/hashicorp/golang-lru a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4
 github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b