kotovalexarian-likes-github
/
moby--moby
mirror of https://github.com/moby/moby.git
1
0
Fork 0
Code Releases Activity

builder-next: update adapters to new buildkit interfaces 

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
Tonis Tiigi 2019-10-09 17:20:17 -07:00
parent fb1601d5ab
commit fe16d95dcd
6 changed files with 75 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
builder/builder-next/adapters/containerimage/pull.go
View File

@ -53,6 +53,7 @@ type SourceOpt struct {
MetadataStore metadata.V2MetadataService
ImageStore image.Store
ResolverOpt resolver.ResolveOptionsFunc
LayerStore layer.Store
}
type imageSource struct {
@ -360,6 +361,23 @@ func (p *puller) CacheKey(ctx context.Context, index int) (string, bool, error)
return k, true, nil
}
func (p *puller) getRef(ctx context.Context, diffIDs []layer.DiffID, opts ...cache.RefOption) (cache.ImmutableRef, error) {
var parent cache.ImmutableRef
if len(diffIDs) > 1 {
var err error
parent, err = p.getRef(ctx, diffIDs[:len(diffIDs)-1], opts...)
if err != nil {
return nil, err
}
defer parent.Release(context.TODO())
}
return p.is.CacheAccessor.GetByBlob(ctx, ocispec.Descriptor{
Annotations: map[string]string{
"containerd.io/uncompressed": diffIDs[len(diffIDs)-1].String(),
},
}, parent, opts...)
}
func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
p.resolveLocal()
if err := p.resolve(ctx); err != nil {
@ -372,11 +390,15 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
if len(img.RootFS.DiffIDs) == 0 {
return nil, nil
}
ref, err := p.is.CacheAccessor.GetFromSnapshotter(ctx, string(img.RootFS.ChainID()), cache.WithDescription(fmt.Sprintf("from local %s", p.ref)))
if err != nil {
return nil, err
l, err := p.is.LayerStore.Get(img.RootFS.ChainID())
if err == nil {
layer.ReleaseAndLog(p.is.LayerStore, l)
ref, err := p.getRef(ctx, img.RootFS.DiffIDs, cache.WithDescription(fmt.Sprintf("from local %s", p.ref)))
if err != nil {
return nil, err
}
return ref, nil
}
return ref, nil
}
}
@ -550,7 +572,7 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
return nil, err
}
ref, err := p.is.CacheAccessor.GetFromSnapshotter(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("pulled from %s", p.ref)))
ref, err := p.getRef(ctx, rootFS.DiffIDs, cache.WithDescription(fmt.Sprintf("pulled from %s", p.ref)))
release()
if err != nil {
return nil, err

14
builder/builder-next/adapters/snapshot/snapshot.go
View File

@ -51,10 +51,8 @@ type snapshotter struct {
reg graphIDRegistrar
}
var _ snapshot.SnapshotterBase = &snapshotter{}
// NewSnapshotter creates a new snapshotter
func NewSnapshotter(opt Opt) (snapshot.SnapshotterBase, error) {
func NewSnapshotter(opt Opt) (snapshot.Snapshotter, error) {
dbPath := filepath.Join(opt.Root, "snapshots.db")
db, err := bolt.Open(dbPath, 0600, nil)
if err != nil {
@ -87,11 +85,11 @@ func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s
origParent := parent
if parent != "" {
if l, err := s.getLayer(parent, false); err != nil {
return err
return errors.Wrapf(err, "failed to get parent layer %s", parent)
} else if l != nil {
parent, err = getGraphID(l)
if err != nil {
return err
return errors.Wrapf(err, "failed to get parent graphid %s", l.ChainID())
}
} else {
parent, _ = s.getGraphDriverID(parent)
@ -146,7 +144,7 @@ func (s *snapshotter) getLayer(key string, withCommitted bool) (layer.Layer, err
return nil
}); err != nil {
s.mu.Unlock()
return nil, err
return nil, errors.WithStack(err)
}
if id == "" {
s.mu.Unlock()
@ -157,12 +155,12 @@ func (s *snapshotter) getLayer(key string, withCommitted bool) (layer.Layer, err
l, err = s.opt.LayerStore.Get(id)
if err != nil {
s.mu.Unlock()
return nil, err
return nil, errors.WithStack(err)
}
s.refs[key] = l
if err := s.db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(key))
return err
return errors.WithStack(err)
}); err != nil {
s.mu.Unlock()
return nil, err

9
builder/builder-next/builder.go
View File

@ -10,7 +10,6 @@ import (
"sync"
"time"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/platforms"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
@ -468,14 +467,6 @@ func (sp *pruneProxy) SendMsg(m interface{}) error {
return nil
}
type contentStoreNoLabels struct {
content.Store
}
func (c *contentStoreNoLabels) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) {
return content.Info{}, nil
}
type wrapRC struct {
io.ReadCloser
once sync.Once

33
builder/builder-next/controller.go
View File

@ -6,7 +6,9 @@ import (
"path/filepath"
"github.com/containerd/containerd/content/local"
ctdmetadata "github.com/containerd/containerd/metadata"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/snapshots"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/builder/builder-next/adapters/containerimage"
@ -29,13 +31,15 @@ import (
dockerfile "github.com/moby/buildkit/frontend/dockerfile/builder"
"github.com/moby/buildkit/frontend/gateway"
"github.com/moby/buildkit/frontend/gateway/forwarder"
"github.com/moby/buildkit/snapshot/blobmapping"
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
"github.com/moby/buildkit/solver/bboltcachestorage"
"github.com/moby/buildkit/util/binfmt_misc"
"github.com/moby/buildkit/util/entitlements"
"github.com/moby/buildkit/util/leaseutil"
"github.com/moby/buildkit/worker"
specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
)
func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
@ -55,7 +59,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
return nil, errors.Errorf("could not access graphdriver")
}
sbase, err := snapshot.NewSnapshotter(snapshot.Opt{
snapshotter, err := snapshot.NewSnapshotter(snapshot.Opt{
GraphDriver: driver,
LayerStore: dist.LayerStore,
Root: root,
@ -69,20 +73,22 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
if err != nil {
return nil, err
}
store = &contentStoreNoLabels{store}
db, err := bolt.Open(filepath.Join(root, "containerdmeta.db"), 0644, nil)
if err != nil {
return nil, errors.WithStack(err)
}
mdb := ctdmetadata.NewDB(db, store, map[string]snapshots.Snapshotter{})
store = containerdsnapshot.NewContentStore(mdb.ContentStore(), "buildkit")
md, err := metadata.NewStore(filepath.Join(root, "metadata.db"))
if err != nil {
return nil, err
}
snapshotter := blobmapping.NewSnapshotter(blobmapping.Opt{
Content: store,
Snapshotter: sbase,
MetadataStore: md,
})
layerGetter, ok := sbase.(imagerefchecker.LayerGetter)
layerGetter, ok := snapshotter.(imagerefchecker.LayerGetter)
if !ok {
return nil, errors.Errorf("snapshotter does not implement layergetter")
}
@ -96,6 +102,8 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
Snapshotter: snapshotter,
MetadataStore: md,
PruneRefChecker: refChecker,
LeaseManager: leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit"),
ContentStore: store,
})
if err != nil {
return nil, err
@ -109,6 +117,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
ImageStore: dist.ImageStore,
ReferenceStore: dist.ReferenceStore,
ResolverOpt: opt.ResolverOpt,
LayerStore: dist.LayerStore,
})
if err != nil {
return nil, err
@ -121,7 +130,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
return nil, err
}
differ, ok := sbase.(containerimageexp.Differ)
differ, ok := snapshotter.(containerimageexp.Differ)
if !ok {
return nil, errors.Errorf("snapshotter doesn't support differ")
}
@ -145,7 +154,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
return nil, errors.Wrap(err, "could not get builder GC policy")
}
layers, ok := sbase.(mobyworker.LayerAccess)
layers, ok := snapshotter.(mobyworker.LayerAccess)
if !ok {
return nil, errors.Errorf("snapshotter doesn't support differ")
}

4
builder/builder-next/imagerefchecker/checker.go
View File

@ -6,6 +6,7 @@ import (
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/moby/buildkit/cache"
digest "github.com/opencontainers/go-digest"
)
// LayerGetter abstracts away the snapshotter
@ -57,7 +58,8 @@ type checker struct {
cache map[string]bool
}
func (c *checker) Exists(key string) bool {
func (c *checker) Exists(key string, chain []digest.Digest) bool {
// TODO: neeeds update
if c.opt.ImageStore == nil {
return false
}

19
builder/builder-next/worker/worker.go
View File

@ -305,6 +305,23 @@ func (w *Worker) PruneCacheMounts(ctx context.Context, ids []string) error {
return nil
}
func (w *Worker) getRef(ctx context.Context, diffIDs []layer.DiffID, opts ...cache.RefOption) (cache.ImmutableRef, error) {
var parent cache.ImmutableRef
if len(diffIDs) > 1 {
var err error
parent, err = w.getRef(ctx, diffIDs[:len(diffIDs)-1], opts...)
if err != nil {
return nil, err
}
defer parent.Release(context.TODO())
}
return w.CacheManager.GetByBlob(context.TODO(), ocispec.Descriptor{
Annotations: map[string]string{
"containerd.io/uncompressed": diffIDs[len(diffIDs)-1].String(),
},
}, parent, opts...)
}
// FromRemote converts a remote snapshot reference to a local one
func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.ImmutableRef, error) {
rootfs, err := getLayers(ctx, remote.Descriptors)
@ -353,7 +370,7 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I
if v, ok := remote.Descriptors[i].Annotations["buildkit/description"]; ok {
descr = v
}
ref, err := w.CacheManager.GetFromSnapshotter(ctx, string(layer.CreateChainID(rootFS.DiffIDs[:i+1])), cache.WithDescription(descr), cache.WithCreationTime(tm))
ref, err := w.getRef(ctx, rootFS.DiffIDs[:i+1], cache.WithDescription(descr), cache.WithCreationTime(tm))
if err != nil {
return nil, err
}