builder-next: update adapters to new buildkit interfaces
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
This commit is contained in:
parent
fb1601d5ab
commit
fe16d95dcd
|
@ -53,6 +53,7 @@ type SourceOpt struct {
|
||||||
MetadataStore metadata.V2MetadataService
|
MetadataStore metadata.V2MetadataService
|
||||||
ImageStore image.Store
|
ImageStore image.Store
|
||||||
ResolverOpt resolver.ResolveOptionsFunc
|
ResolverOpt resolver.ResolveOptionsFunc
|
||||||
|
LayerStore layer.Store
|
||||||
}
|
}
|
||||||
|
|
||||||
type imageSource struct {
|
type imageSource struct {
|
||||||
|
@ -360,6 +361,23 @@ func (p *puller) CacheKey(ctx context.Context, index int) (string, bool, error)
|
||||||
return k, true, nil
|
return k, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *puller) getRef(ctx context.Context, diffIDs []layer.DiffID, opts ...cache.RefOption) (cache.ImmutableRef, error) {
|
||||||
|
var parent cache.ImmutableRef
|
||||||
|
if len(diffIDs) > 1 {
|
||||||
|
var err error
|
||||||
|
parent, err = p.getRef(ctx, diffIDs[:len(diffIDs)-1], opts...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer parent.Release(context.TODO())
|
||||||
|
}
|
||||||
|
return p.is.CacheAccessor.GetByBlob(ctx, ocispec.Descriptor{
|
||||||
|
Annotations: map[string]string{
|
||||||
|
"containerd.io/uncompressed": diffIDs[len(diffIDs)-1].String(),
|
||||||
|
},
|
||||||
|
}, parent, opts...)
|
||||||
|
}
|
||||||
|
|
||||||
func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
|
func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
|
||||||
p.resolveLocal()
|
p.resolveLocal()
|
||||||
if err := p.resolve(ctx); err != nil {
|
if err := p.resolve(ctx); err != nil {
|
||||||
|
@ -372,11 +390,15 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
|
||||||
if len(img.RootFS.DiffIDs) == 0 {
|
if len(img.RootFS.DiffIDs) == 0 {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
ref, err := p.is.CacheAccessor.GetFromSnapshotter(ctx, string(img.RootFS.ChainID()), cache.WithDescription(fmt.Sprintf("from local %s", p.ref)))
|
l, err := p.is.LayerStore.Get(img.RootFS.ChainID())
|
||||||
if err != nil {
|
if err == nil {
|
||||||
return nil, err
|
layer.ReleaseAndLog(p.is.LayerStore, l)
|
||||||
|
ref, err := p.getRef(ctx, img.RootFS.DiffIDs, cache.WithDescription(fmt.Sprintf("from local %s", p.ref)))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return ref, nil
|
||||||
}
|
}
|
||||||
return ref, nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -550,7 +572,7 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
ref, err := p.is.CacheAccessor.GetFromSnapshotter(ctx, string(rootFS.ChainID()), cache.WithDescription(fmt.Sprintf("pulled from %s", p.ref)))
|
ref, err := p.getRef(ctx, rootFS.DiffIDs, cache.WithDescription(fmt.Sprintf("pulled from %s", p.ref)))
|
||||||
release()
|
release()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|
|
@ -51,10 +51,8 @@ type snapshotter struct {
|
||||||
reg graphIDRegistrar
|
reg graphIDRegistrar
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ snapshot.SnapshotterBase = &snapshotter{}
|
|
||||||
|
|
||||||
// NewSnapshotter creates a new snapshotter
|
// NewSnapshotter creates a new snapshotter
|
||||||
func NewSnapshotter(opt Opt) (snapshot.SnapshotterBase, error) {
|
func NewSnapshotter(opt Opt) (snapshot.Snapshotter, error) {
|
||||||
dbPath := filepath.Join(opt.Root, "snapshots.db")
|
dbPath := filepath.Join(opt.Root, "snapshots.db")
|
||||||
db, err := bolt.Open(dbPath, 0600, nil)
|
db, err := bolt.Open(dbPath, 0600, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -87,11 +85,11 @@ func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s
|
||||||
origParent := parent
|
origParent := parent
|
||||||
if parent != "" {
|
if parent != "" {
|
||||||
if l, err := s.getLayer(parent, false); err != nil {
|
if l, err := s.getLayer(parent, false); err != nil {
|
||||||
return err
|
return errors.Wrapf(err, "failed to get parent layer %s", parent)
|
||||||
} else if l != nil {
|
} else if l != nil {
|
||||||
parent, err = getGraphID(l)
|
parent, err = getGraphID(l)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return errors.Wrapf(err, "failed to get parent graphid %s", l.ChainID())
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
parent, _ = s.getGraphDriverID(parent)
|
parent, _ = s.getGraphDriverID(parent)
|
||||||
|
@ -146,7 +144,7 @@ func (s *snapshotter) getLayer(key string, withCommitted bool) (layer.Layer, err
|
||||||
return nil
|
return nil
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
return nil, err
|
return nil, errors.WithStack(err)
|
||||||
}
|
}
|
||||||
if id == "" {
|
if id == "" {
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
|
@ -157,12 +155,12 @@ func (s *snapshotter) getLayer(key string, withCommitted bool) (layer.Layer, err
|
||||||
l, err = s.opt.LayerStore.Get(id)
|
l, err = s.opt.LayerStore.Get(id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
return nil, err
|
return nil, errors.WithStack(err)
|
||||||
}
|
}
|
||||||
s.refs[key] = l
|
s.refs[key] = l
|
||||||
if err := s.db.Update(func(tx *bolt.Tx) error {
|
if err := s.db.Update(func(tx *bolt.Tx) error {
|
||||||
_, err := tx.CreateBucketIfNotExists([]byte(key))
|
_, err := tx.CreateBucketIfNotExists([]byte(key))
|
||||||
return err
|
return errors.WithStack(err)
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|
|
@ -10,7 +10,6 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/containerd/containerd/content"
|
|
||||||
"github.com/containerd/containerd/platforms"
|
"github.com/containerd/containerd/platforms"
|
||||||
"github.com/docker/docker/api/types"
|
"github.com/docker/docker/api/types"
|
||||||
"github.com/docker/docker/api/types/backend"
|
"github.com/docker/docker/api/types/backend"
|
||||||
|
@ -468,14 +467,6 @@ func (sp *pruneProxy) SendMsg(m interface{}) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
type contentStoreNoLabels struct {
|
|
||||||
content.Store
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *contentStoreNoLabels) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) {
|
|
||||||
return content.Info{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type wrapRC struct {
|
type wrapRC struct {
|
||||||
io.ReadCloser
|
io.ReadCloser
|
||||||
once sync.Once
|
once sync.Once
|
||||||
|
|
|
@ -6,7 +6,9 @@ import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
"github.com/containerd/containerd/content/local"
|
"github.com/containerd/containerd/content/local"
|
||||||
|
ctdmetadata "github.com/containerd/containerd/metadata"
|
||||||
"github.com/containerd/containerd/platforms"
|
"github.com/containerd/containerd/platforms"
|
||||||
|
"github.com/containerd/containerd/snapshots"
|
||||||
"github.com/docker/docker/api/types"
|
"github.com/docker/docker/api/types"
|
||||||
"github.com/docker/docker/api/types/filters"
|
"github.com/docker/docker/api/types/filters"
|
||||||
"github.com/docker/docker/builder/builder-next/adapters/containerimage"
|
"github.com/docker/docker/builder/builder-next/adapters/containerimage"
|
||||||
|
@ -29,13 +31,15 @@ import (
|
||||||
dockerfile "github.com/moby/buildkit/frontend/dockerfile/builder"
|
dockerfile "github.com/moby/buildkit/frontend/dockerfile/builder"
|
||||||
"github.com/moby/buildkit/frontend/gateway"
|
"github.com/moby/buildkit/frontend/gateway"
|
||||||
"github.com/moby/buildkit/frontend/gateway/forwarder"
|
"github.com/moby/buildkit/frontend/gateway/forwarder"
|
||||||
"github.com/moby/buildkit/snapshot/blobmapping"
|
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
|
||||||
"github.com/moby/buildkit/solver/bboltcachestorage"
|
"github.com/moby/buildkit/solver/bboltcachestorage"
|
||||||
"github.com/moby/buildkit/util/binfmt_misc"
|
"github.com/moby/buildkit/util/binfmt_misc"
|
||||||
"github.com/moby/buildkit/util/entitlements"
|
"github.com/moby/buildkit/util/entitlements"
|
||||||
|
"github.com/moby/buildkit/util/leaseutil"
|
||||||
"github.com/moby/buildkit/worker"
|
"github.com/moby/buildkit/worker"
|
||||||
specs "github.com/opencontainers/image-spec/specs-go/v1"
|
specs "github.com/opencontainers/image-spec/specs-go/v1"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
bolt "go.etcd.io/bbolt"
|
||||||
)
|
)
|
||||||
|
|
||||||
func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
||||||
|
@ -55,7 +59,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
||||||
return nil, errors.Errorf("could not access graphdriver")
|
return nil, errors.Errorf("could not access graphdriver")
|
||||||
}
|
}
|
||||||
|
|
||||||
sbase, err := snapshot.NewSnapshotter(snapshot.Opt{
|
snapshotter, err := snapshot.NewSnapshotter(snapshot.Opt{
|
||||||
GraphDriver: driver,
|
GraphDriver: driver,
|
||||||
LayerStore: dist.LayerStore,
|
LayerStore: dist.LayerStore,
|
||||||
Root: root,
|
Root: root,
|
||||||
|
@ -69,20 +73,22 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
store = &contentStoreNoLabels{store}
|
|
||||||
|
db, err := bolt.Open(filepath.Join(root, "containerdmeta.db"), 0644, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.WithStack(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
mdb := ctdmetadata.NewDB(db, store, map[string]snapshots.Snapshotter{})
|
||||||
|
|
||||||
|
store = containerdsnapshot.NewContentStore(mdb.ContentStore(), "buildkit")
|
||||||
|
|
||||||
md, err := metadata.NewStore(filepath.Join(root, "metadata.db"))
|
md, err := metadata.NewStore(filepath.Join(root, "metadata.db"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
snapshotter := blobmapping.NewSnapshotter(blobmapping.Opt{
|
layerGetter, ok := snapshotter.(imagerefchecker.LayerGetter)
|
||||||
Content: store,
|
|
||||||
Snapshotter: sbase,
|
|
||||||
MetadataStore: md,
|
|
||||||
})
|
|
||||||
|
|
||||||
layerGetter, ok := sbase.(imagerefchecker.LayerGetter)
|
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, errors.Errorf("snapshotter does not implement layergetter")
|
return nil, errors.Errorf("snapshotter does not implement layergetter")
|
||||||
}
|
}
|
||||||
|
@ -96,6 +102,8 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
||||||
Snapshotter: snapshotter,
|
Snapshotter: snapshotter,
|
||||||
MetadataStore: md,
|
MetadataStore: md,
|
||||||
PruneRefChecker: refChecker,
|
PruneRefChecker: refChecker,
|
||||||
|
LeaseManager: leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit"),
|
||||||
|
ContentStore: store,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -109,6 +117,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
||||||
ImageStore: dist.ImageStore,
|
ImageStore: dist.ImageStore,
|
||||||
ReferenceStore: dist.ReferenceStore,
|
ReferenceStore: dist.ReferenceStore,
|
||||||
ResolverOpt: opt.ResolverOpt,
|
ResolverOpt: opt.ResolverOpt,
|
||||||
|
LayerStore: dist.LayerStore,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
@ -121,7 +130,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
differ, ok := sbase.(containerimageexp.Differ)
|
differ, ok := snapshotter.(containerimageexp.Differ)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, errors.Errorf("snapshotter doesn't support differ")
|
return nil, errors.Errorf("snapshotter doesn't support differ")
|
||||||
}
|
}
|
||||||
|
@ -145,7 +154,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
|
||||||
return nil, errors.Wrap(err, "could not get builder GC policy")
|
return nil, errors.Wrap(err, "could not get builder GC policy")
|
||||||
}
|
}
|
||||||
|
|
||||||
layers, ok := sbase.(mobyworker.LayerAccess)
|
layers, ok := snapshotter.(mobyworker.LayerAccess)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, errors.Errorf("snapshotter doesn't support differ")
|
return nil, errors.Errorf("snapshotter doesn't support differ")
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,6 +6,7 @@ import (
|
||||||
"github.com/docker/docker/image"
|
"github.com/docker/docker/image"
|
||||||
"github.com/docker/docker/layer"
|
"github.com/docker/docker/layer"
|
||||||
"github.com/moby/buildkit/cache"
|
"github.com/moby/buildkit/cache"
|
||||||
|
digest "github.com/opencontainers/go-digest"
|
||||||
)
|
)
|
||||||
|
|
||||||
// LayerGetter abstracts away the snapshotter
|
// LayerGetter abstracts away the snapshotter
|
||||||
|
@ -57,7 +58,8 @@ type checker struct {
|
||||||
cache map[string]bool
|
cache map[string]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *checker) Exists(key string) bool {
|
func (c *checker) Exists(key string, chain []digest.Digest) bool {
|
||||||
|
// TODO: neeeds update
|
||||||
if c.opt.ImageStore == nil {
|
if c.opt.ImageStore == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
|
@ -305,6 +305,23 @@ func (w *Worker) PruneCacheMounts(ctx context.Context, ids []string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (w *Worker) getRef(ctx context.Context, diffIDs []layer.DiffID, opts ...cache.RefOption) (cache.ImmutableRef, error) {
|
||||||
|
var parent cache.ImmutableRef
|
||||||
|
if len(diffIDs) > 1 {
|
||||||
|
var err error
|
||||||
|
parent, err = w.getRef(ctx, diffIDs[:len(diffIDs)-1], opts...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer parent.Release(context.TODO())
|
||||||
|
}
|
||||||
|
return w.CacheManager.GetByBlob(context.TODO(), ocispec.Descriptor{
|
||||||
|
Annotations: map[string]string{
|
||||||
|
"containerd.io/uncompressed": diffIDs[len(diffIDs)-1].String(),
|
||||||
|
},
|
||||||
|
}, parent, opts...)
|
||||||
|
}
|
||||||
|
|
||||||
// FromRemote converts a remote snapshot reference to a local one
|
// FromRemote converts a remote snapshot reference to a local one
|
||||||
func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.ImmutableRef, error) {
|
func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.ImmutableRef, error) {
|
||||||
rootfs, err := getLayers(ctx, remote.Descriptors)
|
rootfs, err := getLayers(ctx, remote.Descriptors)
|
||||||
|
@ -353,7 +370,7 @@ func (w *Worker) FromRemote(ctx context.Context, remote *solver.Remote) (cache.I
|
||||||
if v, ok := remote.Descriptors[i].Annotations["buildkit/description"]; ok {
|
if v, ok := remote.Descriptors[i].Annotations["buildkit/description"]; ok {
|
||||||
descr = v
|
descr = v
|
||||||
}
|
}
|
||||||
ref, err := w.CacheManager.GetFromSnapshotter(ctx, string(layer.CreateChainID(rootFS.DiffIDs[:i+1])), cache.WithDescription(descr), cache.WithCreationTime(tm))
|
ref, err := w.getRef(ctx, rootFS.DiffIDs[:i+1], cache.WithDescription(descr), cache.WithCreationTime(tm))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue