From 8ff7847d1cf0f156b8a7cf1450aa944ef636c747 Mon Sep 17 00:00:00 2001
From: Tibor Vass <tibor@docker.com>
Date: Wed, 15 Aug 2018 21:24:37 +0000
Subject: [PATCH 1/3] builder: add prune options to the API
Signed-off-by: Tibor Vass <tibor@docker.com>
---
api/server/backend/build/backend.go | 7 ++-
api/server/router/build/backend.go | 2 +-
api/server/router/build/build_routes.go | 22 ++++++-
api/swagger.yaml | 57 +++++++++++++++++
api/types/types.go | 22 ++++---
builder/builder-next/builder.go | 83 +++++++++++++++++++++----
client/build_prune.go | 19 +++++-
client/interface.go | 2 +-
integration/build/build_session_test.go | 3 +-
9 files changed, 190 insertions(+), 27 deletions(-)
diff --git a/api/server/backend/build/backend.go b/api/server/backend/build/backend.go
index 5e04e837a1..33df264cca 100644
--- a/api/server/backend/build/backend.go
+++ b/api/server/backend/build/backend.go
@@ -88,7 +88,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
}
// PruneCache removes all cached build sources
-func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
+func (b *Backend) PruneCache(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) {
eg, ctx := errgroup.WithContext(ctx)
var fsCacheSize uint64
@@ -102,9 +102,10 @@ func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport,
})
var buildCacheSize int64
+ var cacheIDs []string
eg.Go(func() error {
var err error
- buildCacheSize, err = b.buildkit.Prune(ctx)
+ buildCacheSize, cacheIDs, err = b.buildkit.Prune(ctx, opts)
if err != nil {
return errors.Wrap(err, "failed to prune build cache")
}
@@ -115,7 +116,7 @@ func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport,
return nil, err
}
- return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize)}, nil
+ return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize), CachesDeleted: cacheIDs}, nil
}
// Cancel cancels the build by ID
diff --git a/api/server/router/build/backend.go b/api/server/router/build/backend.go
index 2ceae9d946..2983e3b3d2 100644
--- a/api/server/router/build/backend.go
+++ b/api/server/router/build/backend.go
@@ -14,7 +14,7 @@ type Backend interface {
Build(context.Context, backend.BuildConfig) (string, error)
// Prune build cache
- PruneCache(context.Context) (*types.BuildCachePruneReport, error)
+ PruneCache(context.Context, types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error)
Cancel(context.Context, string) error
}
diff --git a/api/server/router/build/build_routes.go b/api/server/router/build/build_routes.go
index c2a15c0ad3..5b9ec676e2 100644
--- a/api/server/router/build/build_routes.go
+++ b/api/server/router/build/build_routes.go
@@ -18,6 +18,7 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/backend"
"github.com/docker/docker/api/types/container"
+ "github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/versions"
"github.com/docker/docker/errdefs"
"github.com/docker/docker/pkg/ioutils"
@@ -161,7 +162,26 @@ func parseVersion(s string) (types.BuilderVersion, error) {
}
func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
- report, err := br.backend.PruneCache(ctx)
+ if err := httputils.ParseForm(r); err != nil {
+ return err
+ }
+ filters, err := filters.FromJSON(r.Form.Get("filters"))
+ if err != nil {
+ return errors.Wrap(err, "could not parse filters")
+ }
+ ksfv := r.FormValue("keep-storage")
+ ks, err := strconv.Atoi(ksfv)
+ if err != nil {
+ return errors.Wrapf(err, "keep-storage is in bytes and expects an integer, got %v", ksfv)
+ }
+
+ opts := types.BuildCachePruneOptions{
+ All: httputils.BoolValue(r, "all"),
+ Filters: filters,
+ KeepStorage: int64(ks),
+ }
+
+ report, err := br.backend.PruneCache(ctx, opts)
if err != nil {
return err
}
diff --git a/api/swagger.yaml b/api/swagger.yaml
index 0462d02618..fd4f2f59ef 100644
--- a/api/swagger.yaml
+++ b/api/swagger.yaml
@@ -1513,6 +1513,31 @@ definitions:
aux:
$ref: "#/definitions/ImageID"
+ BuildCache:
+ type: "object"
+ properties:
+ ID:
+ type: "string"
+ Parent:
+ type: "string"
+ Type:
+ type: "string"
+ Description:
+ type: "string"
+ InUse:
+ type: "boolean"
+ Shared:
+ type: "boolean"
+ Size:
+ type: "integer"
+ CreatedAt:
+ type: "integer"
+ LastUsedAt:
+ type: "integer"
+ x-nullable: true
+ UsageCount:
+ type: "integer"
+
ImageID:
type: "object"
description: "Image ID or Digest"
@@ -6358,6 +6383,29 @@ paths:
produces:
- "application/json"
operationId: "BuildPrune"
+ parameters:
+ - name: "keep-storage"
+ in: "query"
+ description: "Amount of disk space in bytes to keep for cache"
+ type: "integer"
+ format: "int64"
+ - name: "all"
+ in: "query"
+ type: "boolean"
+ description: "Remove all types of build cache"
+ - name: "filters"
+ in: "query"
+ type: "string"
+ description: |
+ A JSON encoded value of the filters (a `map[string][]string`) to process on the list of build cache objects. Available filters:
+ - `unused-for=<duration>`: duration relative to daemon's time, during which build cache was not used, in Go's duration format (e.g., '24h')
+ - `id=<id>`
+ - `parent=<id>`
+ - `type=<string>`
+ - `description=<string>`
+ - `inuse`
+ - `shared`
+ - `private`
responses:
200:
description: "No error"
@@ -6365,6 +6413,11 @@ paths:
type: "object"
title: "BuildPruneResponse"
properties:
+ CachesDeleted:
+ type: "array"
+ items:
+ description: "ID of build cache object"
+ type: "string"
SpaceReclaimed:
description: "Disk space reclaimed in bytes"
type: "integer"
@@ -7199,6 +7252,10 @@ paths:
type: "array"
items:
$ref: "#/definitions/Volume"
+ BuildCache:
+ type: "array"
+ items:
+ $ref: "#/definitions/BuildCache"
example:
LayersSize: 1092588
Images:
diff --git a/api/types/types.go b/api/types/types.go
index ed62fd41e5..a8fae3ba32 100644
--- a/api/types/types.go
+++ b/api/types/types.go
@@ -543,6 +543,7 @@ type ImagesPruneReport struct {
// BuildCachePruneReport contains the response for Engine API:
// POST "/build/prune"
type BuildCachePruneReport struct {
+ CachesDeleted []string
SpaceReclaimed uint64
}
@@ -592,14 +593,21 @@ type BuildResult struct {
// BuildCache contains information about a build cache record
type BuildCache struct {
- ID string
- Mutable bool
- InUse bool
- Size int64
-
+ ID string
+ Parent string
+ Type string
+ Description string
+ InUse bool
+ Shared bool
+ Size int64
CreatedAt time.Time
LastUsedAt *time.Time
UsageCount int
- Parent string
- Description string
+}
+
+// BuildCachePruneOptions hold parameters to prune the build cache
+type BuildCachePruneOptions struct {
+ All bool
+ KeepStorage int64
+ Filters filters.Args
}
diff --git a/builder/builder-next/builder.go b/builder/builder-next/builder.go
index 6d3098aa63..aff43f6fbd 100644
--- a/builder/builder-next/builder.go
+++ b/builder/builder-next/builder.go
@@ -29,6 +29,21 @@ import (
grpcmetadata "google.golang.org/grpc/metadata"
)
+var errMultipleFilterValues = errors.New("filters expect only one value")
+
+var cacheFields = map[string]bool{
+ "id": true,
+ "parent": true,
+ "type": true,
+ "description": true,
+ "inuse": true,
+ "shared": true,
+ "private": true,
+ // fields from buildkit that are not exposed
+ "mutable": false,
+ "immutable": false,
+}
+
func init() {
llbsolver.AllowNetworkHostUnstable = true
}
@@ -87,48 +102,94 @@ func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) {
var items []*types.BuildCache
for _, r := range duResp.Record {
items = append(items, &types.BuildCache{
- ID: r.ID,
- Mutable: r.Mutable,
- InUse: r.InUse,
- Size: r.Size_,
-
+ ID: r.ID,
+ Parent: r.Parent,
+ Type: r.RecordType,
+ Description: r.Description,
+ InUse: r.InUse,
+ Shared: r.Shared,
+ Size: r.Size_,
CreatedAt: r.CreatedAt,
LastUsedAt: r.LastUsedAt,
UsageCount: int(r.UsageCount),
- Parent: r.Parent,
- Description: r.Description,
})
}
return items, nil
}
// Prune clears all reclaimable build cache
-func (b *Builder) Prune(ctx context.Context) (int64, error) {
+func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions) (int64, []string, error) {
ch := make(chan *controlapi.UsageRecord)
eg, ctx := errgroup.WithContext(ctx)
+ validFilters := make(map[string]bool, 1+len(cacheFields))
+ validFilters["unused-for"] = true
+ for k, v := range cacheFields {
+ validFilters[k] = v
+ }
+ if err := opts.Filters.Validate(validFilters); err != nil {
+ return 0, nil, err
+ }
+
+ var unusedFor time.Duration
+ unusedForValues := opts.Filters.Get("unused-for")
+
+ switch len(unusedForValues) {
+ case 0:
+
+ case 1:
+ var err error
+ unusedFor, err = time.ParseDuration(unusedForValues[0])
+ if err != nil {
+ return 0, nil, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')")
+ }
+
+ default:
+ return 0, nil, errMultipleFilterValues
+ }
+
+ bkFilter := make([]string, 0, opts.Filters.Len())
+ for cacheField := range cacheFields {
+ values := opts.Filters.Get(cacheField)
+ switch len(values) {
+ case 0:
+ bkFilter = append(bkFilter, cacheField)
+ case 1:
+ bkFilter = append(bkFilter, cacheField+"=="+values[0])
+ default:
+ return 0, nil, errMultipleFilterValues
+ }
+ }
+
eg.Go(func() error {
defer close(ch)
- return b.controller.Prune(&controlapi.PruneRequest{}, &pruneProxy{
+ return b.controller.Prune(&controlapi.PruneRequest{
+ All: opts.All,
+ KeepDuration: int64(unusedFor),
+ KeepBytes: opts.KeepStorage,
+ Filter: bkFilter,
+ }, &pruneProxy{
streamProxy: streamProxy{ctx: ctx},
ch: ch,
})
})
var size int64
+ var cacheIDs []string
eg.Go(func() error {
for r := range ch {
size += r.Size_
+ cacheIDs = append(cacheIDs, r.ID)
}
return nil
})
if err := eg.Wait(); err != nil {
- return 0, err
+ return 0, nil, err
}
- return size, nil
+ return size, cacheIDs, nil
}
// Build executes a build request
diff --git a/client/build_prune.go b/client/build_prune.go
index c4772a04e7..42bbf99ef1 100644
--- a/client/build_prune.go
+++ b/client/build_prune.go
@@ -4,19 +4,34 @@ import (
"context"
"encoding/json"
"fmt"
+ "net/url"
"github.com/docker/docker/api/types"
+ "github.com/docker/docker/api/types/filters"
+ "github.com/pkg/errors"
)
// BuildCachePrune requests the daemon to delete unused cache data
-func (cli *Client) BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error) {
+func (cli *Client) BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) {
if err := cli.NewVersionError("1.31", "build prune"); err != nil {
return nil, err
}
report := types.BuildCachePruneReport{}
- serverResp, err := cli.post(ctx, "/build/prune", nil, nil, nil)
+ query := url.Values{}
+ if opts.All {
+ query.Set("all", "1")
+ }
+ query.Set("keep-storage", fmt.Sprintf("%d", opts.KeepStorage))
+ filters, err := filters.ToJSON(opts.Filters)
+ if err != nil {
+ return nil, errors.Wrap(err, "prune could not marshal filters option")
+ }
+ query.Set("filters", filters)
+
+ serverResp, err := cli.post(ctx, "/build/prune", query, nil, nil)
+
if err != nil {
return nil, err
}
diff --git a/client/interface.go b/client/interface.go
index 663749f008..d190f8e58d 100644
--- a/client/interface.go
+++ b/client/interface.go
@@ -86,7 +86,7 @@ type DistributionAPIClient interface {
// ImageAPIClient defines API client methods for the images
type ImageAPIClient interface {
ImageBuild(ctx context.Context, context io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error)
- BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error)
+ BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error)
BuildCancel(ctx context.Context, id string) error
ImageCreate(ctx context.Context, parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error)
ImageHistory(ctx context.Context, image string) ([]image.HistoryResponseItem, error)
diff --git a/integration/build/build_session_test.go b/integration/build/build_session_test.go
index dde4b427b4..4d88e3831f 100644
--- a/integration/build/build_session_test.go
+++ b/integration/build/build_session_test.go
@@ -7,6 +7,7 @@ import (
"strings"
"testing"
+ "github.com/docker/docker/api/types"
dclient "github.com/docker/docker/client"
"github.com/docker/docker/internal/test/fakecontext"
"github.com/docker/docker/internal/test/request"
@@ -76,7 +77,7 @@ func TestBuildWithSession(t *testing.T) {
assert.Check(t, is.Contains(string(outBytes), "Successfully built"))
assert.Check(t, is.Equal(strings.Count(string(outBytes), "Using cache"), 4))
- _, err = client.BuildCachePrune(context.TODO())
+ _, err = client.BuildCachePrune(context.TODO(), types.BuildCachePruneOptions{All: true})
assert.Check(t, err)
du, err = client.DiskUsage(context.TODO())
From d47435a004e36531a594cd2636dddaff61a5f4d0 Mon Sep 17 00:00:00 2001
From: Tibor Vass <tibor@docker.com>
Date: Fri, 31 Aug 2018 01:40:41 +0000
Subject: [PATCH 2/3] builder: fix pruning all cache
Signed-off-by: Tibor Vass <tibor@docker.com>
---
builder/builder-next/adapters/containerimage/pull.go | 9 +++++++++
builder/builder-next/controller.go | 2 ++
2 files changed, 11 insertions(+)
diff --git a/builder/builder-next/adapters/containerimage/pull.go b/builder/builder-next/adapters/containerimage/pull.go
index e56efd5134..f187f5c16f 100644
--- a/builder/builder-next/adapters/containerimage/pull.go
+++ b/builder/builder-next/adapters/containerimage/pull.go
@@ -516,6 +516,15 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
return nil, err
}
+ // TODO: handle windows layers for cross platform builds
+
+ if p.src.RecordType != "" && cache.GetRecordType(ref) == "" {
+ if err := cache.SetRecordType(ref, p.src.RecordType); err != nil {
+ ref.Release(context.TODO())
+ return nil, err
+ }
+ }
+
return ref, nil
}
diff --git a/builder/builder-next/controller.go b/builder/builder-next/controller.go
index 48e74cb851..1d0ed2d496 100644
--- a/builder/builder-next/controller.go
+++ b/builder/builder-next/controller.go
@@ -72,6 +72,8 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
cm, err := cache.NewManager(cache.ManagerOpt{
Snapshotter: snapshotter,
MetadataStore: md,
+ // TODO: implement PruneRefChecker to correctly mark cache objects as "Shared"
+ PruneRefChecker: nil,
})
if err != nil {
return nil, err
From 354c241041ec67d4c500cccfda476dc73435d38e Mon Sep 17 00:00:00 2001
From: Tonis Tiigi <tonistiigi@gmail.com>
Date: Fri, 31 Aug 2018 18:28:25 -0700
Subject: [PATCH 3/3] builder: implement ref checker
Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
Signed-off-by: Tibor Vass <tibor@docker.com>
---
.../adapters/snapshot/snapshot.go | 4 +
builder/builder-next/controller.go | 18 +++-
.../builder-next/imagerefchecker/checker.go | 96 +++++++++++++++++++
3 files changed, 114 insertions(+), 4 deletions(-)
create mode 100644 builder/builder-next/imagerefchecker/checker.go
diff --git a/builder/builder-next/adapters/snapshot/snapshot.go b/builder/builder-next/adapters/snapshot/snapshot.go
index e1a12a4563..d8efb759c2 100644
--- a/builder/builder-next/adapters/snapshot/snapshot.go
+++ b/builder/builder-next/adapters/snapshot/snapshot.go
@@ -110,6 +110,10 @@ func (s *snapshotter) chainID(key string) (layer.ChainID, bool) {
return "", false
}
+func (s *snapshotter) GetLayer(key string) (layer.Layer, error) {
+ return s.getLayer(key, true)
+}
+
func (s *snapshotter) getLayer(key string, withCommitted bool) (layer.Layer, error) {
s.mu.Lock()
l, ok := s.refs[key]
diff --git a/builder/builder-next/controller.go b/builder/builder-next/controller.go
index 1d0ed2d496..df0f3d48e5 100644
--- a/builder/builder-next/controller.go
+++ b/builder/builder-next/controller.go
@@ -9,6 +9,7 @@ import (
"github.com/docker/docker/builder/builder-next/adapters/containerimage"
"github.com/docker/docker/builder/builder-next/adapters/snapshot"
containerimageexp "github.com/docker/docker/builder/builder-next/exporter"
+ "github.com/docker/docker/builder/builder-next/imagerefchecker"
mobyworker "github.com/docker/docker/builder/builder-next/worker"
"github.com/docker/docker/daemon/graphdriver"
"github.com/moby/buildkit/cache"
@@ -69,11 +70,20 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
MetadataStore: md,
})
+ layerGetter, ok := sbase.(imagerefchecker.LayerGetter)
+ if !ok {
+ return nil, errors.Errorf("snapshotter does not implement layergetter")
+ }
+
+ refChecker := imagerefchecker.New(imagerefchecker.Opt{
+ ImageStore: dist.ImageStore,
+ LayerGetter: layerGetter,
+ })
+
cm, err := cache.NewManager(cache.ManagerOpt{
- Snapshotter: snapshotter,
- MetadataStore: md,
- // TODO: implement PruneRefChecker to correctly mark cache objects as "Shared"
- PruneRefChecker: nil,
+ Snapshotter: snapshotter,
+ MetadataStore: md,
+ PruneRefChecker: refChecker,
})
if err != nil {
return nil, err
diff --git a/builder/builder-next/imagerefchecker/checker.go b/builder/builder-next/imagerefchecker/checker.go
new file mode 100644
index 0000000000..052391d589
--- /dev/null
+++ b/builder/builder-next/imagerefchecker/checker.go
@@ -0,0 +1,96 @@
+package imagerefchecker
+
+import (
+ "sync"
+
+ "github.com/docker/docker/image"
+ "github.com/docker/docker/layer"
+ "github.com/moby/buildkit/cache"
+)
+
+// LayerGetter abstracts away the snapshotter
+type LayerGetter interface {
+ GetLayer(string) (layer.Layer, error)
+}
+
+// Opt represents the options needed to create a refchecker
+type Opt struct {
+ LayerGetter LayerGetter
+ ImageStore image.Store
+}
+
+// New creates new image reference checker that can be used to see if a reference
+// is being used by any of the images in the image store
+func New(opt Opt) cache.ExternalRefCheckerFunc {
+ return func() (cache.ExternalRefChecker, error) {
+ return &checker{opt: opt, layers: lchain{}, cache: map[string]bool{}}, nil
+ }
+}
+
+type lchain map[layer.DiffID]lchain
+
+func (c lchain) add(ids []layer.DiffID) {
+ if len(ids) == 0 {
+ return
+ }
+ id := ids[0]
+ ch, ok := c[id]
+ if !ok {
+ ch = lchain{}
+ c[id] = ch
+ }
+ ch.add(ids[1:])
+}
+
+func (c lchain) has(ids []layer.DiffID) bool {
+ if len(ids) == 0 {
+ return true
+ }
+ ch, ok := c[ids[0]]
+ return ok && ch.has(ids[1:])
+}
+
+type checker struct {
+ opt Opt
+ once sync.Once
+ layers lchain
+ cache map[string]bool
+}
+
+func (c *checker) Exists(key string) bool {
+ if c.opt.ImageStore == nil {
+ return false
+ }
+
+ c.once.Do(c.init)
+
+ if b, ok := c.cache[key]; ok {
+ return b
+ }
+
+ l, err := c.opt.LayerGetter.GetLayer(key)
+ if err != nil || l == nil {
+ c.cache[key] = false
+ return false
+ }
+
+ ok := c.layers.has(diffIDs(l))
+ c.cache[key] = ok
+ return ok
+}
+
+func (c *checker) init() {
+ imgs := c.opt.ImageStore.Map()
+
+ for _, img := range imgs {
+ c.layers.add(img.RootFS.DiffIDs)
+ }
+}
+
+func diffIDs(l layer.Layer) []layer.DiffID {
+ p := l.Parent()
+ if p == nil {
+ return []layer.DiffID{l.DiffID()}
+ }
+ return append(diffIDs(p), l.DiffID())
+}