Merge pull request #37651 from tiborvass/new-builder-prune

builder: add prune options to the API

api/server/backend/build/backend.go

@@ -88,7 +88,7 @@ func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string
 }
 
 // PruneCache removes all cached build sources
-func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport, error) {
+func (b *Backend) PruneCache(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) {
 	eg, ctx := errgroup.WithContext(ctx)
 
 	var fsCacheSize uint64
@@ -102,9 +102,10 @@ func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport,
 	})
 
 	var buildCacheSize int64
+	var cacheIDs []string
 	eg.Go(func() error {
 		var err error
-		buildCacheSize, err = b.buildkit.Prune(ctx)
+		buildCacheSize, cacheIDs, err = b.buildkit.Prune(ctx, opts)
 		if err != nil {
 			return errors.Wrap(err, "failed to prune build cache")
 		}
@@ -115,7 +116,7 @@ func (b *Backend) PruneCache(ctx context.Context) (*types.BuildCachePruneReport,
 		return nil, err
 	}
 
-	return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize)}, nil
+	return &types.BuildCachePruneReport{SpaceReclaimed: fsCacheSize + uint64(buildCacheSize), CachesDeleted: cacheIDs}, nil
 }
 
 // Cancel cancels the build by ID

api/server/router/build/backend.go

@@ -14,7 +14,7 @@ type Backend interface {
 	Build(context.Context, backend.BuildConfig) (string, error)
 
 	// Prune build cache
-	PruneCache(context.Context) (*types.BuildCachePruneReport, error)
+	PruneCache(context.Context, types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error)
 
 	Cancel(context.Context, string) error
 }

api/server/router/build/build_routes.go

@@ -18,6 +18,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/api/types/versions"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/pkg/ioutils"
@@ -161,7 +162,26 @@ func parseVersion(s string) (types.BuilderVersion, error) {
 }
 
 func (br *buildRouter) postPrune(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	report, err := br.backend.PruneCache(ctx)
+	if err := httputils.ParseForm(r); err != nil {
+		return err
+	}
+	filters, err := filters.FromJSON(r.Form.Get("filters"))
+	if err != nil {
+		return errors.Wrap(err, "could not parse filters")
+	}
+	ksfv := r.FormValue("keep-storage")
+	ks, err := strconv.Atoi(ksfv)
+	if err != nil {
+		return errors.Wrapf(err, "keep-storage is in bytes and expects an integer, got %v", ksfv)
+	}
+
+	opts := types.BuildCachePruneOptions{
+		All:         httputils.BoolValue(r, "all"),
+		Filters:     filters,
+		KeepStorage: int64(ks),
+	}
+
+	report, err := br.backend.PruneCache(ctx, opts)
 	if err != nil {
 		return err
 	}

api/swagger.yaml

@@ -1513,6 +1513,31 @@ definitions:
       aux:
         $ref: "#/definitions/ImageID"
 
+  BuildCache:
+    type: "object"
+    properties:
+      ID:
+        type: "string"
+      Parent:
+        type: "string"
+      Type:
+        type: "string"
+      Description:
+        type: "string"
+      InUse:
+        type: "boolean"
+      Shared:
+        type: "boolean"
+      Size:
+        type: "integer"
+      CreatedAt:
+        type: "integer"
+      LastUsedAt:
+        type: "integer"
+        x-nullable: true
+      UsageCount:
+        type: "integer"
+
   ImageID:
     type: "object"
     description: "Image ID or Digest"
@@ -6358,6 +6383,29 @@ paths:
       produces:
         - "application/json"
       operationId: "BuildPrune"
+      parameters:
+        - name: "keep-storage"
+          in: "query"
+          description: "Amount of disk space in bytes to keep for cache"
+          type: "integer"
+          format: "int64"
+        - name: "all"
+          in: "query"
+          type: "boolean"
+          description: "Remove all types of build cache"
+        - name: "filters"
+          in: "query"
+          type: "string"
+          description: |
+            A JSON encoded value of the filters (a `map[string][]string`) to process on the list of build cache objects. Available filters:
+            - `unused-for=<duration>`: duration relative to daemon's time, during which build cache was not used, in Go's duration format (e.g., '24h')
+            - `id=<id>`
+            - `parent=<id>`
+            - `type=<string>`
+            - `description=<string>`
+            - `inuse`
+            - `shared`
+            - `private`
       responses:
         200:
           description: "No error"
@@ -6365,6 +6413,11 @@ paths:
             type: "object"
             title: "BuildPruneResponse"
             properties:
+              CachesDeleted:
+                type: "array"
+                items:
+                  description: "ID of build cache object"
+                  type: "string"
               SpaceReclaimed:
                 description: "Disk space reclaimed in bytes"
                 type: "integer"
@@ -7199,6 +7252,10 @@ paths:
                 type: "array"
                 items:
                   $ref: "#/definitions/Volume"
+              BuildCache:
+                type: "array"
+                items:
+                  $ref: "#/definitions/BuildCache"
             example:
               LayersSize: 1092588
               Images:

api/types/types.go

@@ -543,6 +543,7 @@ type ImagesPruneReport struct {
 // BuildCachePruneReport contains the response for Engine API:
 // POST "/build/prune"
 type BuildCachePruneReport struct {
+	CachesDeleted  []string
 	SpaceReclaimed uint64
 }
 
@@ -592,14 +593,21 @@ type BuildResult struct {
 
 // BuildCache contains information about a build cache record
 type BuildCache struct {
-	ID      string
-	Mutable bool
-	InUse   bool
-	Size    int64
-
+	ID          string
+	Parent      string
+	Type        string
+	Description string
+	InUse       bool
+	Shared      bool
+	Size        int64
 	CreatedAt   time.Time
 	LastUsedAt  *time.Time
 	UsageCount  int
-	Parent      string
-	Description string
+}
+
+// BuildCachePruneOptions hold parameters to prune the build cache
+type BuildCachePruneOptions struct {
+	All         bool
+	KeepStorage int64
+	Filters     filters.Args
 }

builder/builder-next/adapters/containerimage/pull.go

@@ -516,6 +516,15 @@ func (p *puller) Snapshot(ctx context.Context) (cache.ImmutableRef, error) {
 		return nil, err
 	}
 
+	// TODO: handle windows layers for cross platform builds
+
+	if p.src.RecordType != "" && cache.GetRecordType(ref) == "" {
+		if err := cache.SetRecordType(ref, p.src.RecordType); err != nil {
+			ref.Release(context.TODO())
+			return nil, err
+		}
+	}
+
 	return ref, nil
 }
 

builder/builder-next/adapters/snapshot/snapshot.go

@@ -110,6 +110,10 @@ func (s *snapshotter) chainID(key string) (layer.ChainID, bool) {
 	return "", false
 }
 
+func (s *snapshotter) GetLayer(key string) (layer.Layer, error) {
+	return s.getLayer(key, true)
+}
+
 func (s *snapshotter) getLayer(key string, withCommitted bool) (layer.Layer, error) {
 	s.mu.Lock()
 	l, ok := s.refs[key]

builder/builder-next/builder.go

@@ -29,6 +29,21 @@ import (
 	grpcmetadata "google.golang.org/grpc/metadata"
 )
 
+var errMultipleFilterValues = errors.New("filters expect only one value")
+
+var cacheFields = map[string]bool{
+	"id":          true,
+	"parent":      true,
+	"type":        true,
+	"description": true,
+	"inuse":       true,
+	"shared":      true,
+	"private":     true,
+	// fields from buildkit that are not exposed
+	"mutable":   false,
+	"immutable": false,
+}
+
 func init() {
 	llbsolver.AllowNetworkHostUnstable = true
 }
@@ -87,48 +102,94 @@ func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) {
 	var items []*types.BuildCache
 	for _, r := range duResp.Record {
 		items = append(items, &types.BuildCache{
-			ID:      r.ID,
-			Mutable: r.Mutable,
-			InUse:   r.InUse,
-			Size:    r.Size_,
-
+			ID:          r.ID,
+			Parent:      r.Parent,
+			Type:        r.RecordType,
+			Description: r.Description,
+			InUse:       r.InUse,
+			Shared:      r.Shared,
+			Size:        r.Size_,
 			CreatedAt:   r.CreatedAt,
 			LastUsedAt:  r.LastUsedAt,
 			UsageCount:  int(r.UsageCount),
-			Parent:      r.Parent,
-			Description: r.Description,
 		})
 	}
 	return items, nil
 }
 
 // Prune clears all reclaimable build cache
-func (b *Builder) Prune(ctx context.Context) (int64, error) {
+func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions) (int64, []string, error) {
 	ch := make(chan *controlapi.UsageRecord)
 
 	eg, ctx := errgroup.WithContext(ctx)
 
+	validFilters := make(map[string]bool, 1+len(cacheFields))
+	validFilters["unused-for"] = true
+	for k, v := range cacheFields {
+		validFilters[k] = v
+	}
+	if err := opts.Filters.Validate(validFilters); err != nil {
+		return 0, nil, err
+	}
+
+	var unusedFor time.Duration
+	unusedForValues := opts.Filters.Get("unused-for")
+
+	switch len(unusedForValues) {
+	case 0:
+
+	case 1:
+		var err error
+		unusedFor, err = time.ParseDuration(unusedForValues[0])
+		if err != nil {
+			return 0, nil, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')")
+		}
+
+	default:
+		return 0, nil, errMultipleFilterValues
+	}
+
+	bkFilter := make([]string, 0, opts.Filters.Len())
+	for cacheField := range cacheFields {
+		values := opts.Filters.Get(cacheField)
+		switch len(values) {
+		case 0:
+			bkFilter = append(bkFilter, cacheField)
+		case 1:
+			bkFilter = append(bkFilter, cacheField+"=="+values[0])
+		default:
+			return 0, nil, errMultipleFilterValues
+		}
+	}
+
 	eg.Go(func() error {
 		defer close(ch)
-		return b.controller.Prune(&controlapi.PruneRequest{}, &pruneProxy{
+		return b.controller.Prune(&controlapi.PruneRequest{
+			All:          opts.All,
+			KeepDuration: int64(unusedFor),
+			KeepBytes:    opts.KeepStorage,
+			Filter:       bkFilter,
+		}, &pruneProxy{
 			streamProxy: streamProxy{ctx: ctx},
 			ch:          ch,
 		})
 	})
 
 	var size int64
+	var cacheIDs []string
 	eg.Go(func() error {
 		for r := range ch {
 			size += r.Size_
+			cacheIDs = append(cacheIDs, r.ID)
 		}
 		return nil
 	})
 
 	if err := eg.Wait(); err != nil {
-		return 0, err
+		return 0, nil, err
 	}
 
-	return size, nil
+	return size, cacheIDs, nil
 }
 
 // Build executes a build request

builder/builder-next/controller.go

@@ -9,6 +9,7 @@ import (
 	"github.com/docker/docker/builder/builder-next/adapters/containerimage"
 	"github.com/docker/docker/builder/builder-next/adapters/snapshot"
 	containerimageexp "github.com/docker/docker/builder/builder-next/exporter"
+	"github.com/docker/docker/builder/builder-next/imagerefchecker"
 	mobyworker "github.com/docker/docker/builder/builder-next/worker"
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/moby/buildkit/cache"
@@ -69,9 +70,20 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		MetadataStore: md,
 	})
 
+	layerGetter, ok := sbase.(imagerefchecker.LayerGetter)
+	if !ok {
+		return nil, errors.Errorf("snapshotter does not implement layergetter")
+	}
+
+	refChecker := imagerefchecker.New(imagerefchecker.Opt{
+		ImageStore:  dist.ImageStore,
+		LayerGetter: layerGetter,
+	})
+
 	cm, err := cache.NewManager(cache.ManagerOpt{
-		Snapshotter:   snapshotter,
-		MetadataStore: md,
+		Snapshotter:     snapshotter,
+		MetadataStore:   md,
+		PruneRefChecker: refChecker,
 	})
 	if err != nil {
 		return nil, err

builder/builder-next/imagerefchecker/checker.go

@@ -0,0 +1,96 @@
+package imagerefchecker
+
+import (
+	"sync"
+
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/moby/buildkit/cache"
+)
+
+// LayerGetter abstracts away the snapshotter
+type LayerGetter interface {
+	GetLayer(string) (layer.Layer, error)
+}
+
+// Opt represents the options needed to create a refchecker
+type Opt struct {
+	LayerGetter LayerGetter
+	ImageStore  image.Store
+}
+
+// New creates new image reference checker that can be used to see if a reference
+// is being used by any of the images in the image store
+func New(opt Opt) cache.ExternalRefCheckerFunc {
+	return func() (cache.ExternalRefChecker, error) {
+		return &checker{opt: opt, layers: lchain{}, cache: map[string]bool{}}, nil
+	}
+}
+
+type lchain map[layer.DiffID]lchain
+
+func (c lchain) add(ids []layer.DiffID) {
+	if len(ids) == 0 {
+		return
+	}
+	id := ids[0]
+	ch, ok := c[id]
+	if !ok {
+		ch = lchain{}
+		c[id] = ch
+	}
+	ch.add(ids[1:])
+}
+
+func (c lchain) has(ids []layer.DiffID) bool {
+	if len(ids) == 0 {
+		return true
+	}
+	ch, ok := c[ids[0]]
+	return ok && ch.has(ids[1:])
+}
+
+type checker struct {
+	opt    Opt
+	once   sync.Once
+	layers lchain
+	cache  map[string]bool
+}
+
+func (c *checker) Exists(key string) bool {
+	if c.opt.ImageStore == nil {
+		return false
+	}
+
+	c.once.Do(c.init)
+
+	if b, ok := c.cache[key]; ok {
+		return b
+	}
+
+	l, err := c.opt.LayerGetter.GetLayer(key)
+	if err != nil || l == nil {
+		c.cache[key] = false
+		return false
+	}
+
+	ok := c.layers.has(diffIDs(l))
+	c.cache[key] = ok
+	return ok
+}
+
+func (c *checker) init() {
+	imgs := c.opt.ImageStore.Map()
+
+	for _, img := range imgs {
+		c.layers.add(img.RootFS.DiffIDs)
+	}
+}
+
+func diffIDs(l layer.Layer) []layer.DiffID {
+	p := l.Parent()
+	if p == nil {
+		return []layer.DiffID{l.DiffID()}
+	}
+	return append(diffIDs(p), l.DiffID())
+}

client/build_prune.go

@@ -4,19 +4,34 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/url"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/pkg/errors"
 )
 
 // BuildCachePrune requests the daemon to delete unused cache data
-func (cli *Client) BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error) {
+func (cli *Client) BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error) {
 	if err := cli.NewVersionError("1.31", "build prune"); err != nil {
 		return nil, err
 	}
 
 	report := types.BuildCachePruneReport{}
 
-	serverResp, err := cli.post(ctx, "/build/prune", nil, nil, nil)
+	query := url.Values{}
+	if opts.All {
+		query.Set("all", "1")
+	}
+	query.Set("keep-storage", fmt.Sprintf("%d", opts.KeepStorage))
+	filters, err := filters.ToJSON(opts.Filters)
+	if err != nil {
+		return nil, errors.Wrap(err, "prune could not marshal filters option")
+	}
+	query.Set("filters", filters)
+
+	serverResp, err := cli.post(ctx, "/build/prune", query, nil, nil)
+
 	if err != nil {
 		return nil, err
 	}

client/interface.go

@@ -86,7 +86,7 @@ type DistributionAPIClient interface {
 // ImageAPIClient defines API client methods for the images
 type ImageAPIClient interface {
 	ImageBuild(ctx context.Context, context io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error)
-	BuildCachePrune(ctx context.Context) (*types.BuildCachePruneReport, error)
+	BuildCachePrune(ctx context.Context, opts types.BuildCachePruneOptions) (*types.BuildCachePruneReport, error)
 	BuildCancel(ctx context.Context, id string) error
 	ImageCreate(ctx context.Context, parentReference string, options types.ImageCreateOptions) (io.ReadCloser, error)
 	ImageHistory(ctx context.Context, image string) ([]image.HistoryResponseItem, error)

integration/build/build_session_test.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/docker/docker/api/types"
 	dclient "github.com/docker/docker/client"
 	"github.com/docker/docker/internal/test/fakecontext"
 	"github.com/docker/docker/internal/test/request"
@@ -76,7 +77,7 @@ func TestBuildWithSession(t *testing.T) {
 	assert.Check(t, is.Contains(string(outBytes), "Successfully built"))
 	assert.Check(t, is.Equal(strings.Count(string(outBytes), "Using cache"), 4))
 
-	_, err = client.BuildCachePrune(context.TODO())
+	_, err = client.BuildCachePrune(context.TODO(), types.BuildCachePruneOptions{All: true})
 	assert.Check(t, err)
 
 	du, err = client.DiskUsage(context.TODO())