builder: use buildkit's GC for build cache

This allows users to configure the buildkit GC.

The following enables the default GC:
```
{
  "builder": {
    "gc": {
      "enabled": true
    }
  }
}
```

The default GC policy has a simple config:
```
{
  "builder": {
    "gc": {
      "enabled": true,
      "defaultKeepStorage": "30GB"
    }
  }
}
```

A custom GC policy can be used instead by specifying a list of cache prune rules:
```
{
  "builder": {
    "gc": {
      "enabled": true,
      "policy": [
        {"keepStorage": "512MB", "filter": ["unused-for=1400h"]]},
        {"keepStorage": "30GB", "all": true}
      ]
    }
  }
}
```

Signed-off-by: Tibor Vass <tibor@docker.com>

builder/builder-next/builder.go

@@ -13,11 +13,13 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/builder"
+	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/daemon/images"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/libnetwork"
 	controlapi "github.com/moby/buildkit/api/services/control"
+	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/control"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/session"
@@ -57,6 +59,7 @@ type Opt struct {
 	NetworkController   libnetwork.NetworkController
 	DefaultCgroupParent string
 	ResolverOpt         resolver.ResolveOptionsFunc
+	BuilderConfig       config.BuilderConfig
 }
 
 // Builder can build using BuildKit backend
@@ -134,43 +137,18 @@ func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions)
 		return 0, nil, err
 	}
 
-	var unusedFor time.Duration
+	pi, err := toBuildkitPruneInfo(opts)
-	unusedForValues := opts.Filters.Get("unused-for")
+	if err != nil {
-
+		return 0, nil, err
-	switch len(unusedForValues) {
-	case 0:
-
-	case 1:
-		var err error
-		unusedFor, err = time.ParseDuration(unusedForValues[0])
-		if err != nil {
-			return 0, nil, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')")
-		}
-
-	default:
-		return 0, nil, errMultipleFilterValues
-	}
-
-	bkFilter := make([]string, 0, opts.Filters.Len())
-	for cacheField := range cacheFields {
-		values := opts.Filters.Get(cacheField)
-		switch len(values) {
-		case 0:
-			bkFilter = append(bkFilter, cacheField)
-		case 1:
-			bkFilter = append(bkFilter, cacheField+"=="+values[0])
-		default:
-			return 0, nil, errMultipleFilterValues
-		}
 	}
 
 	eg.Go(func() error {
 		defer close(ch)
 		return b.controller.Prune(&controlapi.PruneRequest{
-			All:          opts.All,
+			All:          pi.All,
-			KeepDuration: int64(unusedFor),
+			KeepDuration: int64(pi.KeepDuration),
-			KeepBytes:    opts.KeepStorage,
+			KeepBytes:    pi.KeepBytes,
-			Filter:       bkFilter,
+			Filter:       pi.Filter,
 		}, &pruneProxy{
 			streamProxy: streamProxy{ctx: ctx},
 			ch:          ch,
@@ -531,3 +509,41 @@ func toBuildkitExtraHosts(inp []string) (string, error) {
 	}
 	return strings.Join(hosts, ","), nil
 }
+
+func toBuildkitPruneInfo(opts types.BuildCachePruneOptions) (client.PruneInfo, error) {
+	var unusedFor time.Duration
+	unusedForValues := opts.Filters.Get("unused-for")
+
+	switch len(unusedForValues) {
+	case 0:
+
+	case 1:
+		var err error
+		unusedFor, err = time.ParseDuration(unusedForValues[0])
+		if err != nil {
+			return client.PruneInfo{}, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')")
+		}
+
+	default:
+		return client.PruneInfo{}, errMultipleFilterValues
+	}
+
+	bkFilter := make([]string, 0, opts.Filters.Len())
+	for cacheField := range cacheFields {
+		values := opts.Filters.Get(cacheField)
+		switch len(values) {
+		case 0:
+			bkFilter = append(bkFilter, cacheField)
+		case 1:
+			bkFilter = append(bkFilter, cacheField+"=="+values[0])
+		default:
+			return client.PruneInfo{}, errMultipleFilterValues
+		}
+	}
+	return client.PruneInfo{
+		All:          opts.All,
+		KeepDuration: unusedFor,
+		KeepBytes:    opts.KeepStorage,
+		Filter:       bkFilter,
+	}, nil
+}

builder/builder-next/controller.go

@@ -6,15 +6,19 @@ import (
 	"path/filepath"
 
 	"github.com/containerd/containerd/content/local"
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/builder/builder-next/adapters/containerimage"
 	"github.com/docker/docker/builder/builder-next/adapters/snapshot"
 	containerimageexp "github.com/docker/docker/builder/builder-next/exporter"
 	"github.com/docker/docker/builder/builder-next/imagerefchecker"
 	mobyworker "github.com/docker/docker/builder/builder-next/worker"
+	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/daemon/graphdriver"
+	units "github.com/docker/go-units"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/cache/metadata"
 	registryremotecache "github.com/moby/buildkit/cache/remotecache/registry"
+	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/control"
 	"github.com/moby/buildkit/exporter"
 	"github.com/moby/buildkit/frontend"
@@ -127,12 +131,18 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		return nil, err
 	}
 
+	gcPolicy, err := getGCPolicy(opt.BuilderConfig, root)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get builder GC policy")
+	}
+
 	wopt := mobyworker.Opt{
 		ID:                "moby",
 		SessionManager:    opt.SessionManager,
 		MetadataStore:     md,
 		ContentStore:      store,
 		CacheManager:      cm,
+		GCPolicy:          gcPolicy,
 		Snapshotter:       snapshotter,
 		Executor:          exec,
 		ImageSource:       src,
@@ -165,3 +175,44 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		// TODO: set ResolveCacheExporterFunc for exporting cache
 	})
 }
+
+func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, error) {
+	var gcPolicy []client.PruneInfo
+	if conf.GC.Enabled {
+		var (
+			defaultKeepStorage int64
+			err                error
+		)
+
+		if conf.GC.DefaultKeepStorage != "" {
+			defaultKeepStorage, err = units.RAMInBytes(conf.GC.DefaultKeepStorage)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not parse '%s' as Builder.GC.DefaultKeepStorage config", conf.GC.DefaultKeepStorage)
+			}
+		}
+
+		if conf.GC.Policy == nil {
+			gcPolicy = mobyworker.DefaultGCPolicy(root, defaultKeepStorage)
+		} else {
+			gcPolicy = make([]client.PruneInfo, len(conf.GC.Policy))
+			for i, p := range conf.GC.Policy {
+				b, err := units.RAMInBytes(p.KeepStorage)
+				if err != nil {
+					return nil, err
+				}
+				if b == 0 {
+					b = defaultKeepStorage
+				}
+				gcPolicy[i], err = toBuildkitPruneInfo(types.BuildCachePruneOptions{
+					All:         p.All,
+					KeepStorage: b,
+					Filters:     p.Filter,
+				})
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+	}
+	return gcPolicy, nil
+}

builder/builder-next/worker/gc.go

@@ -0,0 +1,51 @@
+package worker
+
+import (
+	"math"
+
+	"github.com/moby/buildkit/client"
+)
+
+const defaultCap int64 = 2e9 // 2GB
+
+// tempCachePercent represents the percentage ratio of the cache size in bytes to temporarily keep for a short period of time (couple of days)
+// over the total cache size in bytes. Because there is no perfect value, a mathematically pleasing one was chosen.
+// The value is approximately 13.8
+const tempCachePercent = math.E * math.Pi * math.Phi
+
+// DefaultGCPolicy returns a default builder GC policy
+func DefaultGCPolicy(p string, defaultKeepBytes int64) []client.PruneInfo {
+	keep := defaultKeepBytes
+	if defaultKeepBytes == 0 {
+		keep = detectDefaultGCCap(p)
+	}
+
+	tempCacheKeepBytes := int64(math.Round(float64(keep) / 100. * float64(tempCachePercent)))
+	const minTempCacheKeepBytes = 512 * 1e6 // 512MB
+	if tempCacheKeepBytes < minTempCacheKeepBytes {
+		tempCacheKeepBytes = minTempCacheKeepBytes
+	}
+
+	return []client.PruneInfo{
+		// if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days
+		{
+			Filter:       []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
+			KeepDuration: 48 * 3600, // 48h
+			KeepBytes:    tempCacheKeepBytes,
+		},
+		// remove any data not used for 60 days
+		{
+			KeepDuration: 60 * 24 * 3600, // 60d
+			KeepBytes:    keep,
+		},
+		// keep the unshared build cache under cap
+		{
+			KeepBytes: keep,
+		},
+		// if previous policies were insufficient start deleting internal data to keep build cache under cap
+		{
+			All:       true,
+			KeepBytes: keep,
+		},
+	}
+}

builder/builder-next/worker/gc_unix.go

@@ -0,0 +1,17 @@
+// +build !windows
+
+package worker
+
+import (
+	"syscall"
+)
+
+func detectDefaultGCCap(root string) int64 {
+	var st syscall.Statfs_t
+	if err := syscall.Statfs(root, &st); err != nil {
+		return defaultCap
+	}
+	diskSize := int64(st.Bsize) * int64(st.Blocks) // nolint unconvert
+	avail := diskSize / 10
+	return (avail/(1<<30) + 1) * 1e9 // round up
+}

builder/builder-next/worker/gc_windows.go

@@ -0,0 +1,7 @@
+// +build windows
+
+package worker
+
+func detectDefaultGCCap(root string) int64 {
+	return defaultCap
+}

cmd/dockerd/daemon.go

@@ -292,6 +292,7 @@ func newRouterOptions(config *config.Config, d *daemon.Daemon) (routerOptions, e
 		NetworkController:   d.NetworkController(),
 		DefaultCgroupParent: cgroupParent,
 		ResolverOpt:         d.NewResolveOptionsFunc(),
+		BuilderConfig:       config.Builder,
 	})
 	if err != nil {
 		return opts, err

daemon/config/builder.go

@@ -0,0 +1,22 @@
+package config
+
+import "github.com/docker/docker/api/types/filters"
+
+// BuilderGCRule represents a GC rule for buildkit cache
+type BuilderGCRule struct {
+	All         bool         `json:",omitempty"`
+	Filter      filters.Args `json:",omitempty"`
+	KeepStorage string       `json:",omitempty"`
+}
+
+// BuilderGCConfig contains GC config for a buildkit builder
+type BuilderGCConfig struct {
+	Enabled            bool            `json:",omitempty"`
+	Policy             []BuilderGCRule `json:",omitempty"`
+	DefaultKeepStorage string          `json:",omitempty"`
+}
+
+// BuilderConfig contains config for the builder
+type BuilderConfig struct {
+	GC BuilderGCConfig `json:",omitempty"`
+}

daemon/config/config.go

@@ -55,6 +55,7 @@ var flatOptions = map[string]bool{
 	"runtimes":           true,
 	"default-ulimits":    true,
 	"features":           true,
+	"builder":            true,
 }
 
 // skipValidateOptions contains configuration keys
@@ -62,6 +63,7 @@ var flatOptions = map[string]bool{
 // for unknown flag validation.
 var skipValidateOptions = map[string]bool{
 	"features": true,
+	"builder":  true,
 }
 
 // skipDuplicates contains configuration keys that
@@ -225,6 +227,8 @@ type CommonConfig struct {
 	// Features contains a list of feature key value pairs indicating what features are enabled or disabled.
 	// If a certain feature doesn't appear in this list then it's unset (i.e. neither true nor false).
 	Features map[string]bool `json:"features,omitempty"`
+
+	Builder BuilderConfig `json:"builder,omitempty"`
 }
 
 // IsValueSet returns true if a configuration value