From 4a776d0ca76c4bdf4399aef8c102361d6c2819eb Mon Sep 17 00:00:00 2001 From: Tibor Vass Date: Wed, 5 Sep 2018 02:12:44 +0000 Subject: [PATCH] builder: use buildkit's GC for build cache This allows users to configure the buildkit GC. The following enables the default GC: ``` { "builder": { "gc": { "enabled": true } } } ``` The default GC policy has a simple config: ``` { "builder": { "gc": { "enabled": true, "defaultKeepStorage": "30GB" } } } ``` A custom GC policy can be used instead by specifying a list of cache prune rules: ``` { "builder": { "gc": { "enabled": true, "policy": [ {"keepStorage": "512MB", "filter": ["unused-for=1400h"]]}, {"keepStorage": "30GB", "all": true} ] } } } ``` Signed-off-by: Tibor Vass --- builder/builder-next/builder.go | 80 ++++++++++++++--------- builder/builder-next/controller.go | 51 +++++++++++++++ builder/builder-next/worker/gc.go | 51 +++++++++++++++ builder/builder-next/worker/gc_unix.go | 17 +++++ builder/builder-next/worker/gc_windows.go | 7 ++ cmd/dockerd/daemon.go | 1 + daemon/config/builder.go | 22 +++++++ daemon/config/config.go | 4 ++ 8 files changed, 201 insertions(+), 32 deletions(-) create mode 100644 builder/builder-next/worker/gc.go create mode 100644 builder/builder-next/worker/gc_unix.go create mode 100644 builder/builder-next/worker/gc_windows.go create mode 100644 daemon/config/builder.go diff --git a/builder/builder-next/builder.go b/builder/builder-next/builder.go index a9792c4e7a..a51d8ec52c 100644 --- a/builder/builder-next/builder.go +++ b/builder/builder-next/builder.go @@ -13,11 +13,13 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/backend" "github.com/docker/docker/builder" + "github.com/docker/docker/daemon/config" "github.com/docker/docker/daemon/images" "github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/pkg/system" "github.com/docker/libnetwork" controlapi "github.com/moby/buildkit/api/services/control" + "github.com/moby/buildkit/client" "github.com/moby/buildkit/control" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/session" @@ -57,6 +59,7 @@ type Opt struct { NetworkController libnetwork.NetworkController DefaultCgroupParent string ResolverOpt resolver.ResolveOptionsFunc + BuilderConfig config.BuilderConfig } // Builder can build using BuildKit backend @@ -134,43 +137,18 @@ func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions) return 0, nil, err } - var unusedFor time.Duration - unusedForValues := opts.Filters.Get("unused-for") - - switch len(unusedForValues) { - case 0: - - case 1: - var err error - unusedFor, err = time.ParseDuration(unusedForValues[0]) - if err != nil { - return 0, nil, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')") - } - - default: - return 0, nil, errMultipleFilterValues - } - - bkFilter := make([]string, 0, opts.Filters.Len()) - for cacheField := range cacheFields { - values := opts.Filters.Get(cacheField) - switch len(values) { - case 0: - bkFilter = append(bkFilter, cacheField) - case 1: - bkFilter = append(bkFilter, cacheField+"=="+values[0]) - default: - return 0, nil, errMultipleFilterValues - } + pi, err := toBuildkitPruneInfo(opts) + if err != nil { + return 0, nil, err } eg.Go(func() error { defer close(ch) return b.controller.Prune(&controlapi.PruneRequest{ - All: opts.All, - KeepDuration: int64(unusedFor), - KeepBytes: opts.KeepStorage, - Filter: bkFilter, + All: pi.All, + KeepDuration: int64(pi.KeepDuration), + KeepBytes: pi.KeepBytes, + Filter: pi.Filter, }, &pruneProxy{ streamProxy: streamProxy{ctx: ctx}, ch: ch, @@ -531,3 +509,41 @@ func toBuildkitExtraHosts(inp []string) (string, error) { } return strings.Join(hosts, ","), nil } + +func toBuildkitPruneInfo(opts types.BuildCachePruneOptions) (client.PruneInfo, error) { + var unusedFor time.Duration + unusedForValues := opts.Filters.Get("unused-for") + + switch len(unusedForValues) { + case 0: + + case 1: + var err error + unusedFor, err = time.ParseDuration(unusedForValues[0]) + if err != nil { + return client.PruneInfo{}, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')") + } + + default: + return client.PruneInfo{}, errMultipleFilterValues + } + + bkFilter := make([]string, 0, opts.Filters.Len()) + for cacheField := range cacheFields { + values := opts.Filters.Get(cacheField) + switch len(values) { + case 0: + bkFilter = append(bkFilter, cacheField) + case 1: + bkFilter = append(bkFilter, cacheField+"=="+values[0]) + default: + return client.PruneInfo{}, errMultipleFilterValues + } + } + return client.PruneInfo{ + All: opts.All, + KeepDuration: unusedFor, + KeepBytes: opts.KeepStorage, + Filter: bkFilter, + }, nil +} diff --git a/builder/builder-next/controller.go b/builder/builder-next/controller.go index cb9819cdd6..ebb28226d2 100644 --- a/builder/builder-next/controller.go +++ b/builder/builder-next/controller.go @@ -6,15 +6,19 @@ import ( "path/filepath" "github.com/containerd/containerd/content/local" + "github.com/docker/docker/api/types" "github.com/docker/docker/builder/builder-next/adapters/containerimage" "github.com/docker/docker/builder/builder-next/adapters/snapshot" containerimageexp "github.com/docker/docker/builder/builder-next/exporter" "github.com/docker/docker/builder/builder-next/imagerefchecker" mobyworker "github.com/docker/docker/builder/builder-next/worker" + "github.com/docker/docker/daemon/config" "github.com/docker/docker/daemon/graphdriver" + units "github.com/docker/go-units" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/metadata" registryremotecache "github.com/moby/buildkit/cache/remotecache/registry" + "github.com/moby/buildkit/client" "github.com/moby/buildkit/control" "github.com/moby/buildkit/exporter" "github.com/moby/buildkit/frontend" @@ -127,12 +131,18 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) { return nil, err } + gcPolicy, err := getGCPolicy(opt.BuilderConfig, root) + if err != nil { + return nil, errors.Wrap(err, "could not get builder GC policy") + } + wopt := mobyworker.Opt{ ID: "moby", SessionManager: opt.SessionManager, MetadataStore: md, ContentStore: store, CacheManager: cm, + GCPolicy: gcPolicy, Snapshotter: snapshotter, Executor: exec, ImageSource: src, @@ -165,3 +175,44 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) { // TODO: set ResolveCacheExporterFunc for exporting cache }) } + +func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, error) { + var gcPolicy []client.PruneInfo + if conf.GC.Enabled { + var ( + defaultKeepStorage int64 + err error + ) + + if conf.GC.DefaultKeepStorage != "" { + defaultKeepStorage, err = units.RAMInBytes(conf.GC.DefaultKeepStorage) + if err != nil { + return nil, errors.Wrapf(err, "could not parse '%s' as Builder.GC.DefaultKeepStorage config", conf.GC.DefaultKeepStorage) + } + } + + if conf.GC.Policy == nil { + gcPolicy = mobyworker.DefaultGCPolicy(root, defaultKeepStorage) + } else { + gcPolicy = make([]client.PruneInfo, len(conf.GC.Policy)) + for i, p := range conf.GC.Policy { + b, err := units.RAMInBytes(p.KeepStorage) + if err != nil { + return nil, err + } + if b == 0 { + b = defaultKeepStorage + } + gcPolicy[i], err = toBuildkitPruneInfo(types.BuildCachePruneOptions{ + All: p.All, + KeepStorage: b, + Filters: p.Filter, + }) + if err != nil { + return nil, err + } + } + } + } + return gcPolicy, nil +} diff --git a/builder/builder-next/worker/gc.go b/builder/builder-next/worker/gc.go new file mode 100644 index 0000000000..13e65f0e6f --- /dev/null +++ b/builder/builder-next/worker/gc.go @@ -0,0 +1,51 @@ +package worker + +import ( + "math" + + "github.com/moby/buildkit/client" +) + +const defaultCap int64 = 2e9 // 2GB + +// tempCachePercent represents the percentage ratio of the cache size in bytes to temporarily keep for a short period of time (couple of days) +// over the total cache size in bytes. Because there is no perfect value, a mathematically pleasing one was chosen. +// The value is approximately 13.8 +const tempCachePercent = math.E * math.Pi * math.Phi + +// DefaultGCPolicy returns a default builder GC policy +func DefaultGCPolicy(p string, defaultKeepBytes int64) []client.PruneInfo { + keep := defaultKeepBytes + if defaultKeepBytes == 0 { + keep = detectDefaultGCCap(p) + } + + tempCacheKeepBytes := int64(math.Round(float64(keep) / 100. * float64(tempCachePercent))) + const minTempCacheKeepBytes = 512 * 1e6 // 512MB + if tempCacheKeepBytes < minTempCacheKeepBytes { + tempCacheKeepBytes = minTempCacheKeepBytes + } + + return []client.PruneInfo{ + // if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days + { + Filter: []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"}, + KeepDuration: 48 * 3600, // 48h + KeepBytes: tempCacheKeepBytes, + }, + // remove any data not used for 60 days + { + KeepDuration: 60 * 24 * 3600, // 60d + KeepBytes: keep, + }, + // keep the unshared build cache under cap + { + KeepBytes: keep, + }, + // if previous policies were insufficient start deleting internal data to keep build cache under cap + { + All: true, + KeepBytes: keep, + }, + } +} diff --git a/builder/builder-next/worker/gc_unix.go b/builder/builder-next/worker/gc_unix.go new file mode 100644 index 0000000000..7457a06654 --- /dev/null +++ b/builder/builder-next/worker/gc_unix.go @@ -0,0 +1,17 @@ +// +build !windows + +package worker + +import ( + "syscall" +) + +func detectDefaultGCCap(root string) int64 { + var st syscall.Statfs_t + if err := syscall.Statfs(root, &st); err != nil { + return defaultCap + } + diskSize := int64(st.Bsize) * int64(st.Blocks) // nolint unconvert + avail := diskSize / 10 + return (avail/(1<<30) + 1) * 1e9 // round up +} diff --git a/builder/builder-next/worker/gc_windows.go b/builder/builder-next/worker/gc_windows.go new file mode 100644 index 0000000000..1ffcce41db --- /dev/null +++ b/builder/builder-next/worker/gc_windows.go @@ -0,0 +1,7 @@ +// +build windows + +package worker + +func detectDefaultGCCap(root string) int64 { + return defaultCap +} diff --git a/cmd/dockerd/daemon.go b/cmd/dockerd/daemon.go index b7b902bdf6..3f9f7323fb 100644 --- a/cmd/dockerd/daemon.go +++ b/cmd/dockerd/daemon.go @@ -292,6 +292,7 @@ func newRouterOptions(config *config.Config, d *daemon.Daemon) (routerOptions, e NetworkController: d.NetworkController(), DefaultCgroupParent: cgroupParent, ResolverOpt: d.NewResolveOptionsFunc(), + BuilderConfig: config.Builder, }) if err != nil { return opts, err diff --git a/daemon/config/builder.go b/daemon/config/builder.go new file mode 100644 index 0000000000..ac85e76b30 --- /dev/null +++ b/daemon/config/builder.go @@ -0,0 +1,22 @@ +package config + +import "github.com/docker/docker/api/types/filters" + +// BuilderGCRule represents a GC rule for buildkit cache +type BuilderGCRule struct { + All bool `json:",omitempty"` + Filter filters.Args `json:",omitempty"` + KeepStorage string `json:",omitempty"` +} + +// BuilderGCConfig contains GC config for a buildkit builder +type BuilderGCConfig struct { + Enabled bool `json:",omitempty"` + Policy []BuilderGCRule `json:",omitempty"` + DefaultKeepStorage string `json:",omitempty"` +} + +// BuilderConfig contains config for the builder +type BuilderConfig struct { + GC BuilderGCConfig `json:",omitempty"` +} diff --git a/daemon/config/config.go b/daemon/config/config.go index 9f215e3068..8b2c844a57 100644 --- a/daemon/config/config.go +++ b/daemon/config/config.go @@ -55,6 +55,7 @@ var flatOptions = map[string]bool{ "runtimes": true, "default-ulimits": true, "features": true, + "builder": true, } // skipValidateOptions contains configuration keys @@ -62,6 +63,7 @@ var flatOptions = map[string]bool{ // for unknown flag validation. var skipValidateOptions = map[string]bool{ "features": true, + "builder": true, } // skipDuplicates contains configuration keys that @@ -225,6 +227,8 @@ type CommonConfig struct { // Features contains a list of feature key value pairs indicating what features are enabled or disabled. // If a certain feature doesn't appear in this list then it's unset (i.e. neither true nor false). Features map[string]bool `json:"features,omitempty"` + + Builder BuilderConfig `json:"builder,omitempty"` } // IsValueSet returns true if a configuration value