kotovalexarian-likes-github/moby--moby
1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00
Code Releases Activity

Add label filter for docker system prune

Browse source 
This fix tries to address the issue raised in 29999 where it was not
possible to mask these items (like important non-removable stuff)
from `docker system prune`.

This fix adds `label` and `label!` field for `--filter` in `system prune`,
so that it is possible to selectively prune items like:
```
$ docker container prune --filter label=foo

$ docker container prune --filter label!=bar
```

Additional unit tests and integration tests have been added.

This fix fixes 29999.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
This commit is contained in:
Yong Tang 2017-02-04 09:10:05 -08:00
parent e4c608abe9
commit 7025247324
14 changed files with 308 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
api/server/router/volume/volume_routes.go
View file

@ -72,7 +72,12 @@ func (v *volumeRouter) postVolumesPrune(ctx context.Context, w http.ResponseWrit
return err
}
pruneReport, err := v.backend.VolumesPrune(filters.Args{})
pruneFilters, err := filters.FromParam(r.Form.Get("filters"))
if err != nil {
return err
}
pruneReport, err := v.backend.VolumesPrune(pruneFilters)
if err != nil {
return err
}

2
cli/command/container/prune.go
View file

@ -49,7 +49,7 @@ const warning = `WARNING! This will remove all stopped containers.
Are you sure you want to continue?`
func runPrune(dockerCli *command.DockerCli, opts pruneOptions) (spaceReclaimed uint64, output string, err error) {
pruneFilters := opts.filter.Value()
pruneFilters := command.PruneFilters(dockerCli, opts.filter.Value())
if !opts.force && !command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), warning) {
return

1
cli/command/image/prune.go
View file

@ -58,6 +58,7 @@ Are you sure you want to continue?`
func runPrune(dockerCli *command.DockerCli, opts pruneOptions) (spaceReclaimed uint64, output string, err error) {
pruneFilters := opts.filter.Value()
pruneFilters.Add("dangling", fmt.Sprintf("%v", !opts.all))
pruneFilters = command.PruneFilters(dockerCli, pruneFilters)
warning := danglingWarning
if opts.all {

2
cli/command/network/prune.go
View file

@ -48,7 +48,7 @@ const warning = `WARNING! This will remove all networks not used by at least one
Are you sure you want to continue?`
func runPrune(dockerCli *command.DockerCli, opts pruneOptions) (output string, err error) {
pruneFilters := opts.filter.Value()
pruneFilters := command.PruneFilters(dockerCli, opts.filter.Value())
if !opts.force && !command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), warning) {
return

2
cli/command/prune/prune.go
View file

@ -37,7 +37,7 @@ func RunContainerPrune(dockerCli *command.DockerCli, filter opts.FilterOpt) (uin
// RunVolumePrune executes a prune command for volumes
func RunVolumePrune(dockerCli *command.DockerCli, filter opts.FilterOpt) (uint64, string, error) {
return volume.RunPrune(dockerCli)
return volume.RunPrune(dockerCli, filter)
}
// RunImagePrune executes a prune command for images

32
cli/command/utils.go
View file

@ -9,6 +9,7 @@ import (
"runtime"
"strings"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/pkg/system"
)
@ -85,3 +86,34 @@ func PromptForConfirmation(ins *InStream, outs *OutStream, message string) bool
answer, _, _ := reader.ReadLine()
return strings.ToLower(string(answer)) == "y"
}
// PruneFilters returns consolidated prune filters obtained from config.json and cli
func PruneFilters(dockerCli Cli, pruneFilters filters.Args) filters.Args {
if dockerCli.ConfigFile() == nil {
return pruneFilters
}
for _, f := range dockerCli.ConfigFile().PruneFilters {
parts := strings.SplitN(f, "=", 2)
if len(parts) != 2 {
continue
}
if parts[0] == "label" {
// CLI label filter supersede config.json.
// If CLI label filter conflict with config.json,
// skip adding label! filter in config.json.
if pruneFilters.Include("label!") && pruneFilters.ExactMatch("label!", parts[1]) {
continue
}
} else if parts[0] == "label!" {
// CLI label! filter supersede config.json.
// If CLI label! filter conflict with config.json,
// skip adding label filter in config.json.
if pruneFilters.Include("label") && pruneFilters.ExactMatch("label", parts[1]) {
continue
}
}
pruneFilters.Add(parts[0], parts[1])
}
return pruneFilters
}

16
cli/command/volume/prune.go
View file

@ -3,21 +3,22 @@ package volume
import (
"fmt"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/cli"
"github.com/docker/docker/cli/command"
"github.com/docker/docker/opts"
units "github.com/docker/go-units"
"github.com/spf13/cobra"
"golang.org/x/net/context"
)
type pruneOptions struct {
force bool
force bool
filter opts.FilterOpt
}
// NewPruneCommand returns a new cobra prune command for volumes
func NewPruneCommand(dockerCli command.Cli) *cobra.Command {
var opts pruneOptions
opts := pruneOptions{filter: opts.NewFilterOpt()}
cmd := &cobra.Command{
Use: "prune [OPTIONS]",
@ -39,6 +40,7 @@ func NewPruneCommand(dockerCli command.Cli) *cobra.Command {
flags := cmd.Flags()
flags.BoolVarP(&opts.force, "force", "f", false, "Do not prompt for confirmation")
flags.Var(&opts.filter, "filter", "Provide filter values (e.g. 'label=<label>')")
return cmd
}
@ -47,11 +49,13 @@ const warning = `WARNING! This will remove all volumes not used by at least one
Are you sure you want to continue?`
func runPrune(dockerCli command.Cli, opts pruneOptions) (spaceReclaimed uint64, output string, err error) {
pruneFilters := command.PruneFilters(dockerCli, opts.filter.Value())
if !opts.force && !command.PromptForConfirmation(dockerCli.In(), dockerCli.Out(), warning) {
return
}
report, err := dockerCli.Client().VolumesPrune(context.Background(), filters.Args{})
report, err := dockerCli.Client().VolumesPrune(context.Background(), pruneFilters)
if err != nil {
return
}
@ -69,6 +73,6 @@ func runPrune(dockerCli command.Cli, opts pruneOptions) (spaceReclaimed uint64,
// RunPrune calls the Volume Prune API
// This returns the amount of space reclaimed and a detailed output string
func RunPrune(dockerCli *command.DockerCli) (uint64, string, error) {
return runPrune(dockerCli, pruneOptions{force: true})
func RunPrune(dockerCli *command.DockerCli, filter opts.FilterOpt) (uint64, string, error) {
return runPrune(dockerCli, pruneOptions{force: true, filter: filter})
}

1
cli/config/configfile/file.go
View file

@ -39,6 +39,7 @@ type ConfigFile struct {
TasksFormat string `json:"tasksFormat,omitempty"`
SecretFormat string `json:"secretFormat,omitempty"`
NodesFormat string `json:"nodesFormat,omitempty"`
PruneFilters []string `json:"pruneFilters,omitempty"`
}
// LegacyLoadFromReader reads the non-nested configuration data given and sets up the

13
client/container_prune_test.go
View file

@ -40,6 +40,11 @@ func TestContainersPrune(t *testing.T) {
danglingUntilFilters.Add("dangling", "true")
danglingUntilFilters.Add("until", "2016-12-15T14:00")
labelFilters := filters.NewArgs()
labelFilters.Add("dangling", "true")
labelFilters.Add("label", "label1=foo")
labelFilters.Add("label", "label2!=bar")
listCases := []struct {
filters filters.Args
expectedQueryParams map[string]string
@ -76,6 +81,14 @@ func TestContainersPrune(t *testing.T) {
"filters": `{"dangling":{"false":true}}`,
},
},
{
filters: labelFilters,
expectedQueryParams: map[string]string{
"until": "",
"filter": "",
"filters": `{"dangling":{"true":true},"label":{"label1=foo":true,"label2!=bar":true}}`,
},
},
}
for _, listCase := range listCases {
client := &Client{

13
client/image_prune_test.go
View file

@ -36,6 +36,11 @@ func TestImagesPrune(t *testing.T) {
noDanglingFilters := filters.NewArgs()
noDanglingFilters.Add("dangling", "false")
labelFilters := filters.NewArgs()
labelFilters.Add("dangling", "true")
labelFilters.Add("label", "label1=foo")
labelFilters.Add("label", "label2!=bar")
listCases := []struct {
filters filters.Args
expectedQueryParams map[string]string
@ -64,6 +69,14 @@ func TestImagesPrune(t *testing.T) {
"filters": `{"dangling":{"false":true}}`,
},
},
{
filters: labelFilters,
expectedQueryParams: map[string]string{
"until": "",
"filter": "",
"filters": `{"dangling":{"true":true},"label":{"label1=foo":true,"label2!=bar":true}}`,
},
},
}
for _, listCase := range listCases {
client := &Client{

13
client/network_prune_test.go
View file

@ -38,6 +38,11 @@ func TestNetworksPrune(t *testing.T) {
noDanglingFilters := filters.NewArgs()
noDanglingFilters.Add("dangling", "false")
labelFilters := filters.NewArgs()
labelFilters.Add("dangling", "true")
labelFilters.Add("label", "label1=foo")
labelFilters.Add("label", "label2!=bar")
listCases := []struct {
filters filters.Args
expectedQueryParams map[string]string
@ -66,6 +71,14 @@ func TestNetworksPrune(t *testing.T) {
"filters": `{"dangling":{"false":true}}`,
},
},
{
filters: labelFilters,
expectedQueryParams: map[string]string{
"until": "",
"filter": "",
"filters": `{"dangling":{"true":true},"label":{"label1=foo":true,"label2!=bar":true}}`,
},
},
}
for _, listCase := range listCases {
client := &Client{

40
daemon/prune.go
View file

@ -34,6 +34,9 @@ func (daemon *Daemon) ContainersPrune(pruneFilters filters.Args) (*types.Contain
if !until.IsZero() && c.Created.After(until) {
continue
}
if !matchLabels(pruneFilters, c.Config.Labels) {
continue
}
cSize, _ := daemon.getSize(c.ID)
// TODO: sets RmLink to true?
err := daemon.ContainerRm(c.ID, &types.ContainerRmConfig{})
@ -60,6 +63,12 @@ func (daemon *Daemon) VolumesPrune(pruneFilters filters.Args) (*types.VolumesPru
refs := daemon.volumes.Refs(v)
if len(refs) == 0 {
detailedVolume, ok := v.(volume.DetailedVolume)
if ok {
if !matchLabels(pruneFilters, detailedVolume.Labels()) {
return nil
}
}
vSize, err := directory.Size(v.Path())
if err != nil {
logrus.Warnf("could not determine size of volume %s: %v", name, err)
@ -122,6 +131,9 @@ func (daemon *Daemon) ImagesPrune(pruneFilters filters.Args) (*types.ImagesPrune
if !until.IsZero() && img.Created.After(until) {
continue
}
if !matchLabels(pruneFilters, img.Config.Labels) {
continue
}
topImages[id] = img
}
@ -200,6 +212,9 @@ func (daemon *Daemon) localNetworksPrune(pruneFilters filters.Args) *types.Netwo
if !until.IsZero() && nw.Info().Created().After(until) {
return false
}
if !matchLabels(pruneFilters, nw.Info().Labels()) {
return false
}
nwName := nw.Name()
if runconfig.IsPreDefinedNetwork(nwName) {
return false
@ -243,6 +258,9 @@ func (daemon *Daemon) clusterNetworksPrune(pruneFilters filters.Args) (*types.Ne
if !until.IsZero() && nw.Created.After(until) {
continue
}
if !matchLabels(pruneFilters, nw.Labels) {
continue
}
// https://github.com/docker/docker/issues/24186
// `docker network inspect` unfortunately displays ONLY those containers that are local to that node.
// So we try to remove it anyway and check the error
@ -266,12 +284,10 @@ func (daemon *Daemon) NetworksPrune(pruneFilters filters.Args) (*types.NetworksP
return nil, err
}
clusterRep, err := daemon.clusterNetworksPrune(pruneFilters)
if err != nil {
return nil, fmt.Errorf("could not remove cluster networks: %s", err)
}
rep := &types.NetworksPruneReport{}
rep.NetworksDeleted = append(rep.NetworksDeleted, clusterRep.NetworksDeleted...)
if clusterRep, err := daemon.clusterNetworksPrune(pruneFilters); err == nil {
rep.NetworksDeleted = append(rep.NetworksDeleted, clusterRep.NetworksDeleted...)
}
localRep := daemon.localNetworksPrune(pruneFilters)
rep.NetworksDeleted = append(rep.NetworksDeleted, localRep.NetworksDeleted...)
@ -298,3 +314,17 @@ func getUntilFromPruneFilters(pruneFilters filters.Args) (time.Time, error) {
until = time.Unix(seconds, nanoseconds)
return until, nil
}
func matchLabels(pruneFilters filters.Args, labels map[string]string) bool {
if !pruneFilters.MatchKVList("label", labels) {
return false
}
// By default MatchKVList will return true if field (like 'label!') does not exist
// So we have to add additional Include("label!") check
if pruneFilters.Include("label!") {
if pruneFilters.MatchKVList("label!", labels) {
return false
}
}
return true
}

5
daemon/volumes.go
View file

@ -295,9 +295,12 @@ func (daemon *Daemon) traverseLocalVolumes(fn func(volume.Volume) error) error {
for _, v := range vols {
name := v.Name()
_, err := daemon.volumes.Get(name)
vol, err := daemon.volumes.Get(name)
if err != nil {
logrus.Warnf("failed to retrieve volume %s from store: %v", name, err)
} else {
// daemon.volumes.Get will return DetailedVolume
v = vol
}
err = fn(v)

177
integration-cli/docker_cli_prune_unix_test.go
View file

@ -3,6 +3,9 @@
package main
import (
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"time"
@ -111,3 +114,177 @@ func (s *DockerSuite) TestPruneContainerUntil(c *check.C) {
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id1)
c.Assert(strings.TrimSpace(out), checker.Contains, id2)
}
func (s *DockerSuite) TestPruneContainerLabel(c *check.C) {
out, _ := dockerCmd(c, "run", "-d", "--label", "foo", "busybox")
id1 := strings.TrimSpace(out)
c.Assert(waitExited(id1, 5*time.Second), checker.IsNil)
out, _ = dockerCmd(c, "run", "-d", "--label", "bar", "busybox")
id2 := strings.TrimSpace(out)
c.Assert(waitExited(id2, 5*time.Second), checker.IsNil)
out, _ = dockerCmd(c, "run", "-d", "busybox")
id3 := strings.TrimSpace(out)
c.Assert(waitExited(id3, 5*time.Second), checker.IsNil)
out, _ = dockerCmd(c, "run", "-d", "--label", "foobar", "busybox")
id4 := strings.TrimSpace(out)
c.Assert(waitExited(id4, 5*time.Second), checker.IsNil)
// Add a config file of label=foobar, that will have no impact if cli is label!=foobar
config := `{"pruneFilters": ["label=foobar"]}`
d, err := ioutil.TempDir("", "integration-cli-")
c.Assert(err, checker.IsNil)
defer os.RemoveAll(d)
err = ioutil.WriteFile(filepath.Join(d, "config.json"), []byte(config), 0644)
c.Assert(err, checker.IsNil)
// With config.json only, prune based on label=foobar
out, _ = dockerCmd(c, "--config", d, "container", "prune", "--force")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id1)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id2)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id3)
c.Assert(strings.TrimSpace(out), checker.Contains, id4)
out, _ = dockerCmd(c, "container", "prune", "--force", "--filter", "label=foo")
c.Assert(strings.TrimSpace(out), checker.Contains, id1)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id2)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id3)
out, _ = dockerCmd(c, "ps", "-a", "-q", "--no-trunc")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id1)
c.Assert(strings.TrimSpace(out), checker.Contains, id2)
c.Assert(strings.TrimSpace(out), checker.Contains, id3)
out, _ = dockerCmd(c, "container", "prune", "--force", "--filter", "label!=bar")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id2)
c.Assert(strings.TrimSpace(out), checker.Contains, id3)
out, _ = dockerCmd(c, "ps", "-a", "-q", "--no-trunc")
c.Assert(strings.TrimSpace(out), checker.Contains, id2)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id3)
// With config.json label=foobar and CLI label!=foobar, CLI label!=foobar supersede
out, _ = dockerCmd(c, "--config", d, "container", "prune", "--force", "--filter", "label!=foobar")
c.Assert(strings.TrimSpace(out), checker.Contains, id2)
out, _ = dockerCmd(c, "ps", "-a", "-q", "--no-trunc")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id2)
}
func (s *DockerSuite) TestPruneVolumeLabel(c *check.C) {
out, _ := dockerCmd(c, "volume", "create", "--label", "foo")
id1 := strings.TrimSpace(out)
c.Assert(id1, checker.Not(checker.Equals), "")
out, _ = dockerCmd(c, "volume", "create", "--label", "bar")
id2 := strings.TrimSpace(out)
c.Assert(id2, checker.Not(checker.Equals), "")
out, _ = dockerCmd(c, "volume", "create")
id3 := strings.TrimSpace(out)
c.Assert(id3, checker.Not(checker.Equals), "")
out, _ = dockerCmd(c, "volume", "create", "--label", "foobar")
id4 := strings.TrimSpace(out)
c.Assert(id4, checker.Not(checker.Equals), "")
// Add a config file of label=foobar, that will have no impact if cli is label!=foobar
config := `{"pruneFilters": ["label=foobar"]}`
d, err := ioutil.TempDir("", "integration-cli-")
c.Assert(err, checker.IsNil)
defer os.RemoveAll(d)
err = ioutil.WriteFile(filepath.Join(d, "config.json"), []byte(config), 0644)
c.Assert(err, checker.IsNil)
// With config.json only, prune based on label=foobar
out, _ = dockerCmd(c, "--config", d, "volume", "prune", "--force")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id1)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id2)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id3)
c.Assert(strings.TrimSpace(out), checker.Contains, id4)
out, _ = dockerCmd(c, "volume", "prune", "--force", "--filter", "label=foo")
c.Assert(strings.TrimSpace(out), checker.Contains, id1)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id2)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id3)
out, _ = dockerCmd(c, "volume", "ls", "--format", "{{.Name}}")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id1)
c.Assert(strings.TrimSpace(out), checker.Contains, id2)
c.Assert(strings.TrimSpace(out), checker.Contains, id3)
out, _ = dockerCmd(c, "volume", "prune", "--force", "--filter", "label!=bar")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id2)
c.Assert(strings.TrimSpace(out), checker.Contains, id3)
out, _ = dockerCmd(c, "volume", "ls", "--format", "{{.Name}}")
c.Assert(strings.TrimSpace(out), checker.Contains, id2)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id3)
// With config.json label=foobar and CLI label!=foobar, CLI label!=foobar supersede
out, _ = dockerCmd(c, "--config", d, "volume", "prune", "--force", "--filter", "label!=foobar")
c.Assert(strings.TrimSpace(out), checker.Contains, id2)
out, _ = dockerCmd(c, "volume", "ls", "--format", "{{.Name}}")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id2)
}
func (s *DockerSuite) TestPruneNetworkLabel(c *check.C) {
dockerCmd(c, "network", "create", "--label", "foo", "n1")
dockerCmd(c, "network", "create", "--label", "bar", "n2")
dockerCmd(c, "network", "create", "n3")
out, _ := dockerCmd(c, "network", "prune", "--force", "--filter", "label=foo")
c.Assert(strings.TrimSpace(out), checker.Contains, "n1")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), "n2")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), "n3")
out, _ = dockerCmd(c, "network", "prune", "--force", "--filter", "label!=bar")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), "n1")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), "n2")
c.Assert(strings.TrimSpace(out), checker.Contains, "n3")
out, _ = dockerCmd(c, "network", "prune", "--force")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), "n1")
c.Assert(strings.TrimSpace(out), checker.Contains, "n2")
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), "n3")
}
func (s *DockerDaemonSuite) TestPruneImageLabel(c *check.C) {
s.d.StartWithBusybox(c)
out, _, err := s.d.BuildImageWithOut("test1",
`FROM busybox
LABEL foo=bar`, true, "-q")
c.Assert(err, checker.IsNil)
id1 := strings.TrimSpace(out)
out, err = s.d.Cmd("images", "-q", "--no-trunc")
c.Assert(err, checker.IsNil)
c.Assert(strings.TrimSpace(out), checker.Contains, id1)
out, _, err = s.d.BuildImageWithOut("test2",
`FROM busybox
LABEL bar=foo`, true, "-q")
c.Assert(err, checker.IsNil)
id2 := strings.TrimSpace(out)
out, err = s.d.Cmd("images", "-q", "--no-trunc")
c.Assert(err, checker.IsNil)
c.Assert(strings.TrimSpace(out), checker.Contains, id2)
out, err = s.d.Cmd("image", "prune", "--force", "--all", "--filter", "label=foo=bar")
c.Assert(err, checker.IsNil)
c.Assert(strings.TrimSpace(out), checker.Contains, id1)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id2)
out, err = s.d.Cmd("image", "prune", "--force", "--all", "--filter", "label!=bar=foo")
c.Assert(err, checker.IsNil)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id1)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id2)
out, err = s.d.Cmd("image", "prune", "--force", "--all", "--filter", "label=bar=foo")
c.Assert(err, checker.IsNil)
c.Assert(strings.TrimSpace(out), checker.Not(checker.Contains), id1)
c.Assert(strings.TrimSpace(out), checker.Contains, id2)
}