From 921da495d24695dda66d3f58e78887dd0bc2402e Mon Sep 17 00:00:00 2001 From: Raghavendra K T Date: Sun, 12 Jul 2015 13:16:33 +0530 Subject: [PATCH] Add the memory swappiness tuning option to docker. Memory swappiness option takes 0-100, and helps to tune swappiness behavior per container. For example, When a lower value of swappiness is chosen the container will see minimum major faults. When no value is specified for memory-swappiness in docker UI, it is inherited from parent cgroup. (generally 60 unless it is changed). Signed-off-by: Raghavendra K T --- daemon/container_unix.go | 21 ++--- daemon/execdriver/driver.go | 21 ++--- daemon/execdriver/driver_linux.go | 1 + daemon/execdriver/lxc/lxc_template.go | 3 + docs/reference/api/docker_remote_api_v1.20.md | 2 + docs/reference/commandline/create.md | 1 + docs/reference/commandline/run.md | 1 + docs/reference/run.md | 15 ++++ integration-cli/docker_cli_run_test.go | 21 +++++ man/docker-create.1.md | 4 + man/docker-run.1.md | 4 + pkg/sysinfo/sysinfo.go | 7 +- pkg/sysinfo/sysinfo_linux.go | 4 + runconfig/hostconfig.go | 69 ++++++++-------- runconfig/parse.go | 81 +++++++++++-------- 15 files changed, 165 insertions(+), 90 deletions(-) diff --git a/daemon/container_unix.go b/daemon/container_unix.go index 766e9b37c3..cc730e4c40 100644 --- a/daemon/container_unix.go +++ b/daemon/container_unix.go @@ -262,16 +262,17 @@ func populateCommand(c *Container, env []string) error { } resources := &execdriver.Resources{ - Memory: c.hostConfig.Memory, - MemorySwap: c.hostConfig.MemorySwap, - CpuShares: c.hostConfig.CpuShares, - CpusetCpus: c.hostConfig.CpusetCpus, - CpusetMems: c.hostConfig.CpusetMems, - CpuPeriod: c.hostConfig.CpuPeriod, - CpuQuota: c.hostConfig.CpuQuota, - BlkioWeight: c.hostConfig.BlkioWeight, - Rlimits: rlimits, - OomKillDisable: c.hostConfig.OomKillDisable, + Memory: c.hostConfig.Memory, + MemorySwap: c.hostConfig.MemorySwap, + CpuShares: c.hostConfig.CpuShares, + CpusetCpus: c.hostConfig.CpusetCpus, + CpusetMems: c.hostConfig.CpusetMems, + CpuPeriod: c.hostConfig.CpuPeriod, + CpuQuota: c.hostConfig.CpuQuota, + BlkioWeight: c.hostConfig.BlkioWeight, + Rlimits: rlimits, + OomKillDisable: c.hostConfig.OomKillDisable, + MemorySwappiness: c.hostConfig.MemorySwappiness, } processConfig := execdriver.ProcessConfig{ diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index 43948ff379..c470ad408e 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -107,16 +107,17 @@ type NetworkInterface struct { // TODO Windows: Factor out ulimit.Rlimit type Resources struct { - Memory int64 `json:"memory"` - MemorySwap int64 `json:"memory_swap"` - CpuShares int64 `json:"cpu_shares"` - CpusetCpus string `json:"cpuset_cpus"` - CpusetMems string `json:"cpuset_mems"` - CpuPeriod int64 `json:"cpu_period"` - CpuQuota int64 `json:"cpu_quota"` - BlkioWeight int64 `json:"blkio_weight"` - Rlimits []*ulimit.Rlimit `json:"rlimits"` - OomKillDisable bool `json:"oom_kill_disable"` + Memory int64 `json:"memory"` + MemorySwap int64 `json:"memory_swap"` + CpuShares int64 `json:"cpu_shares"` + CpusetCpus string `json:"cpuset_cpus"` + CpusetMems string `json:"cpuset_mems"` + CpuPeriod int64 `json:"cpu_period"` + CpuQuota int64 `json:"cpu_quota"` + BlkioWeight int64 `json:"blkio_weight"` + Rlimits []*ulimit.Rlimit `json:"rlimits"` + OomKillDisable bool `json:"oom_kill_disable"` + MemorySwappiness int64 `json:"memory_swappiness"` } type ResourceStats struct { diff --git a/daemon/execdriver/driver_linux.go b/daemon/execdriver/driver_linux.go index 5207e90485..ad7eb30225 100644 --- a/daemon/execdriver/driver_linux.go +++ b/daemon/execdriver/driver_linux.go @@ -58,6 +58,7 @@ func SetupCgroups(container *configs.Config, c *Command) error { container.Cgroups.CpuQuota = c.Resources.CpuQuota container.Cgroups.BlkioWeight = c.Resources.BlkioWeight container.Cgroups.OomKillDisable = c.Resources.OomKillDisable + container.Cgroups.MemorySwappiness = c.Resources.MemorySwappiness } return nil diff --git a/daemon/execdriver/lxc/lxc_template.go b/daemon/execdriver/lxc/lxc_template.go index 26d2074e11..b70178a251 100644 --- a/daemon/execdriver/lxc/lxc_template.go +++ b/daemon/execdriver/lxc/lxc_template.go @@ -115,6 +115,9 @@ lxc.cgroup.blkio.weight = {{.Resources.BlkioWeight}} {{if .Resources.OomKillDisable}} lxc.cgroup.memory.oom_control = {{.Resources.OomKillDisable}} {{end}} +{{if .Resources.MemorySwappiness}} +lxc.cgroup.memory.swappiness = {{.Resources.MemorySwappiness}} +{{end}} {{end}} {{if .LxcConfig}} diff --git a/docs/reference/api/docker_remote_api_v1.20.md b/docs/reference/api/docker_remote_api_v1.20.md index ad0c325e1d..cff023decb 100644 --- a/docs/reference/api/docker_remote_api_v1.20.md +++ b/docs/reference/api/docker_remote_api_v1.20.md @@ -160,6 +160,7 @@ Create a container "CpusetCpus": "0,1", "CpusetMems": "0,1", "BlkioWeight": 300, + "MemorySwappiness": 60, "OomKillDisable": false, "PortBindings": { "22/tcp": [{ "HostPort": "11022" }] }, "PublishAllPorts": false, @@ -208,6 +209,7 @@ Json Parameters: - **CpusetCpus** - String value containing the `cgroups CpusetCpus` to use. - **CpusetMems** - Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems. - **BlkioWeight** - Block IO weight (relative weight) accepts a weight value between 10 and 1000. +- **MemorySwappiness** - Tune a container's memory swappiness behavior. Accepts an integer between 0 and 100. - **OomKillDisable** - Boolean value, whether to disable OOM Killer for the container or not. - **AttachStdin** - Boolean value, attaches to `stdin`. - **AttachStdout** - Boolean value, attaches to `stdout`. diff --git a/docs/reference/commandline/create.md b/docs/reference/commandline/create.md index 3fa795abfc..2e0e591a97 100644 --- a/docs/reference/commandline/create.md +++ b/docs/reference/commandline/create.md @@ -57,6 +57,7 @@ Creates a new container. --privileged=false Give extended privileges to this container --read-only=false Mount the container's root filesystem as read only --restart="no" Restart policy (no, on-failure[:max-retry], always) + --memory-swappiness="" Tune a container's memory swappiness behavior. Accepts an integer between 0 and 100. --security-opt=[] Security options -t, --tty=false Allocate a pseudo-TTY -u, --user="" Username or UID diff --git a/docs/reference/commandline/run.md b/docs/reference/commandline/run.md index 5e700845a3..147bf70495 100644 --- a/docs/reference/commandline/run.md +++ b/docs/reference/commandline/run.md @@ -58,6 +58,7 @@ weight=1 --read-only=false Mount the container's root filesystem as read only --restart="no" Restart policy (no, on-failure[:max-retry], always) --rm=false Automatically remove the container when it exits + --memory-swappiness="" Tune a container's memory swappiness behavior. Accepts an integer between 0 and 100. --security-opt=[] Security Options --sig-proxy=true Proxy received signals to the process -t, --tty=false Allocate a pseudo-TTY diff --git a/docs/reference/run.md b/docs/reference/run.md index 2152947c1a..48662ccdd1 100644 --- a/docs/reference/run.md +++ b/docs/reference/run.md @@ -514,6 +514,7 @@ container: --cpu-quota=0: Limit the CPU CFS (Completely Fair Scheduler) quota --blkio-weight=0: Block IO weight (relative weight) accepts a weight value between 10 and 1000. --oom-kill-disable=true|false: Whether to disable OOM Killer for the container or not. + --memory-swappiness="": Tune a container's memory swappiness behavior. Accepts an integer between 0 and 100. ### Memory constraints @@ -611,6 +612,20 @@ The following example, illustrates a dangerous way to use the flag: The container has unlimited memory which can cause the host to run out memory and require killing system processes to free memory. +### Swappiness constraint + +By default, a container's kernel can swap out a percentage of anonymous pages. +To set this percentage for a container, specify a `--memory-swappiness` value +between 0 and 100. A value of 0 turns off anonymous page swapping. A value of +100 sets all anonymous pages as swappable. + +For example, you can set: + + $ docker run -ti --memory-swappiness=0 ubuntu:14.04 /bin/bash + +Setting the `--memory-swappiness` option is helpful when you want to retain the +container's working set and to avoid swapping performance penalties. + ### CPU share constraint By default, all containers get the same proportion of CPU cycles. This proportion diff --git a/integration-cli/docker_cli_run_test.go b/integration-cli/docker_cli_run_test.go index a6c6d1d2ba..d2377efb4a 100644 --- a/integration-cli/docker_cli_run_test.go +++ b/integration-cli/docker_cli_run_test.go @@ -61,6 +61,27 @@ func (s *DockerSuite) TestRunWithoutMemoryswapLimit(c *check.C) { } } +func (s *DockerSuite) TestRunWithSwappiness(c *check.C) { + runCmd := exec.Command(dockerBinary, "run", "--memory-swappiness", "0", "busybox", "true") + out, _, err := runCommandWithOutput(runCmd) + if err != nil { + c.Fatalf("failed to run container, output: %q", out) + } +} + +func (s *DockerSuite) TestRunWithSwappinessInvalid(c *check.C) { + runCmd := exec.Command(dockerBinary, "run", "--memory-swappiness", "101", "busybox", "true") + out, _, err := runCommandWithOutput(runCmd) + if err == nil { + c.Fatalf("failed. test was able to set invalid value, output: %q", out) + } + runCmd = exec.Command(dockerBinary, "run", "--memory-swappiness", "-1", "busybox", "true") + out, _, err = runCommandWithOutput(runCmd) + if err == nil { + c.Fatalf("failed. test was able to set invalid value, output: %q", out) + } +} + // "test" should be printed func (s *DockerSuite) TestRunEchoStdoutWitCPULimit(c *check.C) { runCmd := exec.Command(dockerBinary, "run", "-c", "1000", "busybox", "echo", "test") diff --git a/man/docker-create.1.md b/man/docker-create.1.md index a41ca04bc1..0bde6271e8 100644 --- a/man/docker-create.1.md +++ b/man/docker-create.1.md @@ -48,6 +48,7 @@ docker-create - Create a new container [**--read-only**[=*false*]] [**--restart**[=*RESTART*]] [**--security-opt**[=*[]*]] +[**--memory-swappiness**[=*MEMORY-SWAPPINESS*]] [**-t**|**--tty**[=*false*]] [**-u**|**--user**[=*USER*]] [**-v**|**--volume**[=*[]*]] @@ -225,6 +226,9 @@ This value should always larger than **-m**, so you should always use this with **--security-opt**=[] Security Options +**--memory-swappiness**="" + Tune a container's memory swappiness behavior. Accepts an integer between 0 and 100. + **-t**, **--tty**=*true*|*false* Allocate a pseudo-TTY. The default is *false*. diff --git a/man/docker-run.1.md b/man/docker-run.1.md index c81f3f8e3e..cdecee8461 100644 --- a/man/docker-run.1.md +++ b/man/docker-run.1.md @@ -51,6 +51,7 @@ docker-run - Run a command in a new container [**--rm**[=*false*]] [**--security-opt**[=*[]*]] [**--sig-proxy**[=*true*]] +[**--memory-swappiness**[=*MEMORY-SWAPPINESS*]] [**-t**|**--tty**[=*false*]] [**-u**|**--user**[=*USER*]] [**-v**|**--volume**[=*[]*]] @@ -371,6 +372,9 @@ its root filesystem mounted as read only prohibiting any writes. **--sig-proxy**=*true*|*false* Proxy received signals to the process (non-TTY mode only). SIGCHLD, SIGSTOP, and SIGKILL are not proxied. The default is *true*. +**--memory-swappiness**="" + Tune a container's memory swappiness behavior. Accepts an integer between 0 and 100. + **-t**, **--tty**=*true*|*false* Allocate a pseudo-TTY. The default is *false*. diff --git a/pkg/sysinfo/sysinfo.go b/pkg/sysinfo/sysinfo.go index 47b84cef6e..8f905aa089 100644 --- a/pkg/sysinfo/sysinfo.go +++ b/pkg/sysinfo/sysinfo.go @@ -13,9 +13,10 @@ type SysInfo struct { } type cgroupMemInfo struct { - MemoryLimit bool - SwapLimit bool - OomKillDisable bool + MemoryLimit bool + SwapLimit bool + OomKillDisable bool + MemorySwappiness bool } type cgroupCpuInfo struct { diff --git a/pkg/sysinfo/sysinfo_linux.go b/pkg/sysinfo/sysinfo_linux.go index 75700d9087..e9d6b3d0a0 100644 --- a/pkg/sysinfo/sysinfo_linux.go +++ b/pkg/sysinfo/sysinfo_linux.go @@ -50,6 +50,10 @@ func checkCgroupMem(quiet bool) *cgroupMemInfo { if !quiet && !info.OomKillDisable { logrus.Warnf("Your kernel does not support oom control.") } + info.MemorySwappiness = cgroupEnabled(mountPoint, "memory.swappiness") + if !quiet && !info.MemorySwappiness { + logrus.Warnf("Your kernel does not support memory swappiness.") + } return info } diff --git a/runconfig/hostconfig.go b/runconfig/hostconfig.go index d851f8ae66..9264d9ae9b 100644 --- a/runconfig/hostconfig.go +++ b/runconfig/hostconfig.go @@ -221,40 +221,41 @@ func NewCapList(caps []string) *CapList { } type HostConfig struct { - Binds []string - ContainerIDFile string - LxcConf *LxcConfig - Memory int64 // Memory limit (in bytes) - MemorySwap int64 // Total memory usage (memory + swap); set `-1` to disable swap - CpuShares int64 // CPU shares (relative weight vs. other containers) - CpuPeriod int64 - CpusetCpus string // CpusetCpus 0-2, 0,1 - CpusetMems string // CpusetMems 0-2, 0,1 - CpuQuota int64 - BlkioWeight int64 // Block IO weight (relative weight vs. other containers) - OomKillDisable bool // Whether to disable OOM Killer or not - Privileged bool - PortBindings nat.PortMap - Links []string - PublishAllPorts bool - Dns []string - DnsSearch []string - ExtraHosts []string - VolumesFrom []string - Devices []DeviceMapping - NetworkMode NetworkMode - IpcMode IpcMode - PidMode PidMode - UTSMode UTSMode - CapAdd *CapList - CapDrop *CapList - RestartPolicy RestartPolicy - SecurityOpt []string - ReadonlyRootfs bool - Ulimits []*ulimit.Ulimit - LogConfig LogConfig - CgroupParent string // Parent cgroup. - ConsoleSize [2]int // Initial console size on Windows + Binds []string + ContainerIDFile string + LxcConf *LxcConfig + Memory int64 // Memory limit (in bytes) + MemorySwap int64 // Total memory usage (memory + swap); set `-1` to disable swap + CpuShares int64 // CPU shares (relative weight vs. other containers) + CpuPeriod int64 + CpusetCpus string // CpusetCpus 0-2, 0,1 + CpusetMems string // CpusetMems 0-2, 0,1 + CpuQuota int64 + BlkioWeight int64 // Block IO weight (relative weight vs. other containers) + OomKillDisable bool // Whether to disable OOM Killer or not + MemorySwappiness int64 // Tuning container memory swappiness behaviour + Privileged bool + PortBindings nat.PortMap + Links []string + PublishAllPorts bool + Dns []string + DnsSearch []string + ExtraHosts []string + VolumesFrom []string + Devices []DeviceMapping + NetworkMode NetworkMode + IpcMode IpcMode + PidMode PidMode + UTSMode UTSMode + CapAdd *CapList + CapDrop *CapList + RestartPolicy RestartPolicy + SecurityOpt []string + ReadonlyRootfs bool + Ulimits []*ulimit.Ulimit + LogConfig LogConfig + CgroupParent string // Parent cgroup. + ConsoleSize [2]int // Initial console size on Windows } func MergeConfigs(config *Config, hostConfig *HostConfig) *ContainerConfigWrapper { diff --git a/runconfig/parse.go b/runconfig/parse.go index e50018bbe6..ed6917acb3 100644 --- a/runconfig/parse.go +++ b/runconfig/parse.go @@ -72,6 +72,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*Config, *HostConfig, *flag.FlagSe flStdin = cmd.Bool([]string{"i", "-interactive"}, false, "Keep STDIN open even if not attached") flTty = cmd.Bool([]string{"t", "-tty"}, false, "Allocate a pseudo-TTY") flOomKillDisable = cmd.Bool([]string{"-oom-kill-disable"}, false, "Disable OOM Killer") + flSwappinessStr = cmd.String([]string{"-memory-swappiness"}, "", "Tuning container memory swappiness (0 to 100)") flContainerIDFile = cmd.String([]string{"#cidfile", "-cidfile"}, "", "Write the container ID to the file") flEntrypoint = cmd.String([]string{"#entrypoint", "-entrypoint"}, "", "Overwrite the default ENTRYPOINT of the image") flHostname = cmd.String([]string{"h", "-hostname"}, "", "Container host name") @@ -187,6 +188,19 @@ func Parse(cmd *flag.FlagSet, args []string) (*Config, *HostConfig, *flag.FlagSe } } + var parsedSwappiness int64 + var flSwappiness int64 + + if *flSwappinessStr != "" { + parsedSwappiness, err = strconv.ParseInt(*flSwappinessStr, 10, 64) + if err != nil || parsedSwappiness < 0 || parsedSwappiness > 100 { + return nil, nil, cmd, fmt.Errorf("invalid value:%s. valid memory swappiness range is 0-100", *flSwappinessStr) + } + flSwappiness = parsedSwappiness + } else { + flSwappiness = -1 + } + var binds []string // add any bind targets to the list of container volumes for bind := range flVolumes.GetMap() { @@ -327,39 +341,40 @@ func Parse(cmd *flag.FlagSet, args []string) (*Config, *HostConfig, *flag.FlagSe } hostConfig := &HostConfig{ - Binds: binds, - ContainerIDFile: *flContainerIDFile, - LxcConf: lxcConf, - Memory: flMemory, - MemorySwap: MemorySwap, - CpuShares: *flCpuShares, - CpuPeriod: *flCpuPeriod, - CpusetCpus: *flCpusetCpus, - CpusetMems: *flCpusetMems, - CpuQuota: *flCpuQuota, - BlkioWeight: *flBlkioWeight, - OomKillDisable: *flOomKillDisable, - Privileged: *flPrivileged, - PortBindings: portBindings, - Links: flLinks.GetAll(), - PublishAllPorts: *flPublishAll, - Dns: flDns.GetAll(), - DnsSearch: flDnsSearch.GetAll(), - ExtraHosts: flExtraHosts.GetAll(), - VolumesFrom: flVolumesFrom.GetAll(), - NetworkMode: netMode, - IpcMode: ipcMode, - PidMode: pidMode, - UTSMode: utsMode, - Devices: deviceMappings, - CapAdd: NewCapList(flCapAdd.GetAll()), - CapDrop: NewCapList(flCapDrop.GetAll()), - RestartPolicy: restartPolicy, - SecurityOpt: flSecurityOpt.GetAll(), - ReadonlyRootfs: *flReadonlyRootfs, - Ulimits: flUlimits.GetList(), - LogConfig: LogConfig{Type: *flLoggingDriver, Config: loggingOpts}, - CgroupParent: *flCgroupParent, + Binds: binds, + ContainerIDFile: *flContainerIDFile, + LxcConf: lxcConf, + Memory: flMemory, + MemorySwap: MemorySwap, + CpuShares: *flCpuShares, + CpuPeriod: *flCpuPeriod, + CpusetCpus: *flCpusetCpus, + CpusetMems: *flCpusetMems, + CpuQuota: *flCpuQuota, + BlkioWeight: *flBlkioWeight, + OomKillDisable: *flOomKillDisable, + MemorySwappiness: flSwappiness, + Privileged: *flPrivileged, + PortBindings: portBindings, + Links: flLinks.GetAll(), + PublishAllPorts: *flPublishAll, + Dns: flDns.GetAll(), + DnsSearch: flDnsSearch.GetAll(), + ExtraHosts: flExtraHosts.GetAll(), + VolumesFrom: flVolumesFrom.GetAll(), + NetworkMode: netMode, + IpcMode: ipcMode, + PidMode: pidMode, + UTSMode: utsMode, + Devices: deviceMappings, + CapAdd: NewCapList(flCapAdd.GetAll()), + CapDrop: NewCapList(flCapDrop.GetAll()), + RestartPolicy: restartPolicy, + SecurityOpt: flSecurityOpt.GetAll(), + ReadonlyRootfs: *flReadonlyRootfs, + Ulimits: flUlimits.GetList(), + LogConfig: LogConfig{Type: *flLoggingDriver, Config: loggingOpts}, + CgroupParent: *flCgroupParent, } applyExperimentalFlags(expFlags, config, hostConfig)