From 846baf1fd3efcbfbf9d3eb99e436ca9a59d3e185 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Tue, 1 Nov 2016 10:12:29 -0700 Subject: [PATCH] Add `--cpus` flag to control cpu resources This fix tries to address the proposal raised in 27921 and add `--cpus` flag for `docker run/create`. Basically, `--cpus` will allow user to specify a number (possibly partial) about how many CPUs the container will use. For example, on a 2-CPU system `--cpus 1.5` means the container will take 75% (1.5/2) of the CPU share. This fix adds a `NanoCPUs` field to `HostConfig` since swarmkit alreay have a concept of NanoCPUs for tasks. The `--cpus` flag will translate the number into reused `NanoCPUs` to be consistent. This fix adds integration tests to cover the changes. Related docs (`docker run` and Remote APIs) have been updated. This fix fixes 27921. Signed-off-by: Yong Tang --- api/types/container/host_config.go | 1 + cli/command/service/opts.go | 32 ++---------------- cli/command/service/opts_test.go | 5 +-- daemon/daemon_unix.go | 24 ++++++++++++++ daemon/daemon_windows.go | 11 +++++++ daemon/oci_windows.go | 4 +++ docs/reference/api/docker_remote_api.md | 1 + docs/reference/api/docker_remote_api_v1.25.md | 2 ++ docs/reference/commandline/create.md | 1 + docs/reference/commandline/run.md | 1 + docs/reference/run.md | 8 +++++ integration-cli/docker_cli_run_unix_test.go | 20 +++++++++++ man/docker-create.1.md | 4 +++ man/docker-run.1.md | 4 +++ opts/opts.go | 33 +++++++++++++++++++ runconfig/opts/parse.go | 3 ++ 16 files changed, 122 insertions(+), 32 deletions(-) diff --git a/api/types/container/host_config.go b/api/types/container/host_config.go index 6cc1dc7591..0c82d625e8 100644 --- a/api/types/container/host_config.go +++ b/api/types/container/host_config.go @@ -234,6 +234,7 @@ type Resources struct { // Applicable to all platforms CPUShares int64 `json:"CpuShares"` // CPU shares (relative weight vs. other containers) Memory int64 // Memory limit (in bytes) + NanoCPUs int64 `json:"NanoCpus"` // CPU quota in units of 10-9 CPUs. // Applicable to UNIX platforms CgroupParent string // Parent cgroup. diff --git a/cli/command/service/opts.go b/cli/command/service/opts.go index c89d40a767..2199e9f363 100644 --- a/cli/command/service/opts.go +++ b/cli/command/service/opts.go @@ -2,7 +2,6 @@ package service import ( "fmt" - "math/big" "strconv" "strings" "time" @@ -40,33 +39,6 @@ func (m *memBytes) Value() int64 { return int64(*m) } -type nanoCPUs int64 - -func (c *nanoCPUs) String() string { - return big.NewRat(c.Value(), 1e9).FloatString(3) -} - -func (c *nanoCPUs) Set(value string) error { - cpu, ok := new(big.Rat).SetString(value) - if !ok { - return fmt.Errorf("Failed to parse %v as a rational number", value) - } - nano := cpu.Mul(cpu, big.NewRat(1e9, 1)) - if !nano.IsInt() { - return fmt.Errorf("value is too precise") - } - *c = nanoCPUs(nano.Num().Int64()) - return nil -} - -func (c *nanoCPUs) Type() string { - return "NanoCPUs" -} - -func (c *nanoCPUs) Value() int64 { - return int64(*c) -} - // PositiveDurationOpt is an option type for time.Duration that uses a pointer. // It bahave similarly to DurationOpt but only allows positive duration values. type PositiveDurationOpt struct { @@ -156,9 +128,9 @@ type updateOptions struct { } type resourceOptions struct { - limitCPU nanoCPUs + limitCPU opts.NanoCPUs limitMemBytes memBytes - resCPU nanoCPUs + resCPU opts.NanoCPUs resMemBytes memBytes } diff --git a/cli/command/service/opts_test.go b/cli/command/service/opts_test.go index 26534cf0f5..aa2d999dcf 100644 --- a/cli/command/service/opts_test.go +++ b/cli/command/service/opts_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/opts" "github.com/docker/docker/pkg/testutil/assert" ) @@ -21,12 +22,12 @@ func TestMemBytesSetAndValue(t *testing.T) { } func TestNanoCPUsString(t *testing.T) { - var cpus nanoCPUs = 6100000000 + var cpus opts.NanoCPUs = 6100000000 assert.Equal(t, cpus.String(), "6.100") } func TestNanoCPUsSetAndValue(t *testing.T) { - var cpus nanoCPUs + var cpus opts.NanoCPUs assert.NilError(t, cpus.Set("0.35")) assert.Equal(t, cpus.Value(), int64(350000000)) } diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index 5d106714d2..b8b994de57 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -15,6 +15,7 @@ import ( "strconv" "strings" "syscall" + "time" "github.com/Sirupsen/logrus" "github.com/docker/docker/api/types" @@ -110,6 +111,16 @@ func getCPUResources(config containertypes.Resources) *specs.CPU { cpu.Mems = &cpuset } + if config.NanoCPUs > 0 { + // Use the default setting of 100ms, as is specified in: + // https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt + // cpu.cfs_period_us=100ms + period := uint64(100 * time.Millisecond / time.Microsecond) + quota := uint64(config.NanoCPUs) * period / 1e9 + cpu.Period = &period + cpu.Quota = "a + } + if config.CPUPeriod != 0 { period := uint64(config.CPUPeriod) cpu.Period = &period @@ -341,6 +352,19 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi } // cpu subsystem checks and adjustments + if resources.NanoCPUs > 0 && resources.CPUPeriod > 0 { + return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Period cannot both be set") + } + if resources.NanoCPUs > 0 && resources.CPUQuota > 0 { + return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Quota cannot both be set") + } + if resources.NanoCPUs > 0 && (!sysInfo.CPUCfsPeriod || !sysInfo.CPUCfsQuota) { + return warnings, fmt.Errorf("NanoCPUs can not be set, as your kernel does not support CPU cfs period/quota or the cgroup is not mounted") + } + if resources.NanoCPUs < 0 || resources.NanoCPUs > int64(sysinfo.NumCPU())*1e9 { + return warnings, fmt.Errorf("Range of Nano CPUs is from 1 to %d", int64(sysinfo.NumCPU())*1e9) + } + if resources.CPUShares > 0 && !sysInfo.CPUShares { warnings = append(warnings, "Your kernel does not support CPU shares or the cgroup is not mounted. Shares discarded.") logrus.Warn("Your kernel does not support CPU shares or the cgroup is not mounted. Shares discarded.") diff --git a/daemon/daemon_windows.go b/daemon/daemon_windows.go index 0402c8eb93..dd327ad6c7 100644 --- a/daemon/daemon_windows.go +++ b/daemon/daemon_windows.go @@ -103,6 +103,17 @@ func verifyContainerResources(resources *containertypes.Resources, sysInfo *sysi return warnings, fmt.Errorf("Conflicting options: CPU Shares and CPU Percent cannot both be set") } + if resources.NanoCPUs > 0 && resources.CPUPercent > 0 { + return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Percent cannot both be set") + } + + if resources.NanoCPUs > 0 && resources.CPUShares > 0 { + return warnings, fmt.Errorf("Conflicting options: Nano CPUs and CPU Shares cannot both be set") + } + if resources.NanoCPUs < 0 || resources.NanoCPUs > int64(sysinfo.NumCPU())*1e9 { + return warnings, fmt.Errorf("Range of Nano CPUs is from 1 to %d", int64(sysinfo.NumCPU())*1e9) + } + // TODO Windows: Add more validation of resource settings not supported on Windows if resources.BlkioWeight > 0 { diff --git a/daemon/oci_windows.go b/daemon/oci_windows.go index a3e6431db0..602b9bf85a 100644 --- a/daemon/oci_windows.go +++ b/daemon/oci_windows.go @@ -6,6 +6,7 @@ import ( containertypes "github.com/docker/docker/api/types/container" "github.com/docker/docker/container" "github.com/docker/docker/oci" + "github.com/docker/docker/pkg/sysinfo" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -82,6 +83,9 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) { // @darrenstahlmsft implement these resources cpuShares := uint16(c.HostConfig.CPUShares) cpuPercent := uint8(c.HostConfig.CPUPercent) + if c.HostConfig.NanoCPUs > 0 { + cpuPercent = uint8(c.HostConfig.NanoCPUs * 100 / int64(sysinfo.NumCPU()) / 1e9) + } memoryLimit := uint64(c.HostConfig.Memory) s.Windows.Resources = &specs.WindowsResources{ CPU: &specs.WindowsCPUResources{ diff --git a/docs/reference/api/docker_remote_api.md b/docs/reference/api/docker_remote_api.md index 447ca0e43a..51814a5f30 100644 --- a/docs/reference/api/docker_remote_api.md +++ b/docs/reference/api/docker_remote_api.md @@ -164,6 +164,7 @@ This section lists each version from latest to oldest. Each listing includes a * The `hostConfig` option now accepts the fields `CpuRealtimePeriod` and `CpuRtRuntime` to allocate cpu runtime to rt tasks when `CONFIG_RT_GROUP_SCHED` is enabled in the kernel. * The `SecurityOptions` field within the `GET /info` response now includes `userns` if user namespaces are enabled in the daemon. * `GET /nodes` and `GET /node/(id or name)` now return `Addr` as part of a node's `Status`, which is the address that that node connects to the manager from. +* The `HostConfig` field now includes `NanoCPUs` that represents CPU quota in units of 10-9 CPUs. ### v1.24 API changes diff --git a/docs/reference/api/docker_remote_api_v1.25.md b/docs/reference/api/docker_remote_api_v1.25.md index b802ecc243..78f3000b68 100644 --- a/docs/reference/api/docker_remote_api_v1.25.md +++ b/docs/reference/api/docker_remote_api_v1.25.md @@ -302,6 +302,7 @@ Create a container "MemorySwap": 0, "MemoryReservation": 0, "KernelMemory": 0, + "NanoCPUs": 500000, "CpuPercent": 80, "CpuShares": 512, "CpuPeriod": 100000, @@ -425,6 +426,7 @@ Create a container You must use this with `memory` and make the swap value larger than `memory`. - **MemoryReservation** - Memory soft limit in bytes. - **KernelMemory** - Kernel memory limit in bytes. + - **NanoCPUs** - CPU quota in units of 10-9 CPUs. - **CpuPercent** - An integer value containing the usable percentage of the available CPUs. (Windows daemon only) - **CpuShares** - An integer value containing the container's CPU Shares (ie. the relative weight vs other containers). diff --git a/docs/reference/commandline/create.md b/docs/reference/commandline/create.md index 5e3c04efa5..23c8797686 100644 --- a/docs/reference/commandline/create.md +++ b/docs/reference/commandline/create.md @@ -35,6 +35,7 @@ Options: --cpu-period int Limit CPU CFS (Completely Fair Scheduler) period --cpu-quota int Limit CPU CFS (Completely Fair Scheduler) quota -c, --cpu-shares int CPU shares (relative weight) + --cpus NanoCPUs Number of CPUs (default 0.000) --cpu-rt-period int Limit the CPU real-time period in microseconds --cpu-rt-runtime int Limit the CPU real-time runtime in microseconds --cpuset-cpus string CPUs in which to allow execution (0-3, 0,1) diff --git a/docs/reference/commandline/run.md b/docs/reference/commandline/run.md index 0c4a2688d8..82745a2efe 100644 --- a/docs/reference/commandline/run.md +++ b/docs/reference/commandline/run.md @@ -33,6 +33,7 @@ Options: --cpu-period int Limit CPU CFS (Completely Fair Scheduler) period --cpu-quota int Limit CPU CFS (Completely Fair Scheduler) quota -c, --cpu-shares int CPU shares (relative weight) + --cpus NanoCPUs Number of CPUs (default 0.000) --cpu-rt-period int Limit the CPU real-time period in microseconds --cpu-rt-runtime int Limit the CPU real-time runtime in microseconds --cpuset-cpus string CPUs in which to allow execution (0-3, 0,1) diff --git a/docs/reference/run.md b/docs/reference/run.md index f513f82ca0..9bb6cfc11e 100644 --- a/docs/reference/run.md +++ b/docs/reference/run.md @@ -686,6 +686,7 @@ container: | `--memory-reservation=""` | Memory soft limit (format: `[]`). Number is a positive integer. Unit can be one of `b`, `k`, `m`, or `g`. | | `--kernel-memory=""` | Kernel memory limit (format: `[]`). Number is a positive integer. Unit can be one of `b`, `k`, `m`, or `g`. Minimum is 4M. | | `-c`, `--cpu-shares=0` | CPU shares (relative weight) | +| `--cpus=0.000` | Number of CPUs. Number is a fractional number. 0.000 means no limit. | | `--cpu-period=0` | Limit the CPU CFS (Completely Fair Scheduler) period | | `--cpuset-cpus=""` | CPUs in which to allow execution (0-3, 0,1) | | `--cpuset-mems=""` | Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems. | @@ -970,6 +971,13 @@ Examples: If there is 1 CPU, this means the container can get 50% CPU worth of run-time every 50ms. +In addition to use `--cpu-period` and `--cpu-quota` for setting CPU period constraints, +it is possible to specify `--cpus` with a float number to achieve the same purpose. +For example, if there is 1 CPU, then `--cpus=0.5` will achieve the same result as +setting `--cpu-period=50000` and `--cpu-quota=25000` (50% CPU). + +The default value for `--cpus` is `0.000`, which means there is no limit. + For more information, see the [CFS documentation on bandwidth limiting](https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt). ### Cpuset constraint diff --git a/integration-cli/docker_cli_run_unix_test.go b/integration-cli/docker_cli_run_unix_test.go index 29d0462de2..169d128412 100644 --- a/integration-cli/docker_cli_run_unix_test.go +++ b/integration-cli/docker_cli_run_unix_test.go @@ -1409,3 +1409,23 @@ func (s *DockerDaemonSuite) TestRunWithDaemonDefaultSeccompProfile(c *check.C) { c.Assert(err, check.NotNil) c.Assert(out, checker.Contains, "Operation not permitted") } + +func (s *DockerSuite) TestRunWithNanoCPUs(c *check.C) { + testRequires(c, cpuCfsQuota, cpuCfsPeriod) + + file1 := "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" + file2 := "/sys/fs/cgroup/cpu/cpu.cfs_period_us" + out, _ := dockerCmd(c, "run", "--cpus", "0.5", "--name", "test", "busybox", "sh", "-c", fmt.Sprintf("cat %s && cat %s", file1, file2)) + c.Assert(strings.TrimSpace(out), checker.Equals, "50000\n100000") + + out = inspectField(c, "test", "HostConfig.NanoCpus") + c.Assert(out, checker.Equals, "5e+08", check.Commentf("setting the Nano CPUs failed")) + out = inspectField(c, "test", "HostConfig.CpuQuota") + c.Assert(out, checker.Equals, "0", check.Commentf("CPU CFS quota should be 0")) + out = inspectField(c, "test", "HostConfig.CpuPeriod") + c.Assert(out, checker.Equals, "0", check.Commentf("CPU CFS period should be 0")) + + out, _, err := dockerCmdWithError("run", "--cpus", "0.5", "--cpu-quota", "50000", "--cpu-period", "100000", "busybox", "sh") + c.Assert(err, check.NotNil) + c.Assert(out, checker.Contains, "Conflicting options: Nano CPUs and CPU Period cannot both be set") +} diff --git a/man/docker-create.1.md b/man/docker-create.1.md index fd19ee45c6..ff378397ed 100644 --- a/man/docker-create.1.md +++ b/man/docker-create.1.md @@ -19,6 +19,7 @@ docker-create - Create a new container [**--cpu-quota**[=*0*]] [**--cpu-rt-period**[=*0*]] [**--cpu-rt-runtime**[=*0*]] +[**--cpus**[=*0.0*]] [**--cpuset-cpus**[=*CPUSET-CPUS*]] [**--cpuset-mems**[=*CPUSET-MEMS*]] [**--device**[=*[]*]] @@ -154,6 +155,9 @@ two memory nodes. The sum of all runtimes across containers cannot exceed the amount allotted to the parent cgroup. +**--cpus**=0.0 + Number of CPUs. The default is *0.0*. + **--device**=[] Add a host device to the container (e.g. --device=/dev/sdc:/dev/xvdc:rwm) diff --git a/man/docker-run.1.md b/man/docker-run.1.md index d430ff58a5..5af64627be 100644 --- a/man/docker-run.1.md +++ b/man/docker-run.1.md @@ -19,6 +19,7 @@ docker-run - Run a command in a new container [**--cpu-quota**[=*0*]] [**--cpu-rt-period**[=*0*]] [**--cpu-rt-runtime**[=*0*]] +[**--cpus**[=*0.0*]] [**--cpuset-cpus**[=*CPUSET-CPUS*]] [**--cpuset-mems**[=*CPUSET-MEMS*]] [**-d**|**--detach**] @@ -208,6 +209,9 @@ to the quota you specify. The sum of all runtimes across containers cannot exceed the amount allotted to the parent cgroup. +**--cpus**=0.0 + Number of CPUs. The default is *0.0* which means no limit. + **-d**, **--detach**=*true*|*false* Detached mode: run the container in the background and print the new container ID. The default is *false*. diff --git a/opts/opts.go b/opts/opts.go index f8bb3ba745..e452943ffc 100644 --- a/opts/opts.go +++ b/opts/opts.go @@ -2,6 +2,7 @@ package opts import ( "fmt" + "math/big" "net" "regexp" "strings" @@ -319,3 +320,35 @@ func (o *FilterOpt) Type() string { func (o *FilterOpt) Value() filters.Args { return o.filter } + +// NanoCPUs is a type for fixed point fractional number. +type NanoCPUs int64 + +// String returns the string format of the number +func (c *NanoCPUs) String() string { + return big.NewRat(c.Value(), 1e9).FloatString(3) +} + +// Set sets the value of the NanoCPU by passing a string +func (c *NanoCPUs) Set(value string) error { + cpu, ok := new(big.Rat).SetString(value) + if !ok { + return fmt.Errorf("Failed to parse %v as a rational number", value) + } + nano := cpu.Mul(cpu, big.NewRat(1e9, 1)) + if !nano.IsInt() { + return fmt.Errorf("value is too precise") + } + *c = NanoCPUs(nano.Num().Int64()) + return nil +} + +// Type returns the type +func (c *NanoCPUs) Type() string { + return "NanoCPUs" +} + +// Value returns the value in int64 +func (c *NanoCPUs) Value() int64 { + return int64(*c) +} diff --git a/runconfig/opts/parse.go b/runconfig/opts/parse.go index a2323bff41..18e43cdfaf 100644 --- a/runconfig/opts/parse.go +++ b/runconfig/opts/parse.go @@ -79,6 +79,7 @@ type ContainerOptions struct { cpuRealtimePeriod int64 cpuRealtimeRuntime int64 cpuQuota int64 + cpus opts.NanoCPUs cpusetCpus string cpusetMems string blkioWeight uint16 @@ -232,6 +233,7 @@ func AddFlags(flags *pflag.FlagSet) *ContainerOptions { flags.Int64Var(&copts.cpuRealtimePeriod, "cpu-rt-period", 0, "Limit CPU real-time period in microseconds") flags.Int64Var(&copts.cpuRealtimeRuntime, "cpu-rt-runtime", 0, "Limit CPU real-time runtime in microseconds") flags.Int64VarP(&copts.cpuShares, "cpu-shares", "c", 0, "CPU shares (relative weight)") + flags.Var(&copts.cpus, "cpus", "Number of CPUs") flags.Var(&copts.deviceReadBps, "device-read-bps", "Limit read rate (bytes per second) from a device") flags.Var(&copts.deviceReadIOps, "device-read-iops", "Limit read rate (IO per second) from a device") flags.Var(&copts.deviceWriteBps, "device-write-bps", "Limit write rate (bytes per second) to a device") @@ -526,6 +528,7 @@ func Parse(flags *pflag.FlagSet, copts *ContainerOptions) (*container.Config, *c MemorySwappiness: &copts.swappiness, KernelMemory: kernelMemory, OomKillDisable: &copts.oomKillDisable, + NanoCPUs: copts.cpus.Value(), CPUPercent: copts.cpuPercent, CPUShares: copts.cpuShares, CPUPeriod: copts.cpuPeriod,