From 42fb2973c690fe4e4f65da235ce4dfa4c388e8a3 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 18 Apr 2014 21:14:58 -0700 Subject: [PATCH 01/14] Refactor cgroups file locations Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/cgroups.go | 70 +------------------------------ pkg/cgroups/utils.go | 73 +++++++++++++++++++++++++++++++++ pkg/libcontainer/nsinit/exec.go | 2 +- 3 files changed, 75 insertions(+), 70 deletions(-) create mode 100644 pkg/cgroups/utils.go diff --git a/pkg/cgroups/cgroups.go b/pkg/cgroups/cgroups.go index e5e8f82db6..343e70f215 100644 --- a/pkg/cgroups/cgroups.go +++ b/pkg/cgroups/cgroups.go @@ -1,14 +1,7 @@ package cgroups import ( - "bufio" "errors" - "github.com/dotcloud/docker/pkg/mount" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" ) var ( @@ -32,68 +25,7 @@ type ActiveCgroup interface { Cleanup() error } -// https://www.kernel.org/doc/Documentation/cgroups/cgroups.txt -func FindCgroupMountpoint(subsystem string) (string, error) { - mounts, err := mount.GetMounts() - if err != nil { - return "", err - } - - for _, mount := range mounts { - if mount.Fstype == "cgroup" { - for _, opt := range strings.Split(mount.VfsOpts, ",") { - if opt == subsystem { - return mount.Mountpoint, nil - } - } - } - } - return "", ErrNotFound -} - -// Returns the relative path to the cgroup docker is running in. -func GetThisCgroupDir(subsystem string) (string, error) { - f, err := os.Open("/proc/self/cgroup") - if err != nil { - return "", err - } - defer f.Close() - - return parseCgroupFile(subsystem, f) -} - -func GetInitCgroupDir(subsystem string) (string, error) { - f, err := os.Open("/proc/1/cgroup") - if err != nil { - return "", err - } - defer f.Close() - - return parseCgroupFile(subsystem, f) -} - -func parseCgroupFile(subsystem string, r io.Reader) (string, error) { - s := bufio.NewScanner(r) - for s.Scan() { - if err := s.Err(); err != nil { - return "", err - } - text := s.Text() - parts := strings.Split(text, ":") - for _, subs := range strings.Split(parts[1], ",") { - if subs == subsystem { - return parts[2], nil - } - } - } - return "", ErrNotFound -} - -func writeFile(dir, file, data string) error { - return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) -} - -func (c *Cgroup) Apply(pid int) (ActiveCgroup, error) { +func Apply(c *Cgroup, pid int) (ActiveCgroup, error) { // We have two implementation of cgroups support, one is based on // systemd and the dbus api, and one is based on raw cgroup fs operations // following the pre-single-writer model docs at: diff --git a/pkg/cgroups/utils.go b/pkg/cgroups/utils.go new file mode 100644 index 0000000000..82d5a955a3 --- /dev/null +++ b/pkg/cgroups/utils.go @@ -0,0 +1,73 @@ +package cgroups + +import ( + "bufio" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/dotcloud/docker/pkg/mount" +) + +// https://www.kernel.org/doc/Documentation/cgroups/cgroups.txt +func FindCgroupMountpoint(subsystem string) (string, error) { + mounts, err := mount.GetMounts() + if err != nil { + return "", err + } + + for _, mount := range mounts { + if mount.Fstype == "cgroup" { + for _, opt := range strings.Split(mount.VfsOpts, ",") { + if opt == subsystem { + return mount.Mountpoint, nil + } + } + } + } + return "", ErrNotFound +} + +// Returns the relative path to the cgroup docker is running in. +func GetThisCgroupDir(subsystem string) (string, error) { + f, err := os.Open("/proc/self/cgroup") + if err != nil { + return "", err + } + defer f.Close() + + return parseCgroupFile(subsystem, f) +} + +func GetInitCgroupDir(subsystem string) (string, error) { + f, err := os.Open("/proc/1/cgroup") + if err != nil { + return "", err + } + defer f.Close() + + return parseCgroupFile(subsystem, f) +} + +func parseCgroupFile(subsystem string, r io.Reader) (string, error) { + s := bufio.NewScanner(r) + for s.Scan() { + if err := s.Err(); err != nil { + return "", err + } + text := s.Text() + parts := strings.Split(text, ":") + for _, subs := range strings.Split(parts[1], ",") { + if subs == subsystem { + return parts[2], nil + } + } + } + return "", ErrNotFound +} + +func writeFile(dir, file, data string) error { + return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) +} diff --git a/pkg/libcontainer/nsinit/exec.go b/pkg/libcontainer/nsinit/exec.go index c07c45de3c..4e2fcefd47 100644 --- a/pkg/libcontainer/nsinit/exec.go +++ b/pkg/libcontainer/nsinit/exec.go @@ -99,7 +99,7 @@ func (ns *linuxNs) Exec(container *libcontainer.Container, term Terminal, args [ func (ns *linuxNs) SetupCgroups(container *libcontainer.Container, nspid int) (cgroups.ActiveCgroup, error) { if container.Cgroups != nil { - return container.Cgroups.Apply(nspid) + return cgroups.Apply(container.Cgroups, nspid) } return nil, nil } From ec43ec50b44cff3f043c78cad97466c68e2ba8cd Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 18 Apr 2014 21:30:08 -0700 Subject: [PATCH 02/14] Move systemd code into pkg Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/apply_nosystemd.go | 15 -------- pkg/cgroups/cgroups.go | 13 ------- pkg/cgroups/systemd/apply_nosystemd.go | 16 ++++++++ pkg/cgroups/{ => systemd}/apply_systemd.go | 45 ++++++++++++---------- pkg/libcontainer/nsinit/exec.go | 7 +++- 5 files changed, 46 insertions(+), 50 deletions(-) delete mode 100644 pkg/cgroups/apply_nosystemd.go create mode 100644 pkg/cgroups/systemd/apply_nosystemd.go rename pkg/cgroups/{ => systemd}/apply_systemd.go (86%) diff --git a/pkg/cgroups/apply_nosystemd.go b/pkg/cgroups/apply_nosystemd.go deleted file mode 100644 index f94d475907..0000000000 --- a/pkg/cgroups/apply_nosystemd.go +++ /dev/null @@ -1,15 +0,0 @@ -// +build !linux - -package cgroups - -import ( - "fmt" -) - -func useSystemd() bool { - return false -} - -func systemdApply(c *Cgroup, pid int) (ActiveCgroup, error) { - return nil, fmt.Errorf("Systemd not supported") -} diff --git a/pkg/cgroups/cgroups.go b/pkg/cgroups/cgroups.go index 343e70f215..3aac971340 100644 --- a/pkg/cgroups/cgroups.go +++ b/pkg/cgroups/cgroups.go @@ -24,16 +24,3 @@ type Cgroup struct { type ActiveCgroup interface { Cleanup() error } - -func Apply(c *Cgroup, pid int) (ActiveCgroup, error) { - // We have two implementation of cgroups support, one is based on - // systemd and the dbus api, and one is based on raw cgroup fs operations - // following the pre-single-writer model docs at: - // http://www.freedesktop.org/wiki/Software/systemd/PaxControlGroups/ - - if useSystemd() { - return systemdApply(c, pid) - } else { - return rawApply(c, pid) - } -} diff --git a/pkg/cgroups/systemd/apply_nosystemd.go b/pkg/cgroups/systemd/apply_nosystemd.go new file mode 100644 index 0000000000..226aa59f9d --- /dev/null +++ b/pkg/cgroups/systemd/apply_nosystemd.go @@ -0,0 +1,16 @@ +// +build !linux + +package systemd + +import ( + "fmt" + "github.com/dotcloud/docker/pkg/cgroups" +) + +func UseSystemd() bool { + return false +} + +func systemdApply(c *Cgroup, pid int) (cgroups.ActiveCgroup, error) { + return nil, fmt.Errorf("Systemd not supported") +} diff --git a/pkg/cgroups/apply_systemd.go b/pkg/cgroups/systemd/apply_systemd.go similarity index 86% rename from pkg/cgroups/apply_systemd.go rename to pkg/cgroups/systemd/apply_systemd.go index a9b3a8d301..7c26080d6e 100644 --- a/pkg/cgroups/apply_systemd.go +++ b/pkg/cgroups/systemd/apply_systemd.go @@ -1,27 +1,35 @@ // +build linux -package cgroups +package systemd import ( "fmt" - systemd1 "github.com/coreos/go-systemd/dbus" - "github.com/dotcloud/docker/pkg/systemd" - "github.com/godbus/dbus" + "io/ioutil" "path/filepath" "strings" "sync" + + systemd1 "github.com/coreos/go-systemd/dbus" + "github.com/dotcloud/docker/pkg/cgroups" + "github.com/dotcloud/docker/pkg/systemd" + "github.com/godbus/dbus" ) type systemdCgroup struct { } +type DeviceAllow struct { + Node string + Permissions string +} + var ( connLock sync.Mutex theConn *systemd1.Conn hasStartTransientUnit bool ) -func useSystemd() bool { +func UseSystemd() bool { if !systemd.SdBooted() { return false } @@ -48,15 +56,9 @@ func useSystemd() bool { } } } - return hasStartTransientUnit } -type DeviceAllow struct { - Node string - Permissions string -} - func getIfaceForUnit(unitName string) string { if strings.HasSuffix(unitName, ".scope") { return "Scope" @@ -67,11 +69,12 @@ func getIfaceForUnit(unitName string) string { return "Unit" } -func systemdApply(c *Cgroup, pid int) (ActiveCgroup, error) { - unitName := c.Parent + "-" + c.Name + ".scope" - slice := "system.slice" - - var properties []systemd1.Property +func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) { + var ( + unitName = c.Parent + "-" + c.Name + ".scope" + slice = "system.slice" + properties []systemd1.Property + ) for _, v := range c.UnitProperties { switch v[0] { @@ -85,7 +88,8 @@ func systemdApply(c *Cgroup, pid int) (ActiveCgroup, error) { properties = append(properties, systemd1.Property{"Slice", dbus.MakeVariant(slice)}, systemd1.Property{"Description", dbus.MakeVariant("docker container " + c.Name)}, - systemd1.Property{"PIDs", dbus.MakeVariant([]uint32{uint32(pid)})}) + systemd1.Property{"PIDs", dbus.MakeVariant([]uint32{uint32(pid)})}, + ) if !c.DeviceAccess { properties = append(properties, @@ -138,7 +142,7 @@ func systemdApply(c *Cgroup, pid int) (ActiveCgroup, error) { cgroup := props["ControlGroup"].(string) if !c.DeviceAccess { - mountpoint, err := FindCgroupMountpoint("devices") + mountpoint, err := cgroups.FindCgroupMountpoint("devices") if err != nil { return nil, err } @@ -146,15 +150,14 @@ func systemdApply(c *Cgroup, pid int) (ActiveCgroup, error) { path := filepath.Join(mountpoint, cgroup) // /dev/pts/* - if err := writeFile(path, "devices.allow", "c 136:* rwm"); err != nil { + if err := ioutil.WriteFile(filepath.Join(path, "devices.allow"), []byte("c 136:* rwm"), 0700); err != nil { return nil, err } // tuntap - if err := writeFile(path, "devices.allow", "c 10:200 rwm"); err != nil { + if err := ioutil.WriteFile(filepath.Join(path, "devices.allow"), []byte("c 10:200 rwm"), 0700); err != nil { return nil, err } } - return &systemdCgroup{}, nil } diff --git a/pkg/libcontainer/nsinit/exec.go b/pkg/libcontainer/nsinit/exec.go index 4e2fcefd47..7a315d6f80 100644 --- a/pkg/libcontainer/nsinit/exec.go +++ b/pkg/libcontainer/nsinit/exec.go @@ -8,6 +8,7 @@ import ( "syscall" "github.com/dotcloud/docker/pkg/cgroups" + "github.com/dotcloud/docker/pkg/cgroups/systemd" "github.com/dotcloud/docker/pkg/libcontainer" "github.com/dotcloud/docker/pkg/libcontainer/network" "github.com/dotcloud/docker/pkg/system" @@ -99,7 +100,11 @@ func (ns *linuxNs) Exec(container *libcontainer.Container, term Terminal, args [ func (ns *linuxNs) SetupCgroups(container *libcontainer.Container, nspid int) (cgroups.ActiveCgroup, error) { if container.Cgroups != nil { - return cgroups.Apply(container.Cgroups, nspid) + c := container.Cgroups + if systemd.UseSystemd() { + return systemd.Apply(c, nspid) + } + return rawApply(c, nspid) } return nil, nil } From 06db0604e5e5438dc14e13a069ebddcab7bb4bc6 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 18 Apr 2014 21:34:26 -0700 Subject: [PATCH 03/14] Move raw cgroups into fs package (filesystem) Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/{ => fs}/apply_raw.go | 41 ++++++++++++++++++------------- pkg/cgroups/utils.go | 6 ----- pkg/libcontainer/nsinit/exec.go | 3 ++- 3 files changed, 26 insertions(+), 24 deletions(-) rename pkg/cgroups/{ => fs}/apply_raw.go (78%) diff --git a/pkg/cgroups/apply_raw.go b/pkg/cgroups/fs/apply_raw.go similarity index 78% rename from pkg/cgroups/apply_raw.go rename to pkg/cgroups/fs/apply_raw.go index 471d3fcf53..d8267bf4fa 100644 --- a/pkg/cgroups/apply_raw.go +++ b/pkg/cgroups/fs/apply_raw.go @@ -1,10 +1,13 @@ -package cgroups +package fs import ( "fmt" + "io/ioutil" "os" "path/filepath" "strconv" + + "github.com/dotcloud/docker/pkg/cgroups" ) type rawCgroup struct { @@ -12,7 +15,7 @@ type rawCgroup struct { cgroup string } -func rawApply(c *Cgroup, pid int) (ActiveCgroup, error) { +func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) { // We have two implementation of cgroups support, one is based on // systemd and the dbus api, and one is based on raw cgroup fs operations // following the pre-single-writer model docs at: @@ -20,7 +23,7 @@ func rawApply(c *Cgroup, pid int) (ActiveCgroup, error) { // // we can pick any subsystem to find the root - cgroupRoot, err := FindCgroupMountpoint("cpu") + cgroupRoot, err := cgroups.FindCgroupMountpoint("cpu") if err != nil { return nil, err } @@ -39,7 +42,7 @@ func rawApply(c *Cgroup, pid int) (ActiveCgroup, error) { root: cgroupRoot, cgroup: cgroup, } - for _, g := range []func(*Cgroup, int) error{ + for _, g := range []func(*cgroups.Cgroup, int) error{ raw.setupDevices, raw.setupMemory, raw.setupCpu, @@ -58,7 +61,7 @@ func rawApply(c *Cgroup, pid int) (ActiveCgroup, error) { } func (raw *rawCgroup) path(subsystem string) (string, error) { - initPath, err := GetInitCgroupDir(subsystem) + initPath, err := cgroups.GetInitCgroupDir(subsystem) if err != nil { return "", err } @@ -79,7 +82,7 @@ func (raw *rawCgroup) join(subsystem string, pid int) (string, error) { return path, nil } -func (raw *rawCgroup) setupDevices(c *Cgroup, pid int) (err error) { +func (raw *rawCgroup) setupDevices(c *cgroups.Cgroup, pid int) (err error) { dir, err := raw.join("devices", pid) if err != nil { return err @@ -133,7 +136,7 @@ func (raw *rawCgroup) setupDevices(c *Cgroup, pid int) (err error) { return nil } -func (raw *rawCgroup) setupMemory(c *Cgroup, pid int) (err error) { +func (raw *rawCgroup) setupMemory(c *cgroups.Cgroup, pid int) (err error) { dir, err := raw.join("memory", pid) // only return an error for memory if it was not specified if err != nil && (c.Memory != 0 || c.MemorySwap != 0) { @@ -165,7 +168,7 @@ func (raw *rawCgroup) setupMemory(c *Cgroup, pid int) (err error) { return nil } -func (raw *rawCgroup) setupCpu(c *Cgroup, pid int) (err error) { +func (raw *rawCgroup) setupCpu(c *cgroups.Cgroup, pid int) (err error) { // We always want to join the cpu group, to allow fair cpu scheduling // on a container basis dir, err := raw.join("cpu", pid) @@ -180,7 +183,7 @@ func (raw *rawCgroup) setupCpu(c *Cgroup, pid int) (err error) { return nil } -func (raw *rawCgroup) setupCpuset(c *Cgroup, pid int) (err error) { +func (raw *rawCgroup) setupCpuset(c *cgroups.Cgroup, pid int) (err error) { // we don't want to join this cgroup unless it is specified if c.CpusetCpus != "" { dir, err := raw.join("cpuset", pid) @@ -200,33 +203,33 @@ func (raw *rawCgroup) setupCpuset(c *Cgroup, pid int) (err error) { return nil } -func (raw *rawCgroup) setupCpuacct(c *Cgroup, pid int) error { +func (raw *rawCgroup) setupCpuacct(c *cgroups.Cgroup, pid int) error { // we just want to join this group even though we don't set anything - if _, err := raw.join("cpuacct", pid); err != nil && err != ErrNotFound { + if _, err := raw.join("cpuacct", pid); err != nil && err != cgroups.ErrNotFound { return err } return nil } -func (raw *rawCgroup) setupBlkio(c *Cgroup, pid int) error { +func (raw *rawCgroup) setupBlkio(c *cgroups.Cgroup, pid int) error { // we just want to join this group even though we don't set anything - if _, err := raw.join("blkio", pid); err != nil && err != ErrNotFound { + if _, err := raw.join("blkio", pid); err != nil && err != cgroups.ErrNotFound { return err } return nil } -func (raw *rawCgroup) setupPerfevent(c *Cgroup, pid int) error { +func (raw *rawCgroup) setupPerfevent(c *cgroups.Cgroup, pid int) error { // we just want to join this group even though we don't set anything - if _, err := raw.join("perf_event", pid); err != nil && err != ErrNotFound { + if _, err := raw.join("perf_event", pid); err != nil && err != cgroups.ErrNotFound { return err } return nil } -func (raw *rawCgroup) setupFreezer(c *Cgroup, pid int) error { +func (raw *rawCgroup) setupFreezer(c *cgroups.Cgroup, pid int) error { // we just want to join this group even though we don't set anything - if _, err := raw.join("freezer", pid); err != nil && err != ErrNotFound { + if _, err := raw.join("freezer", pid); err != nil && err != cgroups.ErrNotFound { return err } return nil @@ -254,3 +257,7 @@ func (raw *rawCgroup) Cleanup() error { } return nil } + +func writeFile(dir, file, data string) error { + return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) +} diff --git a/pkg/cgroups/utils.go b/pkg/cgroups/utils.go index 82d5a955a3..02a7f357f6 100644 --- a/pkg/cgroups/utils.go +++ b/pkg/cgroups/utils.go @@ -3,9 +3,7 @@ package cgroups import ( "bufio" "io" - "io/ioutil" "os" - "path/filepath" "strings" "github.com/dotcloud/docker/pkg/mount" @@ -67,7 +65,3 @@ func parseCgroupFile(subsystem string, r io.Reader) (string, error) { } return "", ErrNotFound } - -func writeFile(dir, file, data string) error { - return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) -} diff --git a/pkg/libcontainer/nsinit/exec.go b/pkg/libcontainer/nsinit/exec.go index 7a315d6f80..e76e060d1c 100644 --- a/pkg/libcontainer/nsinit/exec.go +++ b/pkg/libcontainer/nsinit/exec.go @@ -8,6 +8,7 @@ import ( "syscall" "github.com/dotcloud/docker/pkg/cgroups" + "github.com/dotcloud/docker/pkg/cgroups/fs" "github.com/dotcloud/docker/pkg/cgroups/systemd" "github.com/dotcloud/docker/pkg/libcontainer" "github.com/dotcloud/docker/pkg/libcontainer/network" @@ -104,7 +105,7 @@ func (ns *linuxNs) SetupCgroups(container *libcontainer.Container, nspid int) (c if systemd.UseSystemd() { return systemd.Apply(c, nspid) } - return rawApply(c, nspid) + return fs.Apply(c, nspid) } return nil, nil } From e92f2fd3950644a6e5eb38259cee9036648090cb Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 18 Apr 2014 21:55:06 -0700 Subject: [PATCH 04/14] Break down groups into subsystems Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/fs/apply_raw.go | 202 ++++++----------------------------- pkg/cgroups/fs/blkio.go | 16 +++ pkg/cgroups/fs/cpu.go | 23 ++++ pkg/cgroups/fs/cpuacct.go | 16 +++ pkg/cgroups/fs/cpuset.go | 28 +++++ pkg/cgroups/fs/devices.go | 61 +++++++++++ pkg/cgroups/fs/freezer.go | 16 +++ pkg/cgroups/fs/memory.go | 41 +++++++ pkg/cgroups/fs/perf_event.go | 16 +++ 9 files changed, 247 insertions(+), 172 deletions(-) create mode 100644 pkg/cgroups/fs/blkio.go create mode 100644 pkg/cgroups/fs/cpu.go create mode 100644 pkg/cgroups/fs/cpuacct.go create mode 100644 pkg/cgroups/fs/cpuset.go create mode 100644 pkg/cgroups/fs/devices.go create mode 100644 pkg/cgroups/fs/freezer.go create mode 100644 pkg/cgroups/fs/memory.go create mode 100644 pkg/cgroups/fs/perf_event.go diff --git a/pkg/cgroups/fs/apply_raw.go b/pkg/cgroups/fs/apply_raw.go index d8267bf4fa..4f1b379726 100644 --- a/pkg/cgroups/fs/apply_raw.go +++ b/pkg/cgroups/fs/apply_raw.go @@ -10,9 +10,28 @@ import ( "github.com/dotcloud/docker/pkg/cgroups" ) -type rawCgroup struct { +var ( + subsystems = []subsystem{ + &devicesGroup{}, + &memoryGroup{}, + &cpuGroup{}, + &cpusetGroup{}, + &cpuacctGroup{}, + &blkioGroup{}, + &perfEventGroup{}, + &freezerGroup{}, + } +) + +type subsystem interface { + Set(*data) error +} + +type data struct { root string cgroup string + c *cgroups.Cgroup + pid int } func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) { @@ -38,29 +57,21 @@ func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) { cgroup = filepath.Join(c.Parent, cgroup) } - raw := &rawCgroup{ + d := &data{ root: cgroupRoot, cgroup: cgroup, + c: c, + pid: pid, } - for _, g := range []func(*cgroups.Cgroup, int) error{ - raw.setupDevices, - raw.setupMemory, - raw.setupCpu, - raw.setupCpuset, - raw.setupCpuacct, - raw.setupBlkio, - raw.setupPerfevent, - raw.setupFreezer, - } { - if err := g(c, pid); err != nil { + for _, sys := range subsystems { + if err := sys.Set(d); err != nil { return nil, err } } - - return raw, nil + return d, nil } -func (raw *rawCgroup) path(subsystem string) (string, error) { +func (raw *data) path(subsystem string) (string, error) { initPath, err := cgroups.GetInitCgroupDir(subsystem) if err != nil { return "", err @@ -68,7 +79,7 @@ func (raw *rawCgroup) path(subsystem string) (string, error) { return filepath.Join(raw.root, subsystem, initPath, raw.cgroup), nil } -func (raw *rawCgroup) join(subsystem string, pid int) (string, error) { +func (raw *data) join(subsystem string) (string, error) { path, err := raw.path(subsystem) if err != nil { return "", err @@ -76,166 +87,13 @@ func (raw *rawCgroup) join(subsystem string, pid int) (string, error) { if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) { return "", err } - if err := writeFile(path, "cgroup.procs", strconv.Itoa(pid)); err != nil { + if err := writeFile(path, "cgroup.procs", strconv.Itoa(raw.pid)); err != nil { return "", err } return path, nil } -func (raw *rawCgroup) setupDevices(c *cgroups.Cgroup, pid int) (err error) { - dir, err := raw.join("devices", pid) - if err != nil { - return err - } - defer func() { - if err != nil { - os.RemoveAll(dir) - } - }() - - if !c.DeviceAccess { - - if err := writeFile(dir, "devices.deny", "a"); err != nil { - return err - } - - allow := []string{ - // allow mknod for any device - "c *:* m", - "b *:* m", - - // /dev/null, zero, full - "c 1:3 rwm", - "c 1:5 rwm", - "c 1:7 rwm", - - // consoles - "c 5:1 rwm", - "c 5:0 rwm", - "c 4:0 rwm", - "c 4:1 rwm", - - // /dev/urandom,/dev/random - "c 1:9 rwm", - "c 1:8 rwm", - - // /dev/pts/ - pts namespaces are "coming soon" - "c 136:* rwm", - "c 5:2 rwm", - - // tuntap - "c 10:200 rwm", - } - - for _, val := range allow { - if err := writeFile(dir, "devices.allow", val); err != nil { - return err - } - } - } - return nil -} - -func (raw *rawCgroup) setupMemory(c *cgroups.Cgroup, pid int) (err error) { - dir, err := raw.join("memory", pid) - // only return an error for memory if it was not specified - if err != nil && (c.Memory != 0 || c.MemorySwap != 0) { - return err - } - defer func() { - if err != nil { - os.RemoveAll(dir) - } - }() - - if c.Memory != 0 || c.MemorySwap != 0 { - if c.Memory != 0 { - if err := writeFile(dir, "memory.limit_in_bytes", strconv.FormatInt(c.Memory, 10)); err != nil { - return err - } - if err := writeFile(dir, "memory.soft_limit_in_bytes", strconv.FormatInt(c.Memory, 10)); err != nil { - return err - } - } - // By default, MemorySwap is set to twice the size of RAM. - // If you want to omit MemorySwap, set it to `-1'. - if c.MemorySwap != -1 { - if err := writeFile(dir, "memory.memsw.limit_in_bytes", strconv.FormatInt(c.Memory*2, 10)); err != nil { - return err - } - } - } - return nil -} - -func (raw *rawCgroup) setupCpu(c *cgroups.Cgroup, pid int) (err error) { - // We always want to join the cpu group, to allow fair cpu scheduling - // on a container basis - dir, err := raw.join("cpu", pid) - if err != nil { - return err - } - if c.CpuShares != 0 { - if err := writeFile(dir, "cpu.shares", strconv.FormatInt(c.CpuShares, 10)); err != nil { - return err - } - } - return nil -} - -func (raw *rawCgroup) setupCpuset(c *cgroups.Cgroup, pid int) (err error) { - // we don't want to join this cgroup unless it is specified - if c.CpusetCpus != "" { - dir, err := raw.join("cpuset", pid) - if err != nil && c.CpusetCpus != "" { - return err - } - defer func() { - if err != nil { - os.RemoveAll(dir) - } - }() - - if err := writeFile(dir, "cpuset.cpus", c.CpusetCpus); err != nil { - return err - } - } - return nil -} - -func (raw *rawCgroup) setupCpuacct(c *cgroups.Cgroup, pid int) error { - // we just want to join this group even though we don't set anything - if _, err := raw.join("cpuacct", pid); err != nil && err != cgroups.ErrNotFound { - return err - } - return nil -} - -func (raw *rawCgroup) setupBlkio(c *cgroups.Cgroup, pid int) error { - // we just want to join this group even though we don't set anything - if _, err := raw.join("blkio", pid); err != nil && err != cgroups.ErrNotFound { - return err - } - return nil -} - -func (raw *rawCgroup) setupPerfevent(c *cgroups.Cgroup, pid int) error { - // we just want to join this group even though we don't set anything - if _, err := raw.join("perf_event", pid); err != nil && err != cgroups.ErrNotFound { - return err - } - return nil -} - -func (raw *rawCgroup) setupFreezer(c *cgroups.Cgroup, pid int) error { - // we just want to join this group even though we don't set anything - if _, err := raw.join("freezer", pid); err != nil && err != cgroups.ErrNotFound { - return err - } - return nil -} - -func (raw *rawCgroup) Cleanup() error { +func (raw *data) Cleanup() error { get := func(subsystem string) string { path, _ := raw.path(subsystem) return path diff --git a/pkg/cgroups/fs/blkio.go b/pkg/cgroups/fs/blkio.go new file mode 100644 index 0000000000..f0096f8767 --- /dev/null +++ b/pkg/cgroups/fs/blkio.go @@ -0,0 +1,16 @@ +package fs + +import ( + "github.com/dotcloud/docker/pkg/cgroups" +) + +type blkioGroup struct { +} + +func (s *blkioGroup) Set(d *data) error { + // we just want to join this group even though we don't set anything + if _, err := d.join("blkio"); err != nil && err != cgroups.ErrNotFound { + return err + } + return nil +} diff --git a/pkg/cgroups/fs/cpu.go b/pkg/cgroups/fs/cpu.go new file mode 100644 index 0000000000..33879bad14 --- /dev/null +++ b/pkg/cgroups/fs/cpu.go @@ -0,0 +1,23 @@ +package fs + +import ( + "strconv" +) + +type cpuGroup struct { +} + +func (s *cpuGroup) Set(d *data) error { + // We always want to join the cpu group, to allow fair cpu scheduling + // on a container basis + dir, err := d.join("cpu") + if err != nil { + return err + } + if d.c.CpuShares != 0 { + if err := writeFile(dir, "cpu.shares", strconv.FormatInt(d.c.CpuShares, 10)); err != nil { + return err + } + } + return nil +} diff --git a/pkg/cgroups/fs/cpuacct.go b/pkg/cgroups/fs/cpuacct.go new file mode 100644 index 0000000000..603b100571 --- /dev/null +++ b/pkg/cgroups/fs/cpuacct.go @@ -0,0 +1,16 @@ +package fs + +import ( + "github.com/dotcloud/docker/pkg/cgroups" +) + +type cpuacctGroup struct { +} + +func (s *cpuacctGroup) Set(d *data) error { + // we just want to join this group even though we don't set anything + if _, err := d.join("cpuacct"); err != nil && err != cgroups.ErrNotFound { + return err + } + return nil +} diff --git a/pkg/cgroups/fs/cpuset.go b/pkg/cgroups/fs/cpuset.go new file mode 100644 index 0000000000..5108365ebd --- /dev/null +++ b/pkg/cgroups/fs/cpuset.go @@ -0,0 +1,28 @@ +package fs + +import ( + "os" +) + +type cpusetGroup struct { +} + +func (s *cpusetGroup) Set(d *data) error { + // we don't want to join this cgroup unless it is specified + if d.c.CpusetCpus != "" { + dir, err := d.join("cpuset") + if err != nil && d.c.CpusetCpus != "" { + return err + } + defer func() { + if err != nil { + os.RemoveAll(dir) + } + }() + + if err := writeFile(dir, "cpuset.cpus", d.c.CpusetCpus); err != nil { + return err + } + } + return nil +} diff --git a/pkg/cgroups/fs/devices.go b/pkg/cgroups/fs/devices.go new file mode 100644 index 0000000000..769df39283 --- /dev/null +++ b/pkg/cgroups/fs/devices.go @@ -0,0 +1,61 @@ +package fs + +import ( + "os" +) + +type devicesGroup struct { +} + +func (s *devicesGroup) Set(d *data) error { + dir, err := d.join("devices") + if err != nil { + return err + } + defer func() { + if err != nil { + os.RemoveAll(dir) + } + }() + + if !d.c.DeviceAccess { + if err := writeFile(dir, "devices.deny", "a"); err != nil { + return err + } + + allow := []string{ + // allow mknod for any device + "c *:* m", + "b *:* m", + + // /dev/null, zero, full + "c 1:3 rwm", + "c 1:5 rwm", + "c 1:7 rwm", + + // consoles + "c 5:1 rwm", + "c 5:0 rwm", + "c 4:0 rwm", + "c 4:1 rwm", + + // /dev/urandom,/dev/random + "c 1:9 rwm", + "c 1:8 rwm", + + // /dev/pts/ - pts namespaces are "coming soon" + "c 136:* rwm", + "c 5:2 rwm", + + // tuntap + "c 10:200 rwm", + } + + for _, val := range allow { + if err := writeFile(dir, "devices.allow", val); err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/cgroups/fs/freezer.go b/pkg/cgroups/fs/freezer.go new file mode 100644 index 0000000000..51db689b0a --- /dev/null +++ b/pkg/cgroups/fs/freezer.go @@ -0,0 +1,16 @@ +package fs + +import ( + "github.com/dotcloud/docker/pkg/cgroups" +) + +type freezerGroup struct { +} + +func (s *freezerGroup) Set(d *data) error { + // we just want to join this group even though we don't set anything + if _, err := d.join("freezer"); err != nil && err != cgroups.ErrNotFound { + return err + } + return nil +} diff --git a/pkg/cgroups/fs/memory.go b/pkg/cgroups/fs/memory.go new file mode 100644 index 0000000000..cf33d66117 --- /dev/null +++ b/pkg/cgroups/fs/memory.go @@ -0,0 +1,41 @@ +package fs + +import ( + "os" + "strconv" +) + +type memoryGroup struct { +} + +func (s *memoryGroup) Set(d *data) error { + dir, err := d.join("memory") + // only return an error for memory if it was not specified + if err != nil && (d.c.Memory != 0 || d.c.MemorySwap != 0) { + return err + } + defer func() { + if err != nil { + os.RemoveAll(dir) + } + }() + + if d.c.Memory != 0 || d.c.MemorySwap != 0 { + if d.c.Memory != 0 { + if err := writeFile(dir, "memory.limit_in_bytes", strconv.FormatInt(d.c.Memory, 10)); err != nil { + return err + } + if err := writeFile(dir, "memory.soft_limit_in_bytes", strconv.FormatInt(d.c.Memory, 10)); err != nil { + return err + } + } + // By default, MemorySwap is set to twice the size of RAM. + // If you want to omit MemorySwap, set it to `-1'. + if d.c.MemorySwap != -1 { + if err := writeFile(dir, "memory.memsw.limit_in_bytes", strconv.FormatInt(d.c.Memory*2, 10)); err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/cgroups/fs/perf_event.go b/pkg/cgroups/fs/perf_event.go new file mode 100644 index 0000000000..2af6d7a512 --- /dev/null +++ b/pkg/cgroups/fs/perf_event.go @@ -0,0 +1,16 @@ +package fs + +import ( + "github.com/dotcloud/docker/pkg/cgroups" +) + +type perfEventGroup struct { +} + +func (s *perfEventGroup) Set(d *data) error { + // we just want to join this group even though we don't set anything + if _, err := d.join("perf_event"); err != nil && err != cgroups.ErrNotFound { + return err + } + return nil +} From 7fdeda87173ee6722ef1cbe7b21a13ac8b173365 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Fri, 18 Apr 2014 22:17:31 -0700 Subject: [PATCH 05/14] Add remove method to subsystems Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/fs/apply_raw.go | 31 +++++++++++++------------------ pkg/cgroups/fs/blkio.go | 4 ++++ pkg/cgroups/fs/cpu.go | 4 ++++ pkg/cgroups/fs/cpuacct.go | 4 ++++ pkg/cgroups/fs/cpuset.go | 4 ++++ pkg/cgroups/fs/devices.go | 4 ++++ pkg/cgroups/fs/freezer.go | 4 ++++ pkg/cgroups/fs/memory.go | 4 ++++ pkg/cgroups/fs/perf_event.go | 4 ++++ 9 files changed, 45 insertions(+), 18 deletions(-) diff --git a/pkg/cgroups/fs/apply_raw.go b/pkg/cgroups/fs/apply_raw.go index 4f1b379726..cdb2b354c6 100644 --- a/pkg/cgroups/fs/apply_raw.go +++ b/pkg/cgroups/fs/apply_raw.go @@ -25,6 +25,7 @@ var ( type subsystem interface { Set(*data) error + Remove(*data) error } type data struct { @@ -94,24 +95,8 @@ func (raw *data) join(subsystem string) (string, error) { } func (raw *data) Cleanup() error { - get := func(subsystem string) string { - path, _ := raw.path(subsystem) - return path - } - - for _, path := range []string{ - get("memory"), - get("devices"), - get("cpu"), - get("cpuset"), - get("cpuacct"), - get("blkio"), - get("perf_event"), - get("freezer"), - } { - if path != "" { - os.RemoveAll(path) - } + for _, sys := range subsystems { + sys.Remove(raw) } return nil } @@ -119,3 +104,13 @@ func (raw *data) Cleanup() error { func writeFile(dir, file, data string) error { return ioutil.WriteFile(filepath.Join(dir, file), []byte(data), 0700) } + +func removePath(p string, err error) error { + if err != nil { + return err + } + if p != "" { + return os.RemoveAll(p) + } + return nil +} diff --git a/pkg/cgroups/fs/blkio.go b/pkg/cgroups/fs/blkio.go index f0096f8767..48dcf1a54d 100644 --- a/pkg/cgroups/fs/blkio.go +++ b/pkg/cgroups/fs/blkio.go @@ -14,3 +14,7 @@ func (s *blkioGroup) Set(d *data) error { } return nil } + +func (s *blkioGroup) Remove(d *data) error { + return removePath(d.path("blkio")) +} diff --git a/pkg/cgroups/fs/cpu.go b/pkg/cgroups/fs/cpu.go index 33879bad14..f458d7915e 100644 --- a/pkg/cgroups/fs/cpu.go +++ b/pkg/cgroups/fs/cpu.go @@ -21,3 +21,7 @@ func (s *cpuGroup) Set(d *data) error { } return nil } + +func (s *cpuGroup) Remove(d *data) error { + return removePath(d.path("cpu")) +} diff --git a/pkg/cgroups/fs/cpuacct.go b/pkg/cgroups/fs/cpuacct.go index 603b100571..22dfb4185f 100644 --- a/pkg/cgroups/fs/cpuacct.go +++ b/pkg/cgroups/fs/cpuacct.go @@ -14,3 +14,7 @@ func (s *cpuacctGroup) Set(d *data) error { } return nil } + +func (s *cpuacctGroup) Remove(d *data) error { + return removePath(d.path("cpuacct")) +} diff --git a/pkg/cgroups/fs/cpuset.go b/pkg/cgroups/fs/cpuset.go index 5108365ebd..3d3b15f113 100644 --- a/pkg/cgroups/fs/cpuset.go +++ b/pkg/cgroups/fs/cpuset.go @@ -26,3 +26,7 @@ func (s *cpusetGroup) Set(d *data) error { } return nil } + +func (s *cpusetGroup) Remove(d *data) error { + return removePath(d.path("cpuset")) +} diff --git a/pkg/cgroups/fs/devices.go b/pkg/cgroups/fs/devices.go index 769df39283..fc5d83b9c9 100644 --- a/pkg/cgroups/fs/devices.go +++ b/pkg/cgroups/fs/devices.go @@ -59,3 +59,7 @@ func (s *devicesGroup) Set(d *data) error { } return nil } + +func (s *devicesGroup) Remove(d *data) error { + return removePath(d.path("devices")) +} diff --git a/pkg/cgroups/fs/freezer.go b/pkg/cgroups/fs/freezer.go index 51db689b0a..05bc584cf4 100644 --- a/pkg/cgroups/fs/freezer.go +++ b/pkg/cgroups/fs/freezer.go @@ -14,3 +14,7 @@ func (s *freezerGroup) Set(d *data) error { } return nil } + +func (s *freezerGroup) Remove(d *data) error { + return removePath(d.path("freezer")) +} diff --git a/pkg/cgroups/fs/memory.go b/pkg/cgroups/fs/memory.go index cf33d66117..a098f21df9 100644 --- a/pkg/cgroups/fs/memory.go +++ b/pkg/cgroups/fs/memory.go @@ -39,3 +39,7 @@ func (s *memoryGroup) Set(d *data) error { } return nil } + +func (s *memoryGroup) Remove(d *data) error { + return removePath(d.path("memory")) +} diff --git a/pkg/cgroups/fs/perf_event.go b/pkg/cgroups/fs/perf_event.go index 2af6d7a512..b5ec6c6fd3 100644 --- a/pkg/cgroups/fs/perf_event.go +++ b/pkg/cgroups/fs/perf_event.go @@ -14,3 +14,7 @@ func (s *perfEventGroup) Set(d *data) error { } return nil } + +func (s *perfEventGroup) Remove(d *data) error { + return removePath(d.path("perf_event")) +} From 86e34ce59fb7f6e6e284a33f2538bafdc9114c89 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sun, 20 Apr 2014 11:34:28 -0700 Subject: [PATCH 06/14] Squashed commit of the following: commit 75af1649b063abbc5d662fd2f8bc4ff62c927687 Author: Evan Hazlett Date: Sun Apr 20 01:32:42 2014 -0400 more refactor commit 43b36d0f15d634497127bcb17dacaa70ae92e903 Author: Evan Hazlett Date: Sun Apr 20 01:11:49 2014 -0400 refactored cgroup param parsing to util func commit e3738b0168a075bd92ec828879b0e46bdbbe3845 Author: Evan Hazlett Date: Sun Apr 20 00:57:19 2014 -0400 dat error checking commit 57872bcc59403ecd308cfe97c78f73d6ca58d165 Author: Evan Hazlett Date: Sun Apr 20 00:43:25 2014 -0400 proper use of fmt.Errorf commit 43dad6acc0cb21aac2b04ce074699879898ee820 Author: Evan Hazlett Date: Sun Apr 20 00:36:45 2014 -0400 proper placement of defer commit b7f20b934b2bc92cd39397dbc608b77bff28493c Author: Evan Hazlett Date: Sun Apr 20 00:34:39 2014 -0400 defers, error checking, panic avoidance commit 7a9a6ff267f8806dfe6676486f73fe89b72968fb Author: Evan Hazlett Date: Sun Apr 20 00:22:00 2014 -0400 data param to use container info instead of host commit 0e0cf7309be1644687160d6519db792b23cd26e9 Author: Evan Hazlett Date: Sun Apr 20 00:11:29 2014 -0400 added stats for cpuacct, memory, and blkio Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/fs/blkio.go | 36 ++++++++++++++++++++ pkg/cgroups/fs/cpuacct.go | 69 +++++++++++++++++++++++++++++++++++++++ pkg/cgroups/fs/memory.go | 26 +++++++++++++++ pkg/cgroups/fs/utils.go | 24 ++++++++++++++ 4 files changed, 155 insertions(+) create mode 100644 pkg/cgroups/fs/utils.go diff --git a/pkg/cgroups/fs/blkio.go b/pkg/cgroups/fs/blkio.go index 48dcf1a54d..fae8d5a037 100644 --- a/pkg/cgroups/fs/blkio.go +++ b/pkg/cgroups/fs/blkio.go @@ -1,6 +1,11 @@ package fs import ( + "bufio" + "fmt" + "os" + "path/filepath" + "github.com/dotcloud/docker/pkg/cgroups" ) @@ -18,3 +23,34 @@ func (s *blkioGroup) Set(d *data) error { func (s *blkioGroup) Remove(d *data) error { return removePath(d.path("blkio")) } + +func (s *blkioGroup) Stats(d *data) (map[string]float64, error) { + paramData := make(map[string]float64) + path, err := d.path("blkio") + if err != nil { + return paramData, fmt.Errorf("Unable to read %s cgroup param: %s", path, err) + } + params := []string{ + "sectors", + "io_service_bytes", + "io_serviced", + "io_queued", + } + for _, param := range params { + p := fmt.Sprintf("blkio.%s", param) + f, err := os.Open(filepath.Join(path, p)) + if err != nil { + return paramData, err + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + _, v, err := getCgroupParamKeyValue(sc.Text()) + if err != nil { + return paramData, fmt.Errorf("Error parsing param data: %s", err) + } + paramData[param] = v + } + } + return paramData, nil +} diff --git a/pkg/cgroups/fs/cpuacct.go b/pkg/cgroups/fs/cpuacct.go index 22dfb4185f..621592e31f 100644 --- a/pkg/cgroups/fs/cpuacct.go +++ b/pkg/cgroups/fs/cpuacct.go @@ -1,6 +1,14 @@ package fs import ( + "bufio" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "github.com/dotcloud/docker/pkg/cgroups" ) @@ -18,3 +26,64 @@ func (s *cpuacctGroup) Set(d *data) error { func (s *cpuacctGroup) Remove(d *data) error { return removePath(d.path("cpuacct")) } + +func (s *cpuacctGroup) Stats(d *data) (map[string]float64, error) { + paramData := make(map[string]float64) + path, err := d.path("cpuacct") + if err != nil { + return paramData, fmt.Errorf("Unable to read %s cgroup param: %s", path, err) + } + f, err := os.Open(filepath.Join(path, "cpuacct.stat")) + if err != nil { + return paramData, err + } + defer f.Close() + sc := bufio.NewScanner(f) + cpuTotal := 0.0 + for sc.Scan() { + t, v, err := getCgroupParamKeyValue(sc.Text()) + if err != nil { + return paramData, fmt.Errorf("Error parsing param data: %s", err) + } + // set the raw data in map + paramData[t] = v + cpuTotal += v + } + // calculate percentage from jiffies + // get sys uptime + uf, err := os.Open("/proc/uptime") + if err != nil { + return paramData, fmt.Errorf("Unable to open /proc/uptime") + } + defer uf.Close() + uptimeData, err := ioutil.ReadAll(uf) + if err != nil { + return paramData, fmt.Errorf("Error reading /proc/uptime: %s", err) + } + uptimeFields := strings.Fields(string(uptimeData)) + uptime, err := strconv.ParseFloat(uptimeFields[0], 64) + if err != nil { + return paramData, fmt.Errorf("Error parsing cpu stats: %s", err) + } + // find starttime of process + pf, err := os.Open(filepath.Join(path, "cgroup.procs")) + if err != nil { + return paramData, fmt.Errorf("Error parsing cpu stats: %s", err) + } + defer pf.Close() + pr := bufio.NewReader(pf) + l, _, err := pr.ReadLine() + if err != nil { + return paramData, fmt.Errorf("Error reading param file: %s", err) + } + starttime, err := strconv.ParseFloat(string(l), 64) + if err != nil { + return paramData, fmt.Errorf("Unable to parse starttime: %s", err) + } + // get total elapsed seconds since proc start + seconds := uptime - (starttime / 100) + // finally calc percentage + cpuPercentage := 100.0 * ((cpuTotal / 100.0) / float64(seconds)) + paramData["percentage"] = cpuPercentage + return paramData, nil +} diff --git a/pkg/cgroups/fs/memory.go b/pkg/cgroups/fs/memory.go index a098f21df9..26281c1c75 100644 --- a/pkg/cgroups/fs/memory.go +++ b/pkg/cgroups/fs/memory.go @@ -1,7 +1,10 @@ package fs import ( + "bufio" + "fmt" "os" + "path/filepath" "strconv" ) @@ -43,3 +46,26 @@ func (s *memoryGroup) Set(d *data) error { func (s *memoryGroup) Remove(d *data) error { return removePath(d.path("memory")) } + +func (s *memoryGroup) Stats(d *data) (map[string]float64, error) { + paramData := make(map[string]float64) + path, err := d.path("memory") + if err != nil { + fmt.Errorf("Unable to read %s cgroup param: %s", path, err) + return paramData, err + } + f, err := os.Open(filepath.Join(path, "memory.stat")) + if err != nil { + return paramData, err + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + t, v, err := getCgroupParamKeyValue(sc.Text()) + if err != nil { + return paramData, fmt.Errorf("Error parsing param data: %s", err) + } + paramData[t] = v + } + return paramData, nil +} diff --git a/pkg/cgroups/fs/utils.go b/pkg/cgroups/fs/utils.go new file mode 100644 index 0000000000..260cf13a52 --- /dev/null +++ b/pkg/cgroups/fs/utils.go @@ -0,0 +1,24 @@ +package fs + +import ( + "fmt" + "strconv" + "strings" +) + +// Parses a cgroup param and returns as name, value +// i.e. "io_service_bytes 1234" will return as io_service_bytes, 1234 +func getCgroupParamKeyValue(t string) (string, float64, error) { + parts := strings.Fields(t) + switch len(parts) { + case 2: + name := parts[0] + value, err := strconv.ParseFloat(parts[1], 64) + if err != nil { + return "", 0.0, fmt.Errorf("Unable to convert param value to float: %s", err) + } + return name, value, nil + default: + return "", 0.0, fmt.Errorf("Unable to parse cgroup param: not enough parts; expected 2") + } +} From 9b65f1635515fa35ed38f8a76b6d8f73ab3a1d02 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sun, 20 Apr 2014 12:02:39 -0700 Subject: [PATCH 07/14] Refactor stats and add them to all subsystems Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/fs/apply_raw.go | 1 + pkg/cgroups/fs/blkio.go | 28 ++++++------ pkg/cgroups/fs/cpu.go | 4 ++ pkg/cgroups/fs/cpuacct.go | 84 ++++++++++++++++++------------------ pkg/cgroups/fs/cpuset.go | 4 ++ pkg/cgroups/fs/devices.go | 4 ++ pkg/cgroups/fs/freezer.go | 4 ++ pkg/cgroups/fs/memory.go | 10 ++--- pkg/cgroups/fs/perf_event.go | 4 ++ pkg/cgroups/fs/utils.go | 6 ++- 10 files changed, 88 insertions(+), 61 deletions(-) diff --git a/pkg/cgroups/fs/apply_raw.go b/pkg/cgroups/fs/apply_raw.go index cdb2b354c6..ecc4bb62e9 100644 --- a/pkg/cgroups/fs/apply_raw.go +++ b/pkg/cgroups/fs/apply_raw.go @@ -26,6 +26,7 @@ var ( type subsystem interface { Set(*data) error Remove(*data) error + Stats(*data) (map[string]float64, error) } type data struct { diff --git a/pkg/cgroups/fs/blkio.go b/pkg/cgroups/fs/blkio.go index fae8d5a037..abf8fb6227 100644 --- a/pkg/cgroups/fs/blkio.go +++ b/pkg/cgroups/fs/blkio.go @@ -25,29 +25,33 @@ func (s *blkioGroup) Remove(d *data) error { } func (s *blkioGroup) Stats(d *data) (map[string]float64, error) { - paramData := make(map[string]float64) + var ( + paramData = make(map[string]float64) + params = []string{ + "sectors", + "io_service_bytes", + "io_serviced", + "io_queued", + } + ) + path, err := d.path("blkio") if err != nil { - return paramData, fmt.Errorf("Unable to read %s cgroup param: %s", path, err) - } - params := []string{ - "sectors", - "io_service_bytes", - "io_serviced", - "io_queued", + return nil, err } + for _, param := range params { - p := fmt.Sprintf("blkio.%s", param) - f, err := os.Open(filepath.Join(path, p)) + f, err := os.Open(filepath.Join(path, fmt.Sprintf("blkio.%s", param))) if err != nil { - return paramData, err + return nil, err } defer f.Close() + sc := bufio.NewScanner(f) for sc.Scan() { _, v, err := getCgroupParamKeyValue(sc.Text()) if err != nil { - return paramData, fmt.Errorf("Error parsing param data: %s", err) + return nil, err } paramData[param] = v } diff --git a/pkg/cgroups/fs/cpu.go b/pkg/cgroups/fs/cpu.go index f458d7915e..5534443412 100644 --- a/pkg/cgroups/fs/cpu.go +++ b/pkg/cgroups/fs/cpu.go @@ -25,3 +25,7 @@ func (s *cpuGroup) Set(d *data) error { func (s *cpuGroup) Remove(d *data) error { return removePath(d.path("cpu")) } + +func (s *cpuGroup) Stats(d *data) (map[string]float64, error) { + return nil, ErrNotSupportStat +} diff --git a/pkg/cgroups/fs/cpuacct.go b/pkg/cgroups/fs/cpuacct.go index 621592e31f..b0b14a1475 100644 --- a/pkg/cgroups/fs/cpuacct.go +++ b/pkg/cgroups/fs/cpuacct.go @@ -2,7 +2,6 @@ package fs import ( "bufio" - "fmt" "io/ioutil" "os" "path/filepath" @@ -10,6 +9,7 @@ import ( "strings" "github.com/dotcloud/docker/pkg/cgroups" + "github.com/dotcloud/docker/pkg/system" ) type cpuacctGroup struct { @@ -28,62 +28,62 @@ func (s *cpuacctGroup) Remove(d *data) error { } func (s *cpuacctGroup) Stats(d *data) (map[string]float64, error) { - paramData := make(map[string]float64) + var ( + uptime, startTime float64 + paramData = make(map[string]float64) + cpuTotal = 0.0 + ) + path, err := d.path("cpuacct") - if err != nil { - return paramData, fmt.Errorf("Unable to read %s cgroup param: %s", path, err) - } - f, err := os.Open(filepath.Join(path, "cpuacct.stat")) if err != nil { return paramData, err } + f, err := os.Open(filepath.Join(path, "cpuacct.stat")) + if err != nil { + return nil, err + } defer f.Close() + sc := bufio.NewScanner(f) - cpuTotal := 0.0 for sc.Scan() { t, v, err := getCgroupParamKeyValue(sc.Text()) if err != nil { - return paramData, fmt.Errorf("Error parsing param data: %s", err) + return paramData, err } // set the raw data in map paramData[t] = v cpuTotal += v } - // calculate percentage from jiffies - // get sys uptime - uf, err := os.Open("/proc/uptime") - if err != nil { - return paramData, fmt.Errorf("Unable to open /proc/uptime") + + if uptime, err = s.getUptime(); err != nil { + return nil, err } - defer uf.Close() - uptimeData, err := ioutil.ReadAll(uf) - if err != nil { - return paramData, fmt.Errorf("Error reading /proc/uptime: %s", err) + if startTime, err = s.getProcStarttime(d); err != nil { + return nil, err } - uptimeFields := strings.Fields(string(uptimeData)) - uptime, err := strconv.ParseFloat(uptimeFields[0], 64) - if err != nil { - return paramData, fmt.Errorf("Error parsing cpu stats: %s", err) - } - // find starttime of process - pf, err := os.Open(filepath.Join(path, "cgroup.procs")) - if err != nil { - return paramData, fmt.Errorf("Error parsing cpu stats: %s", err) - } - defer pf.Close() - pr := bufio.NewReader(pf) - l, _, err := pr.ReadLine() - if err != nil { - return paramData, fmt.Errorf("Error reading param file: %s", err) - } - starttime, err := strconv.ParseFloat(string(l), 64) - if err != nil { - return paramData, fmt.Errorf("Unable to parse starttime: %s", err) - } - // get total elapsed seconds since proc start - seconds := uptime - (starttime / 100) - // finally calc percentage - cpuPercentage := 100.0 * ((cpuTotal / 100.0) / float64(seconds)) - paramData["percentage"] = cpuPercentage + paramData["percentage"] = 100.0 * ((cpuTotal/100.0)/uptime - (startTime / 100)) + return paramData, nil } + +func (s *cpuacctGroup) getUptime() (float64, error) { + f, err := os.Open("/proc/uptime") + if err != nil { + return 0, err + } + defer f.Close() + + data, err := ioutil.ReadAll(f) + if err != nil { + return 0, err + } + return strconv.ParseFloat(strings.Fields(string(data))[0], 64) +} + +func (s *cpuacctGroup) getProcStarttime(d *data) (float64, error) { + rawStart, err := system.GetProcessStartTime(d.pid) + if err != nil { + return 0, err + } + return strconv.ParseFloat(rawStart, 64) +} diff --git a/pkg/cgroups/fs/cpuset.go b/pkg/cgroups/fs/cpuset.go index 3d3b15f113..8a13c56cea 100644 --- a/pkg/cgroups/fs/cpuset.go +++ b/pkg/cgroups/fs/cpuset.go @@ -30,3 +30,7 @@ func (s *cpusetGroup) Set(d *data) error { func (s *cpusetGroup) Remove(d *data) error { return removePath(d.path("cpuset")) } + +func (s *cpusetGroup) Stats(d *data) (map[string]float64, error) { + return nil, ErrNotSupportStat +} diff --git a/pkg/cgroups/fs/devices.go b/pkg/cgroups/fs/devices.go index fc5d83b9c9..a2f91eda14 100644 --- a/pkg/cgroups/fs/devices.go +++ b/pkg/cgroups/fs/devices.go @@ -63,3 +63,7 @@ func (s *devicesGroup) Set(d *data) error { func (s *devicesGroup) Remove(d *data) error { return removePath(d.path("devices")) } + +func (s *devicesGroup) Stats(d *data) (map[string]float64, error) { + return nil, ErrNotSupportStat +} diff --git a/pkg/cgroups/fs/freezer.go b/pkg/cgroups/fs/freezer.go index 05bc584cf4..878a3a5fe9 100644 --- a/pkg/cgroups/fs/freezer.go +++ b/pkg/cgroups/fs/freezer.go @@ -18,3 +18,7 @@ func (s *freezerGroup) Set(d *data) error { func (s *freezerGroup) Remove(d *data) error { return removePath(d.path("freezer")) } + +func (s *freezerGroup) Stats(d *data) (map[string]float64, error) { + return nil, ErrNotSupportStat +} diff --git a/pkg/cgroups/fs/memory.go b/pkg/cgroups/fs/memory.go index 26281c1c75..cf4bf5ab73 100644 --- a/pkg/cgroups/fs/memory.go +++ b/pkg/cgroups/fs/memory.go @@ -2,7 +2,6 @@ package fs import ( "bufio" - "fmt" "os" "path/filepath" "strconv" @@ -51,19 +50,20 @@ func (s *memoryGroup) Stats(d *data) (map[string]float64, error) { paramData := make(map[string]float64) path, err := d.path("memory") if err != nil { - fmt.Errorf("Unable to read %s cgroup param: %s", path, err) - return paramData, err + return nil, err } + f, err := os.Open(filepath.Join(path, "memory.stat")) if err != nil { - return paramData, err + return nil, err } defer f.Close() + sc := bufio.NewScanner(f) for sc.Scan() { t, v, err := getCgroupParamKeyValue(sc.Text()) if err != nil { - return paramData, fmt.Errorf("Error parsing param data: %s", err) + return nil, err } paramData[t] = v } diff --git a/pkg/cgroups/fs/perf_event.go b/pkg/cgroups/fs/perf_event.go index b5ec6c6fd3..789b3e59ad 100644 --- a/pkg/cgroups/fs/perf_event.go +++ b/pkg/cgroups/fs/perf_event.go @@ -18,3 +18,7 @@ func (s *perfEventGroup) Set(d *data) error { func (s *perfEventGroup) Remove(d *data) error { return removePath(d.path("perf_event")) } + +func (s *perfEventGroup) Stats(d *data) (map[string]float64, error) { + return nil, ErrNotSupportStat +} diff --git a/pkg/cgroups/fs/utils.go b/pkg/cgroups/fs/utils.go index 260cf13a52..6a0838f3a2 100644 --- a/pkg/cgroups/fs/utils.go +++ b/pkg/cgroups/fs/utils.go @@ -1,23 +1,25 @@ package fs import ( + "errors" "fmt" "strconv" "strings" ) +var ErrNotSupportStat = errors.New("stats are not supported for subsystem") + // Parses a cgroup param and returns as name, value // i.e. "io_service_bytes 1234" will return as io_service_bytes, 1234 func getCgroupParamKeyValue(t string) (string, float64, error) { parts := strings.Fields(t) switch len(parts) { case 2: - name := parts[0] value, err := strconv.ParseFloat(parts[1], 64) if err != nil { return "", 0.0, fmt.Errorf("Unable to convert param value to float: %s", err) } - return name, value, nil + return parts[0], value, nil default: return "", 0.0, fmt.Errorf("Unable to parse cgroup param: not enough parts; expected 2") } From 7f12260fd13329f3bafe4b42ad4193ff963bb1e2 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sun, 20 Apr 2014 17:36:20 -0700 Subject: [PATCH 08/14] Add external function to get cgroup stats Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/fs/apply_raw.go | 47 ++++++++++++++++++++++++++++++------- pkg/cgroups/fs/blkio.go | 30 +++++++++++++++++++++++ pkg/cgroups/fs/cpuacct.go | 3 ++- 3 files changed, 70 insertions(+), 10 deletions(-) diff --git a/pkg/cgroups/fs/apply_raw.go b/pkg/cgroups/fs/apply_raw.go index ecc4bb62e9..60f318e9ef 100644 --- a/pkg/cgroups/fs/apply_raw.go +++ b/pkg/cgroups/fs/apply_raw.go @@ -11,15 +11,15 @@ import ( ) var ( - subsystems = []subsystem{ - &devicesGroup{}, - &memoryGroup{}, - &cpuGroup{}, - &cpusetGroup{}, - &cpuacctGroup{}, - &blkioGroup{}, - &perfEventGroup{}, - &freezerGroup{}, + subsystems = map[string]subsystem{ + "devices": &devicesGroup{}, + "memory": &memoryGroup{}, + "cpu": &cpuGroup{}, + "cpuset": &cpusetGroup{}, + "cpuacct": &cpuacctGroup{}, + "blkio": &blkioGroup{}, + "perf_event": &perfEventGroup{}, + "freezer": &freezerGroup{}, } ) @@ -73,6 +73,35 @@ func Apply(c *cgroups.Cgroup, pid int) (cgroups.ActiveCgroup, error) { return d, nil } +func GetStats(c *cgroups.Cgroup, subsystem string, pid int) (map[string]float64, error) { + cgroupRoot, err := cgroups.FindCgroupMountpoint("cpu") + if err != nil { + return nil, err + } + cgroupRoot = filepath.Dir(cgroupRoot) + + if _, err := os.Stat(cgroupRoot); err != nil { + return nil, fmt.Errorf("cgroups fs not found") + } + + cgroup := c.Name + if c.Parent != "" { + cgroup = filepath.Join(c.Parent, cgroup) + } + + d := &data{ + root: cgroupRoot, + cgroup: cgroup, + c: c, + pid: pid, + } + sys, exists := subsystems[subsystem] + if !exists { + return nil, fmt.Errorf("subsystem %s does not exist", subsystem) + } + return sys.Stats(d) +} + func (raw *data) path(subsystem string) (string, error) { initPath, err := cgroups.GetInitCgroupDir(subsystem) if err != nil { diff --git a/pkg/cgroups/fs/blkio.go b/pkg/cgroups/fs/blkio.go index abf8fb6227..151f2b450d 100644 --- a/pkg/cgroups/fs/blkio.go +++ b/pkg/cgroups/fs/blkio.go @@ -24,6 +24,36 @@ func (s *blkioGroup) Remove(d *data) error { return removePath(d.path("blkio")) } +/* +examples: + + blkio.sectors + 8:0 6792 + + blkio.io_service_bytes + 8:0 Read 1282048 + 8:0 Write 2195456 + 8:0 Sync 2195456 + 8:0 Async 1282048 + 8:0 Total 3477504 + Total 3477504 + + blkio.io_serviced + 8:0 Read 124 + 8:0 Write 104 + 8:0 Sync 104 + 8:0 Async 124 + 8:0 Total 228 + Total 228 + + blkio.io_queued + 8:0 Read 0 + 8:0 Write 0 + 8:0 Sync 0 + 8:0 Async 0 + 8:0 Total 0 + Total 0 +*/ func (s *blkioGroup) Stats(d *data) (map[string]float64, error) { var ( paramData = make(map[string]float64) diff --git a/pkg/cgroups/fs/cpuacct.go b/pkg/cgroups/fs/cpuacct.go index b0b14a1475..a2cbb81bc3 100644 --- a/pkg/cgroups/fs/cpuacct.go +++ b/pkg/cgroups/fs/cpuacct.go @@ -61,7 +61,8 @@ func (s *cpuacctGroup) Stats(d *data) (map[string]float64, error) { if startTime, err = s.getProcStarttime(d); err != nil { return nil, err } - paramData["percentage"] = 100.0 * ((cpuTotal/100.0)/uptime - (startTime / 100)) + //paramData["percentage"] = 100.0 * ((cpuTotal/100.0)/uptime - (startTime / 100)) + paramData["percentage"] = cpuTotal / (uptime - (startTime / 100)) return paramData, nil } From 37248039e186f57768bb5c4fad23b1bca65448f7 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sun, 20 Apr 2014 18:18:17 -0700 Subject: [PATCH 09/14] Fix parsing of blkio files Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/fs/blkio.go | 47 ++++++++++++++++++++++++++++++++++------- pkg/cgroups/fs/utils.go | 7 ++++-- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/pkg/cgroups/fs/blkio.go b/pkg/cgroups/fs/blkio.go index 151f2b450d..79e14fa2dc 100644 --- a/pkg/cgroups/fs/blkio.go +++ b/pkg/cgroups/fs/blkio.go @@ -3,8 +3,11 @@ package fs import ( "bufio" "fmt" + "io/ioutil" "os" "path/filepath" + "strconv" + "strings" "github.com/dotcloud/docker/pkg/cgroups" ) @@ -58,10 +61,9 @@ func (s *blkioGroup) Stats(d *data) (map[string]float64, error) { var ( paramData = make(map[string]float64) params = []string{ - "sectors", - "io_service_bytes", - "io_serviced", - "io_queued", + "io_service_bytes_recursive", + "io_serviced_recursive", + "io_queued_recursive", } ) @@ -70,6 +72,12 @@ func (s *blkioGroup) Stats(d *data) (map[string]float64, error) { return nil, err } + k, v, err := s.getSectors(path) + if err != nil { + return nil, err + } + paramData[fmt.Sprintf("blkio.sectors_recursive:%s", k)] = v + for _, param := range params { f, err := os.Open(filepath.Join(path, fmt.Sprintf("blkio.%s", param))) if err != nil { @@ -79,12 +87,35 @@ func (s *blkioGroup) Stats(d *data) (map[string]float64, error) { sc := bufio.NewScanner(f) for sc.Scan() { - _, v, err := getCgroupParamKeyValue(sc.Text()) - if err != nil { - return nil, err + // format: dev type amount + fields := strings.Fields(sc.Text()) + switch len(fields) { + case 3: + v, err := strconv.ParseFloat(fields[2], 64) + if err != nil { + return nil, err + } + paramData[fmt.Sprintf("%s:%s:%s", param, fields[0], fields[1])] = v + case 2: + // this is the total line, skip + default: + return nil, ErrNotValidFormat } - paramData[param] = v } } return paramData, nil } + +func (s *blkioGroup) getSectors(path string) (string, float64, error) { + f, err := os.Open(filepath.Join(path, "blkio.sectors_recursive")) + if err != nil { + return "", 0, err + } + defer f.Close() + + data, err := ioutil.ReadAll(f) + if err != nil { + return "", 0, err + } + return getCgroupParamKeyValue(string(data)) +} diff --git a/pkg/cgroups/fs/utils.go b/pkg/cgroups/fs/utils.go index 6a0838f3a2..f4c4846b8c 100644 --- a/pkg/cgroups/fs/utils.go +++ b/pkg/cgroups/fs/utils.go @@ -7,7 +7,10 @@ import ( "strings" ) -var ErrNotSupportStat = errors.New("stats are not supported for subsystem") +var ( + ErrNotSupportStat = errors.New("stats are not supported for subsystem") + ErrNotValidFormat = errors.New("line is not a valid key value format") +) // Parses a cgroup param and returns as name, value // i.e. "io_service_bytes 1234" will return as io_service_bytes, 1234 @@ -21,6 +24,6 @@ func getCgroupParamKeyValue(t string) (string, float64, error) { } return parts[0], value, nil default: - return "", 0.0, fmt.Errorf("Unable to parse cgroup param: not enough parts; expected 2") + return "", 0.0, ErrNotValidFormat } } From 3bfe13de2c38fda5b57e2499f474da701495c5ba Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sun, 20 Apr 2014 18:20:44 -0700 Subject: [PATCH 10/14] Reuse cpuacct stats for cpu subsystem Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/fs/cpu.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/cgroups/fs/cpu.go b/pkg/cgroups/fs/cpu.go index 5534443412..8eb0c4ff46 100644 --- a/pkg/cgroups/fs/cpu.go +++ b/pkg/cgroups/fs/cpu.go @@ -27,5 +27,7 @@ func (s *cpuGroup) Remove(d *data) error { } func (s *cpuGroup) Stats(d *data) (map[string]float64, error) { - return nil, ErrNotSupportStat + // we can reuse the cpuacct subsystem to get the cpu stats + sys := subsystems["cpuacct"] + return sys.Stats(d) } From bce49dff0d46dec620e755ed859efa791054a843 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sun, 20 Apr 2014 18:35:33 -0700 Subject: [PATCH 11/14] Add freezer stats This one is a problem because the most useful stat is a string and not a float like verything else. We may have to change the return type Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/fs/freezer.go | 40 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/pkg/cgroups/fs/freezer.go b/pkg/cgroups/fs/freezer.go index 878a3a5fe9..ebf5bb9672 100644 --- a/pkg/cgroups/fs/freezer.go +++ b/pkg/cgroups/fs/freezer.go @@ -1,7 +1,13 @@ package fs import ( + "fmt" "github.com/dotcloud/docker/pkg/cgroups" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" ) type freezerGroup struct { @@ -20,5 +26,37 @@ func (s *freezerGroup) Remove(d *data) error { } func (s *freezerGroup) Stats(d *data) (map[string]float64, error) { - return nil, ErrNotSupportStat + var ( + paramData = make(map[string]float64) + params = []string{ + "parent_freezing", + "self_freezing", + // comment out right now because this is string "state", + } + ) + + path, err := d.path("freezer") + if err != nil { + return nil, err + } + + for _, param := range params { + f, err := os.Open(filepath.Join(path, fmt.Sprintf("freezer.%s", param))) + if err != nil { + return nil, err + } + defer f.Close() + + data, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + v, err := strconv.ParseFloat(strings.TrimSuffix(string(data), "\n"), 64) + if err != nil { + return nil, err + } + paramData[param] = v + } + return paramData, nil } From 2f24b5a9dcc9ca5da07563fb0dd381e49ed844cc Mon Sep 17 00:00:00 2001 From: Evan Hazlett Date: Sun, 20 Apr 2014 23:27:05 -0400 Subject: [PATCH 12/14] work on cpu stats Docker-DCO-1.1-Signed-off-by: Evan Hazlett (github: ehazlett) --- pkg/cgroups/fs/cpuacct.go | 114 ++++++++++++++++++++++++-------------- 1 file changed, 72 insertions(+), 42 deletions(-) diff --git a/pkg/cgroups/fs/cpuacct.go b/pkg/cgroups/fs/cpuacct.go index a2cbb81bc3..aa2cac5d8f 100644 --- a/pkg/cgroups/fs/cpuacct.go +++ b/pkg/cgroups/fs/cpuacct.go @@ -2,11 +2,13 @@ package fs import ( "bufio" - "io/ioutil" + "fmt" "os" "path/filepath" + "runtime" "strconv" "strings" + "time" "github.com/dotcloud/docker/pkg/cgroups" "github.com/dotcloud/docker/pkg/system" @@ -29,58 +31,36 @@ func (s *cpuacctGroup) Remove(d *data) error { func (s *cpuacctGroup) Stats(d *data) (map[string]float64, error) { var ( - uptime, startTime float64 - paramData = make(map[string]float64) - cpuTotal = 0.0 + startCpu, lastCpu, startSystem, lastSystem float64 + paramData = make(map[string]float64) + percentage = 0.0 ) - path, err := d.path("cpuacct") - if err != nil { - return paramData, err - } - f, err := os.Open(filepath.Join(path, "cpuacct.stat")) - if err != nil { + if startCpu, err = s.getCpuUsage(d, path); err != nil { return nil, err } - defer f.Close() - - sc := bufio.NewScanner(f) - for sc.Scan() { - t, v, err := getCgroupParamKeyValue(sc.Text()) - if err != nil { - return paramData, err - } - // set the raw data in map - paramData[t] = v - cpuTotal += v - } - - if uptime, err = s.getUptime(); err != nil { + if startSystem, err = s.getSystemCpuUsage(d); err != nil { return nil, err } - if startTime, err = s.getProcStarttime(d); err != nil { + // sample for 100ms + time.Sleep(100 * time.Millisecond) + if lastCpu, err = s.getCpuUsage(d, path); err != nil { return nil, err } - //paramData["percentage"] = 100.0 * ((cpuTotal/100.0)/uptime - (startTime / 100)) - paramData["percentage"] = cpuTotal / (uptime - (startTime / 100)) - + if lastSystem, err = s.getSystemCpuUsage(d); err != nil { + return nil, err + } + var ( + deltaProc = lastCpu - startCpu + deltaSystem = lastSystem - startSystem + ) + if deltaSystem > 0.0 { + percentage = ((deltaProc / deltaSystem) * 100.0) * float64(runtime.NumCPU()) + } + paramData["percentage"] = percentage return paramData, nil } -func (s *cpuacctGroup) getUptime() (float64, error) { - f, err := os.Open("/proc/uptime") - if err != nil { - return 0, err - } - defer f.Close() - - data, err := ioutil.ReadAll(f) - if err != nil { - return 0, err - } - return strconv.ParseFloat(strings.Fields(string(data))[0], 64) -} - func (s *cpuacctGroup) getProcStarttime(d *data) (float64, error) { rawStart, err := system.GetProcessStartTime(d.pid) if err != nil { @@ -88,3 +68,53 @@ func (s *cpuacctGroup) getProcStarttime(d *data) (float64, error) { } return strconv.ParseFloat(rawStart, 64) } + +func (s *cpuacctGroup) getSystemCpuUsage(d *data) (float64, error) { + total := 0.0 + f, err := os.Open("/proc/stat") + if err != nil { + return 0, err + } + defer f.Close() + + sc := bufio.NewScanner(f) + for sc.Scan() { + txt := sc.Text() + if strings.Index(txt, "cpu") == 0 { + parts := strings.Fields(txt) + partsLength := len(parts) + if partsLength != 11 { + return 0.0, fmt.Errorf("Unable to parse cpu usage: expected 11 fields ; received %d", partsLength) + } + for _, i := range parts[1:10] { + val, err := strconv.ParseFloat(i, 64) + if err != nil { + return 0.0, fmt.Errorf("Unable to convert value to float: %s", err) + } + total += val + } + break + } + } + return total, nil +} + +func (s *cpuacctGroup) getCpuUsage(d *data, path string) (float64, error) { + cpuTotal := 0.0 + f, err := os.Open(filepath.Join(path, "cpuacct.stat")) + if err != nil { + return 0.0, err + } + defer f.Close() + + sc := bufio.NewScanner(f) + for sc.Scan() { + _, v, err := getCgroupParamKeyValue(sc.Text()) + if err != nil { + return 0.0, err + } + // set the raw data in map + cpuTotal += v + } + return cpuTotal, nil +} From f59be989dc92503f29aa94ebf9ea4097ec28b8c9 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Sun, 20 Apr 2014 20:45:08 -0700 Subject: [PATCH 13/14] Refactor stat parsing to use only 8 fields Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/fs/cpuacct.go | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/pkg/cgroups/fs/cpuacct.go b/pkg/cgroups/fs/cpuacct.go index aa2cac5d8f..8c123b79ee 100644 --- a/pkg/cgroups/fs/cpuacct.go +++ b/pkg/cgroups/fs/cpuacct.go @@ -14,6 +14,8 @@ import ( "github.com/dotcloud/docker/pkg/system" ) +var cpuCount = float64(runtime.NumCPU()) + type cpuacctGroup struct { } @@ -32,8 +34,8 @@ func (s *cpuacctGroup) Remove(d *data) error { func (s *cpuacctGroup) Stats(d *data) (map[string]float64, error) { var ( startCpu, lastCpu, startSystem, lastSystem float64 + percentage float64 paramData = make(map[string]float64) - percentage = 0.0 ) path, err := d.path("cpuacct") if startCpu, err = s.getCpuUsage(d, path); err != nil { @@ -50,13 +52,16 @@ func (s *cpuacctGroup) Stats(d *data) (map[string]float64, error) { if lastSystem, err = s.getSystemCpuUsage(d); err != nil { return nil, err } + var ( deltaProc = lastCpu - startCpu deltaSystem = lastSystem - startSystem ) if deltaSystem > 0.0 { - percentage = ((deltaProc / deltaSystem) * 100.0) * float64(runtime.NumCPU()) + percentage = ((deltaProc / deltaSystem) * 100.0) * cpuCount } + // NOTE: a percentage over 100% is valid for POSIX because that means the + // processes is using multiple cores paramData["percentage"] = percentage return paramData, nil } @@ -70,7 +75,7 @@ func (s *cpuacctGroup) getProcStarttime(d *data) (float64, error) { } func (s *cpuacctGroup) getSystemCpuUsage(d *data) (float64, error) { - total := 0.0 + f, err := os.Open("/proc/stat") if err != nil { return 0, err @@ -79,24 +84,27 @@ func (s *cpuacctGroup) getSystemCpuUsage(d *data) (float64, error) { sc := bufio.NewScanner(f) for sc.Scan() { - txt := sc.Text() - if strings.Index(txt, "cpu") == 0 { - parts := strings.Fields(txt) - partsLength := len(parts) - if partsLength != 11 { - return 0.0, fmt.Errorf("Unable to parse cpu usage: expected 11 fields ; received %d", partsLength) + parts := strings.Fields(sc.Text()) + switch parts[0] { + case "cpu": + if len(parts) < 8 { + return 0, fmt.Errorf("invalid number of cpu fields") } - for _, i := range parts[1:10] { - val, err := strconv.ParseFloat(i, 64) + + var total float64 + for _, i := range parts[1:8] { + v, err := strconv.ParseFloat(i, 64) if err != nil { - return 0.0, fmt.Errorf("Unable to convert value to float: %s", err) + return 0.0, fmt.Errorf("Unable to convert value %s to float: %s", i, err) } - total += val + total += v } - break + return total, nil + default: + continue } } - return total, nil + return 0, fmt.Errorf("invalid stat format") } func (s *cpuacctGroup) getCpuUsage(d *data, path string) (float64, error) { From 004cf556e86a03d8961416b3e8a0a476a424b10f Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Mon, 21 Apr 2014 10:26:03 -0700 Subject: [PATCH 14/14] Use cgo to get systems clock ticks for metrics Docker-DCO-1.1-Signed-off-by: Michael Crosby (github: crosbymichael) --- pkg/cgroups/fs/cpuacct.go | 7 +++++-- pkg/system/sysconfig.go | 13 +++++++++++++ pkg/system/sysconfig_nocgo.go | 9 +++++++++ pkg/system/unsupported.go | 6 ++++++ 4 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 pkg/system/sysconfig.go create mode 100644 pkg/system/sysconfig_nocgo.go diff --git a/pkg/cgroups/fs/cpuacct.go b/pkg/cgroups/fs/cpuacct.go index 8c123b79ee..4ea2b1f51b 100644 --- a/pkg/cgroups/fs/cpuacct.go +++ b/pkg/cgroups/fs/cpuacct.go @@ -14,7 +14,10 @@ import ( "github.com/dotcloud/docker/pkg/system" ) -var cpuCount = float64(runtime.NumCPU()) +var ( + cpuCount = float64(runtime.NumCPU()) + clockTicks = float64(system.GetClockTicks()) +) type cpuacctGroup struct { } @@ -58,7 +61,7 @@ func (s *cpuacctGroup) Stats(d *data) (map[string]float64, error) { deltaSystem = lastSystem - startSystem ) if deltaSystem > 0.0 { - percentage = ((deltaProc / deltaSystem) * 100.0) * cpuCount + percentage = ((deltaProc / deltaSystem) * clockTicks) * cpuCount } // NOTE: a percentage over 100% is valid for POSIX because that means the // processes is using multiple cores diff --git a/pkg/system/sysconfig.go b/pkg/system/sysconfig.go new file mode 100644 index 0000000000..dcbe6c9cdd --- /dev/null +++ b/pkg/system/sysconfig.go @@ -0,0 +1,13 @@ +// +build linux,cgo + +package system + +/* +#include +int get_hz(void) { return sysconf(_SC_CLK_TCK); } +*/ +import "C" + +func GetClockTicks() int { + return int(C.get_hz()) +} diff --git a/pkg/system/sysconfig_nocgo.go b/pkg/system/sysconfig_nocgo.go new file mode 100644 index 0000000000..7ca3488154 --- /dev/null +++ b/pkg/system/sysconfig_nocgo.go @@ -0,0 +1,9 @@ +// +build linux,!cgo + +package system + +func GetClockTicks() int { + // when we cannot call out to C to get the sysconf it is fairly safe to + // just return 100 + return 100 +} diff --git a/pkg/system/unsupported.go b/pkg/system/unsupported.go index c52a1e5d00..4ae2a488aa 100644 --- a/pkg/system/unsupported.go +++ b/pkg/system/unsupported.go @@ -17,3 +17,9 @@ func UsetCloseOnExec(fd uintptr) error { func Gettid() int { return 0 } + +func GetClockTicks() int { + // when we cannot call out to C to get the sysconf it is fairly safe to + // just return 100 + return 100 +}