Merge pull request #10498 from ashahab-altiscale/9875-lxc-stats

Implements stats for lxc driver

daemon/execdriver/driver.go

@@ -2,13 +2,14 @@ package execdriver
 
 import (
 	"errors"
+	"github.com/docker/docker/daemon/execdriver/native/template"
+	"github.com/docker/libcontainer"
+	"github.com/docker/libcontainer/devices"
 	"io"
 	"os"
 	"os/exec"
+	"strings"
 	"time"
-
-	"github.com/docker/libcontainer"
-	"github.com/docker/libcontainer/devices"
 )
 
 // Context is a generic key value pair that allows
@@ -156,3 +157,71 @@ type Command struct {
 	LxcConfig          []string          `json:"lxc_config"`
 	AppArmorProfile    string            `json:"apparmor_profile"`
 }
+
+func InitContainer(c *Command) *libcontainer.Config {
+	container := template.New()
+
+	container.Hostname = getEnv("HOSTNAME", c.ProcessConfig.Env)
+	container.Tty = c.ProcessConfig.Tty
+	container.User = c.ProcessConfig.User
+	container.WorkingDir = c.WorkingDir
+	container.Env = c.ProcessConfig.Env
+	container.Cgroups.Name = c.ID
+	container.Cgroups.AllowedDevices = c.AllowedDevices
+	container.MountConfig.DeviceNodes = c.AutoCreatedDevices
+	container.RootFs = c.Rootfs
+	container.MountConfig.ReadonlyFs = c.ReadonlyRootfs
+
+	// check to see if we are running in ramdisk to disable pivot root
+	container.MountConfig.NoPivotRoot = os.Getenv("DOCKER_RAMDISK") != ""
+	container.RestrictSys = true
+	return container
+}
+
+func getEnv(key string, env []string) string {
+	for _, pair := range env {
+		parts := strings.Split(pair, "=")
+		if parts[0] == key {
+			return parts[1]
+		}
+	}
+	return ""
+}
+
+func SetupCgroups(container *libcontainer.Config, c *Command) error {
+	if c.Resources != nil {
+		container.Cgroups.CpuShares = c.Resources.CpuShares
+		container.Cgroups.Memory = c.Resources.Memory
+		container.Cgroups.MemoryReservation = c.Resources.Memory
+		container.Cgroups.MemorySwap = c.Resources.MemorySwap
+		container.Cgroups.CpusetCpus = c.Resources.Cpuset
+	}
+
+	return nil
+}
+
+func Stats(stateFile string, containerMemoryLimit int64, machineMemory int64) (*ResourceStats, error) {
+	state, err := libcontainer.GetState(stateFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, ErrNotRunning
+		}
+		return nil, err
+	}
+	now := time.Now()
+	stats, err := libcontainer.GetStats(nil, state)
+	if err != nil {
+		return nil, err
+	}
+	// if the container does not have any memory limit specified set the
+	// limit to the machines memory
+	memoryLimit := containerMemoryLimit
+	if memoryLimit == 0 {
+		memoryLimit = machineMemory
+	}
+	return &ResourceStats{
+		Read:           now,
+		ContainerStats: stats,
+		MemoryLimit:    memoryLimit,
+	}, nil
+}

daemon/execdriver/lxc/driver.go

@@ -12,17 +12,19 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 
-	"github.com/kr/pty"
-
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/execdriver"
+	sysinfo "github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/pkg/term"
 	"github.com/docker/docker/utils"
+	"github.com/docker/libcontainer"
 	"github.com/docker/libcontainer/cgroups"
 	"github.com/docker/libcontainer/mount/nodes"
+	"github.com/kr/pty"
 )
 
 const DriverName = "lxc"
@@ -30,10 +32,18 @@ const DriverName = "lxc"
 var ErrExec = errors.New("Unsupported: Exec is not supported by the lxc driver")
 
 type driver struct {
-	root       string // root path for the driver to use
+	root             string // root path for the driver to use
-	initPath   string
+	initPath         string
-	apparmor   bool
+	apparmor         bool
-	sharedRoot bool
+	sharedRoot       bool
+	activeContainers map[string]*activeContainer
+	machineMemory    int64
+	sync.Mutex
+}
+
+type activeContainer struct {
+	container *libcontainer.Config
+	cmd       *exec.Cmd
 }
 
 func NewDriver(root, initPath string, apparmor bool) (*driver, error) {
@@ -41,12 +51,17 @@ func NewDriver(root, initPath string, apparmor bool) (*driver, error) {
 	if err := linkLxcStart(root); err != nil {
 		return nil, err
 	}
-
+	meminfo, err := sysinfo.ReadMemInfo()
+	if err != nil {
+		return nil, err
+	}
 	return &driver{
-		apparmor:   apparmor,
+		apparmor:         apparmor,
-		root:       root,
+		root:             root,
-		initPath:   initPath,
+		initPath:         initPath,
-		sharedRoot: rootIsShared(),
+		sharedRoot:       rootIsShared(),
+		activeContainers: make(map[string]*activeContainer),
+		machineMemory:    meminfo.MemTotal,
 	}, nil
 }
 
@@ -57,8 +72,9 @@ func (d *driver) Name() string {
 
 func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallback execdriver.StartCallback) (execdriver.ExitStatus, error) {
 	var (
-		term execdriver.Terminal
+		term     execdriver.Terminal
-		err  error
+		err      error
+		dataPath = d.containerDir(c.ID)
 	)
 
 	if c.ProcessConfig.Tty {
@@ -67,6 +83,16 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 		term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes)
 	}
 	c.ProcessConfig.Terminal = term
+	container, err := d.createContainer(c)
+	if err != nil {
+		return execdriver.ExitStatus{ExitCode: -1}, err
+	}
+	d.Lock()
+	d.activeContainers[c.ID] = &activeContainer{
+		container: container,
+		cmd:       &c.ProcessConfig.Cmd,
+	}
+	d.Unlock()
 
 	c.Mounts = append(c.Mounts, execdriver.Mount{
 		Source:      d.initPath,
@@ -186,25 +212,89 @@ func (d *driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba
 		close(waitLock)
 	}()
 
-	// Poll lxc for RUNNING status
+	terminate := func(terr error) (execdriver.ExitStatus, error) {
-	pid, err := d.waitForStart(c, waitLock)
-	if err != nil {
 		if c.ProcessConfig.Process != nil {
 			c.ProcessConfig.Process.Kill()
 			c.ProcessConfig.Wait()
 		}
-		return execdriver.ExitStatus{ExitCode: -1}, err
+		return execdriver.ExitStatus{ExitCode: -1}, terr
+	}
+	// Poll lxc for RUNNING status
+	pid, err := d.waitForStart(c, waitLock)
+	if err != nil {
+		return terminate(err)
+	}
+
+	cgroupPaths, err := cgroupPaths(c.ID)
+	if err != nil {
+		return terminate(err)
+	}
+
+	state := &libcontainer.State{
+		InitPid:     pid,
+		CgroupPaths: cgroupPaths,
+	}
+
+	if err := libcontainer.SaveState(dataPath, state); err != nil {
+		return terminate(err)
 	}
 
 	c.ContainerPid = pid
 
 	if startCallback != nil {
+		log.Debugf("Invoking startCallback")
 		startCallback(&c.ProcessConfig, pid)
 	}
+	oomKill := false
+	oomKillNotification, err := libcontainer.NotifyOnOOM(state)
+	if err == nil {
+		_, oomKill = <-oomKillNotification
+		log.Debugf("oomKill error %s waitErr %s", oomKill, waitErr)
+
+	} else {
+		log.Warnf("WARNING: Your kernel does not support OOM notifications: %s", err)
+	}
 
 	<-waitLock
 
-	return execdriver.ExitStatus{ExitCode: getExitCode(c)}, waitErr
+	// check oom error
+	exitCode := getExitCode(c)
+	if oomKill {
+		exitCode = 137
+	}
+	return execdriver.ExitStatus{ExitCode: exitCode, OOMKilled: oomKill}, waitErr
+}
+
+// createContainer populates and configures the container type with the
+// data provided by the execdriver.Command
+func (d *driver) createContainer(c *execdriver.Command) (*libcontainer.Config, error) {
+	container := execdriver.InitContainer(c)
+	if err := execdriver.SetupCgroups(container, c); err != nil {
+		return nil, err
+	}
+	return container, nil
+}
+
+// Return an map of susbystem -> container cgroup
+func cgroupPaths(containerId string) (map[string]string, error) {
+	subsystems, err := cgroups.GetAllSubsystems()
+	if err != nil {
+		return nil, err
+	}
+	log.Debugf("subsystems: %s", subsystems)
+	paths := make(map[string]string)
+	for _, subsystem := range subsystems {
+		cgroupRoot, cgroupDir, err := findCgroupRootAndDir(subsystem)
+		log.Debugf("cgroup path %s %s", cgroupRoot, cgroupDir)
+		if err != nil {
+			//unsupported subystem
+			continue
+		}
+		path := filepath.Join(cgroupRoot, cgroupDir, "lxc", containerId)
+		paths[subsystem] = path
+	}
+
+	return paths, nil
 }
 
 /// Return the exit code of the process
@@ -348,17 +438,25 @@ func (d *driver) Info(id string) execdriver.Info {
 	}
 }
 
+func findCgroupRootAndDir(subsystem string) (string, string, error) {
+	cgroupRoot, err := cgroups.FindCgroupMountpoint(subsystem)
+	if err != nil {
+		return "", "", err
+	}
+
+	cgroupDir, err := cgroups.GetThisCgroupDir(subsystem)
+	if err != nil {
+		return "", "", err
+	}
+	return cgroupRoot, cgroupDir, nil
+}
+
 func (d *driver) GetPidsForContainer(id string) ([]int, error) {
 	pids := []int{}
 
 	// cpu is chosen because it is the only non optional subsystem in cgroups
 	subsystem := "cpu"
-	cgroupRoot, err := cgroups.FindCgroupMountpoint(subsystem)
+	cgroupRoot, cgroupDir, err := findCgroupRootAndDir(subsystem)
-	if err != nil {
-		return pids, err
-	}
-
-	cgroupDir, err := cgroups.GetThisCgroupDir(subsystem)
 	if err != nil {
 		return pids, err
 	}
@@ -418,8 +516,12 @@ func rootIsShared() bool {
 	return true
 }
 
+func (d *driver) containerDir(containerId string) string {
+	return path.Join(d.root, "containers", containerId)
+}
+
 func (d *driver) generateLXCConfig(c *execdriver.Command) (string, error) {
-	root := path.Join(d.root, "containers", c.ID, "config.lxc")
+	root := path.Join(d.containerDir(c.ID), "config.lxc")
 
 	fo, err := os.Create(root)
 	if err != nil {
@@ -537,6 +639,5 @@ func (d *driver) Exec(c *execdriver.Command, processConfig *execdriver.ProcessCo
 }
 
 func (d *driver) Stats(id string) (*execdriver.ResourceStats, error) {
-	return nil, fmt.Errorf("container stats are not supported with LXC")
+	return execdriver.Stats(d.containerDir(id), d.activeContainers[id].container.Cgroups.Memory, d.machineMemory)
-
 }

daemon/execdriver/native/create.go

@@ -4,12 +4,10 @@ package native
 
 import (
 	"fmt"
-	"os"
 	"os/exec"
 	"path/filepath"
 
 	"github.com/docker/docker/daemon/execdriver"
-	"github.com/docker/docker/daemon/execdriver/native/template"
 	"github.com/docker/libcontainer"
 	"github.com/docker/libcontainer/apparmor"
 	"github.com/docker/libcontainer/devices"
@@ -20,22 +18,7 @@ import (
 // createContainer populates and configures the container type with the
 // data provided by the execdriver.Command
 func (d *driver) createContainer(c *execdriver.Command) (*libcontainer.Config, error) {
-	container := template.New()
+	container := execdriver.InitContainer(c)
-
-	container.Hostname = getEnv("HOSTNAME", c.ProcessConfig.Env)
-	container.Tty = c.ProcessConfig.Tty
-	container.User = c.ProcessConfig.User
-	container.WorkingDir = c.WorkingDir
-	container.Env = c.ProcessConfig.Env
-	container.Cgroups.Name = c.ID
-	container.Cgroups.AllowedDevices = c.AllowedDevices
-	container.MountConfig.DeviceNodes = c.AutoCreatedDevices
-	container.RootFs = c.Rootfs
-	container.MountConfig.ReadonlyFs = c.ReadonlyRootfs
-
-	// check to see if we are running in ramdisk to disable pivot root
-	container.MountConfig.NoPivotRoot = os.Getenv("DOCKER_RAMDISK") != ""
-	container.RestrictSys = true
 
 	if err := d.createIpc(container, c); err != nil {
 		return nil, err
@@ -63,7 +46,7 @@ func (d *driver) createContainer(c *execdriver.Command) (*libcontainer.Config, e
 		container.AppArmorProfile = c.AppArmorProfile
 	}
 
-	if err := d.setupCgroups(container, c); err != nil {
+	if err := execdriver.SetupCgroups(container, c); err != nil {
 		return nil, err
 	}
 
@@ -189,18 +172,6 @@ func (d *driver) setCapabilities(container *libcontainer.Config, c *execdriver.C
 	return err
 }
 
-func (d *driver) setupCgroups(container *libcontainer.Config, c *execdriver.Command) error {
-	if c.Resources != nil {
-		container.Cgroups.CpuShares = c.Resources.CpuShares
-		container.Cgroups.Memory = c.Resources.Memory
-		container.Cgroups.MemoryReservation = c.Resources.Memory
-		container.Cgroups.MemorySwap = c.Resources.MemorySwap
-		container.Cgroups.CpusetCpus = c.Resources.Cpuset
-	}
-
-	return nil
-}
-
 func (d *driver) setupMounts(container *libcontainer.Config, c *execdriver.Command) error {
 	for _, m := range c.Mounts {
 		container.MountConfig.Mounts = append(container.MountConfig.Mounts, &mount.Mount{

daemon/execdriver/native/driver.go

@@ -11,10 +11,8 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"strings"
 	"sync"
 	"syscall"
-	"time"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/execdriver"
@@ -291,40 +289,7 @@ func (d *driver) Clean(id string) error {
 }
 
 func (d *driver) Stats(id string) (*execdriver.ResourceStats, error) {
-	c := d.activeContainers[id]
+	return execdriver.Stats(filepath.Join(d.root, id), d.activeContainers[id].container.Cgroups.Memory, d.machineMemory)
-	state, err := libcontainer.GetState(filepath.Join(d.root, id))
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, execdriver.ErrNotRunning
-		}
-		return nil, err
-	}
-	now := time.Now()
-	stats, err := libcontainer.GetStats(nil, state)
-	if err != nil {
-		return nil, err
-	}
-	memoryLimit := c.container.Cgroups.Memory
-	// if the container does not have any memory limit specified set the
-	// limit to the machines memory
-	if memoryLimit == 0 {
-		memoryLimit = d.machineMemory
-	}
-	return &execdriver.ResourceStats{
-		Read:           now,
-		ContainerStats: stats,
-		MemoryLimit:    memoryLimit,
-	}, nil
-}
-
-func getEnv(key string, env []string) string {
-	for _, pair := range env {
-		parts := strings.Split(pair, "=")
-		if parts[0] == key {
-			return parts[1]
-		}
-	}
-	return ""
 }
 
 type TtyConsole struct {

docs/man/docker-stats.1.md

@@ -13,8 +13,6 @@ CONTAINER [CONTAINER...]
 
 Display a live stream of one or more containers' resource usage statistics
 
-Note: this functionality currently only works when using the *libcontainer* exec-driver.
-
 # OPTIONS
 **--help**
   Print usage statement

docs/sources/reference/api/docker_remote_api.md

@@ -86,8 +86,6 @@ root filesystem as read only.
 **New!**
 This endpoint returns a live stream of a container's resource usage statistics.
 
-> **Note**: this functionality currently only works when using the *libcontainer* exec-driver.
-
 
 ## v1.16
 

docs/sources/reference/api/docker_remote_api_v1.17.md

@@ -524,8 +524,6 @@ Status Codes:
 
 This endpoint returns a live stream of a container's resource usage statistics.
 
-> **Note**: this functionality currently only works when using the *libcontainer* exec-driver.
-
 **Example request**:
 
         GET /containers/redis1/stats HTTP/1.1

docs/sources/reference/commandline/cli.md

@@ -2037,8 +2037,6 @@ more details on finding shared images from the command line.
 
       --help=false       Print usage
 
-> **Note**: this functionality currently only works when using the *libcontainer* exec-driver.
-
 Running `docker stats` on multiple containers
 
     $ sudo docker stats redis1 redis2

integration-cli/docker_cli_events_test.go

@@ -35,7 +35,7 @@ func TestEventsContainerFailStartDie(t *testing.T) {
 
 	out, _, _ := dockerCmd(t, "images", "-q")
 	image := strings.Split(out, "\n")[0]
-	eventsCmd := exec.Command(dockerBinary, "run", "-d", "--name", "testeventdie", image, "blerg")
+	eventsCmd := exec.Command(dockerBinary, "run", "--name", "testeventdie", image, "blerg")
 	_, _, err := runCommandWithOutput(eventsCmd)
 	if err == nil {
 		t.Fatalf("Container run with command blerg should have failed, but it did not")