Set OOMKilled state on any OOM event
This restores the behavior that existed prior to #16235 for setting OOMKilled, while retaining the additional benefits it introduced around emitting the oom event. This also adds a test for the most obvious OOM cases which would have caught this regression. Fixes #18510 Signed-off-by: Euan <euank@amazon.com>
This commit is contained in:
parent
adb19755e1
commit
0b5131444d
|
@ -304,8 +304,7 @@ func (m *containerMonitor) shouldRestart(exitCode int) bool {
|
||||||
// received ack from the execution drivers
|
// received ack from the execution drivers
|
||||||
func (m *containerMonitor) callback(processConfig *execdriver.ProcessConfig, pid int, chOOM <-chan struct{}) error {
|
func (m *containerMonitor) callback(processConfig *execdriver.ProcessConfig, pid int, chOOM <-chan struct{}) error {
|
||||||
go func() {
|
go func() {
|
||||||
_, ok := <-chOOM
|
for range chOOM {
|
||||||
if ok {
|
|
||||||
m.logEvent("oom")
|
m.logEvent("oom")
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
|
@ -171,7 +171,10 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd
|
||||||
return execdriver.ExitStatus{ExitCode: -1}, err
|
return execdriver.ExitStatus{ExitCode: -1}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 'oom' is used to emit 'oom' events to the eventstream, 'oomKilled' is used
|
||||||
|
// to set the 'OOMKilled' flag in state
|
||||||
oom := notifyOnOOM(cont)
|
oom := notifyOnOOM(cont)
|
||||||
|
oomKilled := notifyOnOOM(cont)
|
||||||
if hooks.Start != nil {
|
if hooks.Start != nil {
|
||||||
pid, err := p.Pid()
|
pid, err := p.Pid()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -198,7 +201,21 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, hooks execd
|
||||||
}
|
}
|
||||||
cont.Destroy()
|
cont.Destroy()
|
||||||
destroyed = true
|
destroyed = true
|
||||||
_, oomKill := <-oom
|
// oomKilled will have an oom event if any process within the container was
|
||||||
|
// OOM killed at any time, not only if the init process OOMed.
|
||||||
|
//
|
||||||
|
// Perhaps we only want the OOMKilled flag to be set if the OOM
|
||||||
|
// resulted in a container death, but there isn't a good way to do this
|
||||||
|
// because the kernel's cgroup oom notification does not provide information
|
||||||
|
// such as the PID. This could be heuristically done by checking that the OOM
|
||||||
|
// happened within some very small time slice for the container dying (and
|
||||||
|
// optionally exit-code 137), but I don't think the cgroup oom notification
|
||||||
|
// can be used to reliably determine this
|
||||||
|
//
|
||||||
|
// Even if there were multiple OOMs, it's sufficient to read one value
|
||||||
|
// because libcontainer's oom notify will discard the channel after the
|
||||||
|
// cgroup is destroyed
|
||||||
|
_, oomKill := <-oomKilled
|
||||||
return execdriver.ExitStatus{ExitCode: utils.ExitStatus(ps.Sys().(syscall.WaitStatus)), OOMKilled: oomKill}, nil
|
return execdriver.ExitStatus{ExitCode: utils.ExitStatus(ps.Sys().(syscall.WaitStatus)), OOMKilled: oomKill}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,32 @@
|
||||||
|
// +build !windows
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/docker/docker/pkg/integration/checker"
|
||||||
|
"github.com/go-check/check"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *DockerSuite) TestInspectOomKilledTrue(c *check.C) {
|
||||||
|
testRequires(c, DaemonIsLinux, memoryLimitSupport)
|
||||||
|
|
||||||
|
name := "testoomkilled"
|
||||||
|
_, exitCode, _ := dockerCmdWithError("run", "--name", name, "-m", "10MB", "busybox", "sh", "-c", "x=a; while true; do x=$x$x$x$x; done")
|
||||||
|
|
||||||
|
c.Assert(exitCode, checker.Equals, 137, check.Commentf("OOM exit should be 137"))
|
||||||
|
|
||||||
|
oomKilled, err := inspectField(name, "State.OOMKilled")
|
||||||
|
c.Assert(oomKilled, checker.Equals, "true")
|
||||||
|
c.Assert(err, checker.IsNil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *DockerSuite) TestInspectOomKilledFalse(c *check.C) {
|
||||||
|
testRequires(c, DaemonIsLinux, memoryLimitSupport)
|
||||||
|
|
||||||
|
name := "testoomkilled"
|
||||||
|
dockerCmd(c, "run", "--name", name, "-m", "10MB", "busybox", "sh", "-c", "echo hello world")
|
||||||
|
|
||||||
|
oomKilled, err := inspectField(name, "State.OOMKilled")
|
||||||
|
c.Assert(oomKilled, checker.Equals, "false")
|
||||||
|
c.Assert(err, checker.IsNil)
|
||||||
|
}
|
Loading…
Reference in New Issue