Change 'docker run' exit codes to distinguish docker/contained errors

The purpose of this PR is for users to distinguish Docker errors from
contained command errors.
This PR modifies 'docker run' exit codes to follow the chroot standard
for exit codes.
Exit status:
125 if 'docker run' itself fails
126 if contained command cannot be invoked
127 if contained command cannot be found
the exit status otherwise

Signed-off-by: Sally O'Malley <somalley@redhat.com>

api/client/run.go

@@ -6,9 +6,11 @@ import (
 	"net/url"
 	"os"
 	"runtime"
+	"strings"
 
 	"github.com/Sirupsen/logrus"
 	Cli "github.com/docker/docker/cli"
+	derr "github.com/docker/docker/errors"
 	"github.com/docker/docker/opts"
 	"github.com/docker/docker/pkg/promise"
 	"github.com/docker/docker/pkg/signal"
@@ -36,6 +38,29 @@ func (cid *cidFile) Write(id string) error {
 	return nil
 }
 
+// if container start fails with 'command not found' error, return 127
+// if container start fails with 'command cannot be invoked' error, return 126
+// return 125 for generic docker daemon failures
+func runStartContainerErr(err error) error {
+	trimmedErr := strings.Trim(err.Error(), "Error response from daemon: ")
+	statusError := Cli.StatusError{}
+	derrCmdNotFound := derr.ErrorCodeCmdNotFound.Message()
+	derrCouldNotInvoke := derr.ErrorCodeCmdCouldNotBeInvoked.Message()
+	derrNoSuchImage := derr.ErrorCodeNoSuchImageHash.Message()
+	derrNoSuchImageTag := derr.ErrorCodeNoSuchImageTag.Message()
+	switch trimmedErr {
+	case derrCmdNotFound:
+		statusError = Cli.StatusError{StatusCode: 127}
+	case derrCouldNotInvoke:
+		statusError = Cli.StatusError{StatusCode: 126}
+	case derrNoSuchImage, derrNoSuchImageTag:
+		statusError = Cli.StatusError{StatusCode: 125}
+	default:
+		statusError = Cli.StatusError{StatusCode: 125}
+	}
+	return statusError
+}
+
 // CmdRun runs a command in a new container.
 //
 // Usage: docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
@@ -60,7 +85,7 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 	// just in case the Parse does not exit
 	if err != nil {
 		cmd.ReportError(err.Error(), true)
-		os.Exit(1)
+		os.Exit(125)
 	}
 
 	if len(hostConfig.DNS) > 0 {
@@ -115,7 +140,8 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 
 	createResponse, err := cli.createContainer(config, hostConfig, hostConfig.ContainerIDFile, *flName)
 	if err != nil {
-		return err
+		cmd.ReportError(err.Error(), true)
+		return runStartContainerErr(err)
 	}
 	if sigProxy {
 		sigc := cli.forwardAllSignals(createResponse.ID)
@@ -199,8 +225,9 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 	}()
 
 	//start the container
-	if _, _, err = readBody(cli.call("POST", "/containers/"+createResponse.ID+"/start", nil, nil)); err != nil {
-		return err
+	if _, _, err := readBody(cli.call("POST", "/containers/"+createResponse.ID+"/start", nil, nil)); err != nil {
+		cmd.ReportError(err.Error(), false)
+		return runStartContainerErr(err)
 	}
 
 	if (config.AttachStdin || config.AttachStdout || config.AttachStderr) && config.Tty && cli.isTerminalOut {
@@ -230,7 +257,7 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 		// Autoremove: wait for the container to finish, retrieve
 		// the exit code and remove the container
 		if _, _, err := readBody(cli.call("POST", "/containers/"+createResponse.ID+"/wait", nil, nil)); err != nil {
-			return err
+			return runStartContainerErr(err)
 		}
 		if _, status, err = getExitCode(cli, createResponse.ID); err != nil {
 			return err

daemon/monitor.go

@@ -3,13 +3,17 @@ package daemon
 import (
 	"io"
 	"os/exec"
+	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/execdriver"
+	derr "github.com/docker/docker/errors"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/runconfig"
+	"github.com/docker/docker/utils"
 )
 
 const (
@@ -163,11 +167,30 @@ func (m *containerMonitor) Start() error {
 		if exitStatus, err = m.supervisor.Run(m.container, pipes, m.callback); err != nil {
 			// if we receive an internal error from the initial start of a container then lets
 			// return it instead of entering the restart loop
+			// set to 127 for contained cmd not found/does not exist)
+			if strings.Contains(err.Error(), "executable file not found") ||
+				strings.Contains(err.Error(), "no such file or directory") ||
+				strings.Contains(err.Error(), "system cannot find the file specified") {
+				if m.container.RestartCount == 0 {
+					m.container.ExitCode = 127
+					m.resetContainer(false)
+					return derr.ErrorCodeCmdNotFound
+				}
+			}
+			// set to 126 for contained cmd can't be invoked errors
+			if strings.Contains(err.Error(), syscall.EACCES.Error()) {
+				if m.container.RestartCount == 0 {
+					m.container.ExitCode = 126
+					m.resetContainer(false)
+					return derr.ErrorCodeCmdCouldNotBeInvoked
+				}
+			}
+
 			if m.container.RestartCount == 0 {
 				m.container.ExitCode = -1
 				m.resetContainer(false)
 
-				return err
+				return derr.ErrorCodeCantStart.WithArgs(utils.GetErrorMessage(err))
 			}
 
 			logrus.Errorf("Error running container: %s", err)

daemon/start.go

@@ -7,7 +7,6 @@ import (
 	derr "github.com/docker/docker/errors"
 	"github.com/docker/docker/pkg/promise"
 	"github.com/docker/docker/runconfig"
-	"github.com/docker/docker/utils"
 )
 
 // ContainerStart starts a container.
@@ -47,7 +46,7 @@ func (daemon *Daemon) ContainerStart(name string, hostConfig *runconfig.HostConf
 	}
 
 	if err := daemon.containerStart(container); err != nil {
-		return derr.ErrorCodeCantStart.WithArgs(name, utils.GetErrorMessage(err))
+		return err
 	}
 
 	return nil

docs/reference/run.md

@@ -518,6 +518,38 @@ non-zero exit status more than 10 times in a row Docker will abort trying to
 restart the container. Providing a maximum restart limit is only valid for the
 **on-failure** policy.
 
+## Exit Status
+
+The exit code from `docker run` gives information about why the container
+failed to run or why it exited.  When `docker run` exits with a non-zero code,
+the exit codes follow the `chroot` standard, see below:
+
+**_125_** if the error is with Docker daemon **_itself_** 
+
+    $ docker run --foo busybox; echo $?
+    # flag provided but not defined: --foo
+      See 'docker run --help'.
+      125
+
+**_126_** if the **_contained command_** cannot be invoked
+
+    $ docker run busybox /etc; echo $?
+    # exec: "/etc": permission denied
+      docker: Error response from daemon: Contained command could not be invoked
+      126
+
+**_127_** if the **_contained command_** cannot be found
+
+    $ docker run busybox foo; echo $?
+    # exec: "foo": executable file not found in $PATH
+      docker: Error response from daemon: Contained command not found or does not exist
+      127
+
+**_Exit code_** of **_contained command_** otherwise
+
+    $ docker run busybox /bin/sh -c 'exit 3' 
+    # 3
+
 ## Clean up (--rm)
 
 By default a container's file system persists even after the container

errors/daemon.go

@@ -599,15 +599,6 @@ var (
 		HTTPStatusCode: http.StatusInternalServerError,
 	})
 
-	// ErrorCodeCantStart is generated when an error occurred while
-	// trying to start a container.
-	ErrorCodeCantStart = errcode.Register(errGroup, errcode.ErrorDescriptor{
-		Value:          "CANTSTART",
-		Message:        "Cannot start container %s: %s",
-		Description:    "There was an error while trying to start a container",
-		HTTPStatusCode: http.StatusInternalServerError,
-	})
-
 	// ErrorCodeCantRestart is generated when an error occurred while
 	// trying to restart a container.
 	ErrorCodeCantRestart = errcode.Register(errGroup, errcode.ErrorDescriptor{
@@ -930,4 +921,31 @@ var (
 		Description:    "An attempt to create a volume using a driver but the volume already exists with a different driver",
 		HTTPStatusCode: http.StatusInternalServerError,
 	})
+
+	// ErrorCodeCmdNotFound is generated when contained cmd can't start,
+	// contained command not found error, exit code 127
+	ErrorCodeCmdNotFound = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:          "CMDNOTFOUND",
+		Message:        "Contained command not found or does not exist.",
+		Description:    "Command could not be found, command does not exist",
+		HTTPStatusCode: http.StatusInternalServerError,
+	})
+
+	// ErrorCodeCmdCouldNotBeInvoked is generated when contained cmd can't start,
+	// contained command permission denied error, exit code 126
+	ErrorCodeCmdCouldNotBeInvoked = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:          "CMDCOULDNOTBEINVOKED",
+		Message:        "Contained command could not be invoked.",
+		Description:    "Permission denied, cannot invoke command",
+		HTTPStatusCode: http.StatusInternalServerError,
+	})
+
+	// ErrorCodeCantStart is generated when contained cmd can't start,
+	// for any reason other than above 2 errors
+	ErrorCodeCantStart = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:          "CANTSTART",
+		Message:        "Cannot start container %s: %s",
+		Description:    "There was an error while trying to start a container",
+		HTTPStatusCode: http.StatusInternalServerError,
+	})
 )

integration-cli/docker_cli_run_test.go

@@ -1645,9 +1645,9 @@ func (s *DockerSuite) TestRunWorkdirExistsAndIsFile(c *check.C) {
 		expected = "The directory name is invalid"
 	}
 
-	out, exit, err := dockerCmdWithError("run", "-w", existingFile, "busybox")
-	if !(err != nil && exit == 1 && strings.Contains(out, expected)) {
-		c.Fatalf("Docker must complains about making dir, but we got out: %s, exit: %d, err: %s", out, exit, err)
+	out, exitCode, err := dockerCmdWithError("run", "-w", existingFile, "busybox")
+	if !(err != nil && exitCode == 125 && strings.Contains(out, expected)) {
+		c.Fatalf("Docker must complains about making dir with exitCode 125 but we got out: %s, exitCode: %d", out, exitCode)
 	}
 }
 
@@ -3746,17 +3746,72 @@ func (s *DockerSuite) TestRunStdinBlockedAfterContainerExit(c *check.C) {
 func (s *DockerSuite) TestRunWrongCpusetCpusFlagValue(c *check.C) {
 	// TODO Windows: This needs validation (error out) in the daemon.
 	testRequires(c, DaemonIsLinux)
-	out, _, err := dockerCmdWithError("run", "--cpuset-cpus", "1-10,11--", "busybox", "true")
+	out, exitCode, err := dockerCmdWithError("run", "--cpuset-cpus", "1-10,11--", "busybox", "true")
 	c.Assert(err, check.NotNil)
 	expected := "Error response from daemon: Invalid value 1-10,11-- for cpuset cpus.\n"
-	c.Assert(out, check.Equals, expected, check.Commentf("Expected output to contain %q, got %q", expected, out))
+	if !(strings.Contains(out, expected) || exitCode == 125) {
+		c.Fatalf("Expected output to contain %q with exitCode 125, got out: %q exitCode: %v", expected, out, exitCode)
+	}
 }
 
 func (s *DockerSuite) TestRunWrongCpusetMemsFlagValue(c *check.C) {
 	// TODO Windows: This needs validation (error out) in the daemon.
 	testRequires(c, DaemonIsLinux)
-	out, _, err := dockerCmdWithError("run", "--cpuset-mems", "1-42--", "busybox", "true")
+	out, exitCode, err := dockerCmdWithError("run", "--cpuset-mems", "1-42--", "busybox", "true")
 	c.Assert(err, check.NotNil)
 	expected := "Error response from daemon: Invalid value 1-42-- for cpuset mems.\n"
-	c.Assert(out, check.Equals, expected, check.Commentf("Expected output to contain %q, got %q", expected, out))
+	if !(strings.Contains(out, expected) || exitCode == 125) {
+		c.Fatalf("Expected output to contain %q with exitCode 125, got out: %q exitCode: %v", expected, out, exitCode)
+	}
+}
+
+// TestRunNonExecutableCmd checks that 'docker run busybox foo' exits with error code 127'
+func (s *DockerSuite) TestRunNonExecutableCmd(c *check.C) {
+	name := "testNonExecutableCmd"
+	runCmd := exec.Command(dockerBinary, "run", "--name", name, "busybox", "foo")
+	_, exit, _ := runCommandWithOutput(runCmd)
+	stateExitCode := findContainerExitCode(c, name)
+	if !(exit == 127 && strings.Contains(stateExitCode, "127")) {
+		c.Fatalf("Run non-executable command should have errored with exit code 127, but we got exit: %d, State.ExitCode: %s", exit, stateExitCode)
+	}
+}
+
+// TestRunNonExistingCmd checks that 'docker run busybox /bin/foo' exits with code 127.
+func (s *DockerSuite) TestRunNonExistingCmd(c *check.C) {
+	name := "testNonExistingCmd"
+	runCmd := exec.Command(dockerBinary, "run", "--name", name, "busybox", "/bin/foo")
+	_, exit, _ := runCommandWithOutput(runCmd)
+	stateExitCode := findContainerExitCode(c, name)
+	if !(exit == 127 && strings.Contains(stateExitCode, "127")) {
+		c.Fatalf("Run non-existing command should have errored with exit code 127, but we got exit: %d, State.ExitCode: %s", exit, stateExitCode)
+	}
+}
+
+// TestCmdCannotBeInvoked checks that 'docker run busybox /etc' exits with 126.
+func (s *DockerSuite) TestCmdCannotBeInvoked(c *check.C) {
+	name := "testCmdCannotBeInvoked"
+	runCmd := exec.Command(dockerBinary, "run", "--name", name, "busybox", "/etc")
+	_, exit, _ := runCommandWithOutput(runCmd)
+	stateExitCode := findContainerExitCode(c, name)
+	if !(exit == 126 && strings.Contains(stateExitCode, "126")) {
+		c.Fatalf("Run cmd that cannot be invoked should have errored with code 126, but we got exit: %d, State.ExitCode: %s", exit, stateExitCode)
+	}
+}
+
+// TestRunNonExistingImage checks that 'docker run foo' exits with error msg 125 and contains  'Unable to find image'
+func (s *DockerSuite) TestRunNonExistingImage(c *check.C) {
+	runCmd := exec.Command(dockerBinary, "run", "foo")
+	out, exit, err := runCommandWithOutput(runCmd)
+	if !(err != nil && exit == 125 && strings.Contains(out, "Unable to find image")) {
+		c.Fatalf("Run non-existing image should have errored with 'Unable to find image' code 125, but we got out: %s, exit: %d, err: %s", out, exit, err)
+	}
+}
+
+// TestDockerFails checks that 'docker run -foo busybox' exits with 125 to signal docker run failed
+func (s *DockerSuite) TestDockerFails(c *check.C) {
+	runCmd := exec.Command(dockerBinary, "run", "-foo", "busybox")
+	out, exit, err := runCommandWithOutput(runCmd)
+	if !(err != nil && exit == 125) {
+		c.Fatalf("Docker run with flag not defined should exit with 125, but we got out: %s, exit: %d, err: %s", out, exit, err)
+	}
 }

integration-cli/docker_cli_run_unix_test.go

@@ -397,8 +397,10 @@ func (s *DockerSuite) TestRunInvalidCpusetCpusFlagValue(c *check.C) {
 	}
 	out, _, err := dockerCmdWithError("run", "--cpuset-cpus", strconv.Itoa(invalid), "busybox", "true")
 	c.Assert(err, check.NotNil)
-	expected := fmt.Sprintf("Error response from daemon: Requested CPUs are not available - requested %s, available: %s.\n", strconv.Itoa(invalid), sysInfo.Cpus)
-	c.Assert(out, check.Equals, expected, check.Commentf("Expected output to contain %q, got %q", expected, out))
+	expected := fmt.Sprintf("Error response from daemon: Requested CPUs are not available - requested %s, available: %s", strconv.Itoa(invalid), sysInfo.Cpus)
+	if !(strings.Contains(out, expected)) {
+		c.Fatalf("Expected output to contain %q, got %q", expected, out)
+	}
 }
 
 func (s *DockerSuite) TestRunInvalidCpusetMemsFlagValue(c *check.C) {
@@ -416,8 +418,10 @@ func (s *DockerSuite) TestRunInvalidCpusetMemsFlagValue(c *check.C) {
 	}
 	out, _, err := dockerCmdWithError("run", "--cpuset-mems", strconv.Itoa(invalid), "busybox", "true")
 	c.Assert(err, check.NotNil)
-	expected := fmt.Sprintf("Error response from daemon: Requested memory nodes are not available - requested %s, available: %s.\n", strconv.Itoa(invalid), sysInfo.Mems)
-	c.Assert(out, check.Equals, expected, check.Commentf("Expected output to contain %q, got %q", expected, out))
+	expected := fmt.Sprintf("Error response from daemon: Requested memory nodes are not available - requested %s, available: %s", strconv.Itoa(invalid), sysInfo.Mems)
+	if !(strings.Contains(out, expected)) {
+		c.Fatalf("Expected output to contain %q, got %q", expected, out)
+	}
 }
 
 func (s *DockerSuite) TestRunInvalidCPUShares(c *check.C) {

integration-cli/docker_cli_start_test.go

@@ -129,11 +129,15 @@ func (s *DockerSuite) TestStartMultipleContainers(c *check.C) {
 
 	// start all the three containers, container `child_first` start first which should be failed
 	// container 'parent' start second and then start container 'child_second'
+	expOut := "Cannot link to a non running container"
+	expErr := "failed to start containers: [child_first]"
 	out, _, err = dockerCmdWithError("start", "child_first", "parent", "child_second")
 	// err shouldn't be nil because start will fail
 	c.Assert(err, checker.NotNil, check.Commentf("out: %s", out))
 	// output does not correspond to what was expected
-	c.Assert(out, checker.Contains, "Cannot start container child_first")
+	if !(strings.Contains(out, expOut) || strings.Contains(err.Error(), expErr)) {
+		c.Fatalf("Expected out: %v with err: %v  but got out: %v with err: %v", expOut, expErr, out, err)
+	}
 
 	for container, expected := range map[string]string{"parent": "true", "child_first": "false", "child_second": "true"} {
 		out, err := inspectField(container, "State.Running")

integration-cli/docker_utils.go

@@ -815,6 +815,17 @@ func dockerCmdInDirWithTimeout(timeout time.Duration, path string, args ...strin
 	return integration.DockerCmdInDirWithTimeout(dockerBinary, timeout, path, args...)
 }
 
+// find the State.ExitCode in container metadata
+func findContainerExitCode(c *check.C, name string, vargs ...string) string {
+	args := append(vargs, "inspect", "--format='{{ .State.ExitCode }} {{ .State.Error }}'", name)
+	cmd := exec.Command(dockerBinary, args...)
+	out, _, err := runCommandWithOutput(cmd)
+	if err != nil {
+		c.Fatal(err, out)
+	}
+	return out
+}
+
 func findContainerIP(c *check.C, id string, network string) string {
 	out, _ := dockerCmd(c, "inspect", fmt.Sprintf("--format='{{ .NetworkSettings.Networks.%s.IPAddress }}'", network), id)
 	return strings.Trim(out, " \r\n'")

man/docker-run.1.md

@@ -508,6 +508,38 @@ running binaries within a container is the root directory (/). The developer can
 set a different default with the Dockerfile WORKDIR instruction. The operator
 can override the working directory by using the **-w** option.
 
+# Exit Status
+
+The exit code from `docker run` gives information about why the container
+failed to run or why it exited.  When `docker run` exits with a non-zero code,
+the exit codes follow the `chroot` standard, see below:
+
+**_125_** if the error is with Docker daemon **_itself_** 
+
+    $ docker run --foo busybox; echo $?
+    # flag provided but not defined: --foo
+      See 'docker run --help'.
+      125
+
+**_126_** if the **_contained command_** cannot be invoked
+
+    $ docker run busybox /etc; echo $?
+    # exec: "/etc": permission denied
+      docker: Error response from daemon: Contained command could not be invoked
+      126
+
+**_127_** if the **_contained command_** cannot be found
+
+    $ docker run busybox foo; echo $?
+    # exec: "foo": executable file not found in $PATH
+      docker: Error response from daemon: Contained command not found or does not exist
+      127
+
+**_Exit code_** of **_contained command_** otherwise 
+    
+    $ docker run busybox /bin/sh -c 'exit 3' 
+    # 3
+
 # EXAMPLES
 
 ## Exposing log messages from the container to the host's log
@@ -732,3 +764,4 @@ April 2014, Originally compiled by William Henry (whenry at redhat dot com)
 based on docker.com source material and internal work.
 June 2014, updated by Sven Dowideit <SvenDowideit@home.org.au>
 July 2014, updated by Sven Dowideit <SvenDowideit@home.org.au>
+November 2015, updated by Sally O'Malley <somalley@redhat.com>

pkg/mflag/flag.go

@@ -1102,7 +1102,7 @@ func (fs *FlagSet) Parse(arguments []string) error {
 		case ContinueOnError:
 			return err
 		case ExitOnError:
-			os.Exit(2)
+			os.Exit(125)
 		case PanicOnError:
 			panic(err)
 		}