Merge pull request #14012 from sallyom/exitCodes

Change 'docker run' exit codes to distinguish docker/contained errors

api/client/run.go

@@ -6,9 +6,11 @@ import (
 	"net/url"
 	"os"
 	"runtime"
+	"strings"
 
 	"github.com/Sirupsen/logrus"
 	Cli "github.com/docker/docker/cli"
+	derr "github.com/docker/docker/errors"
 	"github.com/docker/docker/opts"
 	"github.com/docker/docker/pkg/promise"
 	"github.com/docker/docker/pkg/signal"
@@ -36,6 +38,29 @@ func (cid *cidFile) Write(id string) error {
 	return nil
 }
 
+// if container start fails with 'command not found' error, return 127
+// if container start fails with 'command cannot be invoked' error, return 126
+// return 125 for generic docker daemon failures
+func runStartContainerErr(err error) error {
+	trimmedErr := strings.Trim(err.Error(), "Error response from daemon: ")
+	statusError := Cli.StatusError{}
+	derrCmdNotFound := derr.ErrorCodeCmdNotFound.Message()
+	derrCouldNotInvoke := derr.ErrorCodeCmdCouldNotBeInvoked.Message()
+	derrNoSuchImage := derr.ErrorCodeNoSuchImageHash.Message()
+	derrNoSuchImageTag := derr.ErrorCodeNoSuchImageTag.Message()
+	switch trimmedErr {
+	case derrCmdNotFound:
+		statusError = Cli.StatusError{StatusCode: 127}
+	case derrCouldNotInvoke:
+		statusError = Cli.StatusError{StatusCode: 126}
+	case derrNoSuchImage, derrNoSuchImageTag:
+		statusError = Cli.StatusError{StatusCode: 125}
+	default:
+		statusError = Cli.StatusError{StatusCode: 125}
+	}
+	return statusError
+}
+
 // CmdRun runs a command in a new container.
 //
 // Usage: docker run [OPTIONS] IMAGE [COMMAND] [ARG...]
@@ -60,7 +85,7 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 	// just in case the Parse does not exit
 	if err != nil {
 		cmd.ReportError(err.Error(), true)
-		os.Exit(1)
+		os.Exit(125)
 	}
 
 	if len(hostConfig.DNS) > 0 {
@@ -115,7 +140,8 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 
 	createResponse, err := cli.createContainer(config, hostConfig, hostConfig.ContainerIDFile, *flName)
 	if err != nil {
-		return err
+		cmd.ReportError(err.Error(), true)
+		return runStartContainerErr(err)
 	}
 	if sigProxy {
 		sigc := cli.forwardAllSignals(createResponse.ID)
@@ -199,8 +225,9 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 	}()
 
 	//start the container
-	if _, _, err = readBody(cli.call("POST", "/containers/"+createResponse.ID+"/start", nil, nil)); err != nil {
-		return err
+	if _, _, err := readBody(cli.call("POST", "/containers/"+createResponse.ID+"/start", nil, nil)); err != nil {
+		cmd.ReportError(err.Error(), false)
+		return runStartContainerErr(err)
 	}
 
 	if (config.AttachStdin || config.AttachStdout || config.AttachStderr) && config.Tty && cli.isTerminalOut {
@@ -230,7 +257,7 @@ func (cli *DockerCli) CmdRun(args ...string) error {
 		// Autoremove: wait for the container to finish, retrieve
 		// the exit code and remove the container
 		if _, _, err := readBody(cli.call("POST", "/containers/"+createResponse.ID+"/wait", nil, nil)); err != nil {
-			return err
+			return runStartContainerErr(err)
 		}
 		if _, status, err = getExitCode(cli, createResponse.ID); err != nil {
 			return err

daemon/monitor.go

@@ -3,13 +3,17 @@ package daemon
 import (
 	"io"
 	"os/exec"
+	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/daemon/execdriver"
+	derr "github.com/docker/docker/errors"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/runconfig"
+	"github.com/docker/docker/utils"
 )
 
 const (
@@ -163,11 +167,30 @@ func (m *containerMonitor) Start() error {
 		if exitStatus, err = m.supervisor.Run(m.container, pipes, m.callback); err != nil {
 			// if we receive an internal error from the initial start of a container then lets
 			// return it instead of entering the restart loop
+			// set to 127 for contained cmd not found/does not exist)
+			if strings.Contains(err.Error(), "executable file not found") ||
+				strings.Contains(err.Error(), "no such file or directory") ||
+				strings.Contains(err.Error(), "system cannot find the file specified") {
+				if m.container.RestartCount == 0 {
+					m.container.ExitCode = 127
+					m.resetContainer(false)
+					return derr.ErrorCodeCmdNotFound
+				}
+			}
+			// set to 126 for contained cmd can't be invoked errors
+			if strings.Contains(err.Error(), syscall.EACCES.Error()) {
+				if m.container.RestartCount == 0 {
+					m.container.ExitCode = 126
+					m.resetContainer(false)
+					return derr.ErrorCodeCmdCouldNotBeInvoked
+				}
+			}
+
 			if m.container.RestartCount == 0 {
 				m.container.ExitCode = -1
 				m.resetContainer(false)
 
-				return err
+				return derr.ErrorCodeCantStart.WithArgs(utils.GetErrorMessage(err))
 			}
 
 			logrus.Errorf("Error running container: %s", err)

daemon/start.go

@@ -7,7 +7,6 @@ import (
 	derr "github.com/docker/docker/errors"
 	"github.com/docker/docker/pkg/promise"
 	"github.com/docker/docker/runconfig"
-	"github.com/docker/docker/utils"
 )
 
 // ContainerStart starts a container.
@@ -47,7 +46,7 @@ func (daemon *Daemon) ContainerStart(name string, hostConfig *runconfig.HostConf
 	}
 
 	if err := daemon.containerStart(container); err != nil {
-		return derr.ErrorCodeCantStart.WithArgs(name, utils.GetErrorMessage(err))
+		return err
 	}
 
 	return nil

docs/reference/run.md

@@ -518,6 +518,38 @@ non-zero exit status more than 10 times in a row Docker will abort trying to
 restart the container. Providing a maximum restart limit is only valid for the
 **on-failure** policy.
 
+## Exit Status
+
+The exit code from `docker run` gives information about why the container
+failed to run or why it exited.  When `docker run` exits with a non-zero code,
+the exit codes follow the `chroot` standard, see below:
+
+**_125_** if the error is with Docker daemon **_itself_** 
+
+    $ docker run --foo busybox; echo $?
+    # flag provided but not defined: --foo
+      See 'docker run --help'.
+      125
+
+**_126_** if the **_contained command_** cannot be invoked
+
+    $ docker run busybox /etc; echo $?
+    # exec: "/etc": permission denied
+      docker: Error response from daemon: Contained command could not be invoked
+      126
+
+**_127_** if the **_contained command_** cannot be found
+
+    $ docker run busybox foo; echo $?
+    # exec: "foo": executable file not found in $PATH
+      docker: Error response from daemon: Contained command not found or does not exist
+      127
+
+**_Exit code_** of **_contained command_** otherwise
+
+    $ docker run busybox /bin/sh -c 'exit 3' 
+    # 3
+
 ## Clean up (--rm)
 
 By default a container's file system persists even after the container

errors/daemon.go

@@ -599,15 +599,6 @@ var (
 		HTTPStatusCode: http.StatusInternalServerError,
 	})
 
-	// ErrorCodeCantStart is generated when an error occurred while
-	// trying to start a container.
-	ErrorCodeCantStart = errcode.Register(errGroup, errcode.ErrorDescriptor{
-		Value:          "CANTSTART",
-		Message:        "Cannot start container %s: %s",
-		Description:    "There was an error while trying to start a container",
-		HTTPStatusCode: http.StatusInternalServerError,
-	})
-
 	// ErrorCodeCantRestart is generated when an error occurred while
 	// trying to restart a container.
 	ErrorCodeCantRestart = errcode.Register(errGroup, errcode.ErrorDescriptor{
@@ -930,4 +921,31 @@ var (
 		Description:    "An attempt to create a volume using a driver but the volume already exists with a different driver",
 		HTTPStatusCode: http.StatusInternalServerError,
 	})
+
+	// ErrorCodeCmdNotFound is generated when contained cmd can't start,
+	// contained command not found error, exit code 127
+	ErrorCodeCmdNotFound = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:          "CMDNOTFOUND",
+		Message:        "Contained command not found or does not exist.",
+		Description:    "Command could not be found, command does not exist",
+		HTTPStatusCode: http.StatusInternalServerError,
+	})
+
+	// ErrorCodeCmdCouldNotBeInvoked is generated when contained cmd can't start,
+	// contained command permission denied error, exit code 126
+	ErrorCodeCmdCouldNotBeInvoked = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:          "CMDCOULDNOTBEINVOKED",
+		Message:        "Contained command could not be invoked.",
+		Description:    "Permission denied, cannot invoke command",
+		HTTPStatusCode: http.StatusInternalServerError,
+	})
+
+	// ErrorCodeCantStart is generated when contained cmd can't start,
+	// for any reason other than above 2 errors
+	ErrorCodeCantStart = errcode.Register(errGroup, errcode.ErrorDescriptor{
+		Value:          "CANTSTART",
+		Message:        "Cannot start container %s: %s",
+		Description:    "There was an error while trying to start a container",
+		HTTPStatusCode: http.StatusInternalServerError,
+	})
 )

integration-cli/docker_cli_run_test.go

@@ -1645,9 +1645,9 @@ func (s *DockerSuite) TestRunWorkdirExistsAndIsFile(c *check.C) {
 		expected = "The directory name is invalid"
 	}
 
-	out, exit, err := dockerCmdWithError("run", "-w", existingFile, "busybox")
-	if !(err != nil && exit == 1 && strings.Contains(out, expected)) {
-		c.Fatalf("Docker must complains about making dir, but we got out: %s, exit: %d, err: %s", out, exit, err)
+	out, exitCode, err := dockerCmdWithError("run", "-w", existingFile, "busybox")
+	if !(err != nil && exitCode == 125 && strings.Contains(out, expected)) {
+		c.Fatalf("Docker must complains about making dir with exitCode 125 but we got out: %s, exitCode: %d", out, exitCode)
 	}
 }
 
@@ -3746,17 +3746,72 @@ func (s *DockerSuite) TestRunStdinBlockedAfterContainerExit(c *check.C) {
 func (s *DockerSuite) TestRunWrongCpusetCpusFlagValue(c *check.C) {
 	// TODO Windows: This needs validation (error out) in the daemon.
 	testRequires(c, DaemonIsLinux)
-	out, _, err := dockerCmdWithError("run", "--cpuset-cpus", "1-10,11--", "busybox", "true")
+	out, exitCode, err := dockerCmdWithError("run", "--cpuset-cpus", "1-10,11--", "busybox", "true")
 	c.Assert(err, check.NotNil)
 	expected := "Error response from daemon: Invalid value 1-10,11-- for cpuset cpus.\n"
-	c.Assert(out, check.Equals, expected, check.Commentf("Expected output to contain %q, got %q", expected, out))
+	if !(strings.Contains(out, expected) || exitCode == 125) {
+		c.Fatalf("Expected output to contain %q with exitCode 125, got out: %q exitCode: %v", expected, out, exitCode)
+	}
 }
 
 func (s *DockerSuite) TestRunWrongCpusetMemsFlagValue(c *check.C) {
 	// TODO Windows: This needs validation (error out) in the daemon.
 	testRequires(c, DaemonIsLinux)
-	out, _, err := dockerCmdWithError("run", "--cpuset-mems", "1-42--", "busybox", "true")
+	out, exitCode, err := dockerCmdWithError("run", "--cpuset-mems", "1-42--", "busybox", "true")
 	c.Assert(err, check.NotNil)
 	expected := "Error response from daemon: Invalid value 1-42-- for cpuset mems.\n"
-	c.Assert(out, check.Equals, expected, check.Commentf("Expected output to contain %q, got %q", expected, out))
+	if !(strings.Contains(out, expected) || exitCode == 125) {
+		c.Fatalf("Expected output to contain %q with exitCode 125, got out: %q exitCode: %v", expected, out, exitCode)
+	}
+}
+
+// TestRunNonExecutableCmd checks that 'docker run busybox foo' exits with error code 127'
+func (s *DockerSuite) TestRunNonExecutableCmd(c *check.C) {
+	name := "testNonExecutableCmd"
+	runCmd := exec.Command(dockerBinary, "run", "--name", name, "busybox", "foo")
+	_, exit, _ := runCommandWithOutput(runCmd)
+	stateExitCode := findContainerExitCode(c, name)
+	if !(exit == 127 && strings.Contains(stateExitCode, "127")) {
+		c.Fatalf("Run non-executable command should have errored with exit code 127, but we got exit: %d, State.ExitCode: %s", exit, stateExitCode)
+	}
+}
+
+// TestRunNonExistingCmd checks that 'docker run busybox /bin/foo' exits with code 127.
+func (s *DockerSuite) TestRunNonExistingCmd(c *check.C) {
+	name := "testNonExistingCmd"
+	runCmd := exec.Command(dockerBinary, "run", "--name", name, "busybox", "/bin/foo")
+	_, exit, _ := runCommandWithOutput(runCmd)
+	stateExitCode := findContainerExitCode(c, name)
+	if !(exit == 127 && strings.Contains(stateExitCode, "127")) {
+		c.Fatalf("Run non-existing command should have errored with exit code 127, but we got exit: %d, State.ExitCode: %s", exit, stateExitCode)
+	}
+}
+
+// TestCmdCannotBeInvoked checks that 'docker run busybox /etc' exits with 126.
+func (s *DockerSuite) TestCmdCannotBeInvoked(c *check.C) {
+	name := "testCmdCannotBeInvoked"
+	runCmd := exec.Command(dockerBinary, "run", "--name", name, "busybox", "/etc")
+	_, exit, _ := runCommandWithOutput(runCmd)
+	stateExitCode := findContainerExitCode(c, name)
+	if !(exit == 126 && strings.Contains(stateExitCode, "126")) {
+		c.Fatalf("Run cmd that cannot be invoked should have errored with code 126, but we got exit: %d, State.ExitCode: %s", exit, stateExitCode)
+	}
+}
+
+// TestRunNonExistingImage checks that 'docker run foo' exits with error msg 125 and contains  'Unable to find image'
+func (s *DockerSuite) TestRunNonExistingImage(c *check.C) {
+	runCmd := exec.Command(dockerBinary, "run", "foo")
+	out, exit, err := runCommandWithOutput(runCmd)
+	if !(err != nil && exit == 125 && strings.Contains(out, "Unable to find image")) {
+		c.Fatalf("Run non-existing image should have errored with 'Unable to find image' code 125, but we got out: %s, exit: %d, err: %s", out, exit, err)
+	}
+}
+
+// TestDockerFails checks that 'docker run -foo busybox' exits with 125 to signal docker run failed
+func (s *DockerSuite) TestDockerFails(c *check.C) {
+	runCmd := exec.Command(dockerBinary, "run", "-foo", "busybox")
+	out, exit, err := runCommandWithOutput(runCmd)
+	if !(err != nil && exit == 125) {
+		c.Fatalf("Docker run with flag not defined should exit with 125, but we got out: %s, exit: %d, err: %s", out, exit, err)
+	}
 }

integration-cli/docker_cli_run_unix_test.go

@@ -397,8 +397,10 @@ func (s *DockerSuite) TestRunInvalidCpusetCpusFlagValue(c *check.C) {
 	}
 	out, _, err := dockerCmdWithError("run", "--cpuset-cpus", strconv.Itoa(invalid), "busybox", "true")
 	c.Assert(err, check.NotNil)
-	expected := fmt.Sprintf("Error response from daemon: Requested CPUs are not available - requested %s, available: %s.\n", strconv.Itoa(invalid), sysInfo.Cpus)
-	c.Assert(out, check.Equals, expected, check.Commentf("Expected output to contain %q, got %q", expected, out))
+	expected := fmt.Sprintf("Error response from daemon: Requested CPUs are not available - requested %s, available: %s", strconv.Itoa(invalid), sysInfo.Cpus)
+	if !(strings.Contains(out, expected)) {
+		c.Fatalf("Expected output to contain %q, got %q", expected, out)
+	}
 }
 
 func (s *DockerSuite) TestRunInvalidCpusetMemsFlagValue(c *check.C) {
@@ -416,8 +418,10 @@ func (s *DockerSuite) TestRunInvalidCpusetMemsFlagValue(c *check.C) {
 	}
 	out, _, err := dockerCmdWithError("run", "--cpuset-mems", strconv.Itoa(invalid), "busybox", "true")
 	c.Assert(err, check.NotNil)
-	expected := fmt.Sprintf("Error response from daemon: Requested memory nodes are not available - requested %s, available: %s.\n", strconv.Itoa(invalid), sysInfo.Mems)
-	c.Assert(out, check.Equals, expected, check.Commentf("Expected output to contain %q, got %q", expected, out))
+	expected := fmt.Sprintf("Error response from daemon: Requested memory nodes are not available - requested %s, available: %s", strconv.Itoa(invalid), sysInfo.Mems)
+	if !(strings.Contains(out, expected)) {
+		c.Fatalf("Expected output to contain %q, got %q", expected, out)
+	}
 }
 
 func (s *DockerSuite) TestRunInvalidCPUShares(c *check.C) {

integration-cli/docker_cli_start_test.go

@@ -129,11 +129,15 @@ func (s *DockerSuite) TestStartMultipleContainers(c *check.C) {
 
 	// start all the three containers, container `child_first` start first which should be failed
 	// container 'parent' start second and then start container 'child_second'
+	expOut := "Cannot link to a non running container"
+	expErr := "failed to start containers: [child_first]"
 	out, _, err = dockerCmdWithError("start", "child_first", "parent", "child_second")
 	// err shouldn't be nil because start will fail
 	c.Assert(err, checker.NotNil, check.Commentf("out: %s", out))
 	// output does not correspond to what was expected
-	c.Assert(out, checker.Contains, "Cannot start container child_first")
+	if !(strings.Contains(out, expOut) || strings.Contains(err.Error(), expErr)) {
+		c.Fatalf("Expected out: %v with err: %v  but got out: %v with err: %v", expOut, expErr, out, err)
+	}
 
 	for container, expected := range map[string]string{"parent": "true", "child_first": "false", "child_second": "true"} {
 		out, err := inspectField(container, "State.Running")

integration-cli/docker_utils.go

@@ -815,6 +815,17 @@ func dockerCmdInDirWithTimeout(timeout time.Duration, path string, args ...strin
 	return integration.DockerCmdInDirWithTimeout(dockerBinary, timeout, path, args...)
 }
 
+// find the State.ExitCode in container metadata
+func findContainerExitCode(c *check.C, name string, vargs ...string) string {
+	args := append(vargs, "inspect", "--format='{{ .State.ExitCode }} {{ .State.Error }}'", name)
+	cmd := exec.Command(dockerBinary, args...)
+	out, _, err := runCommandWithOutput(cmd)
+	if err != nil {
+		c.Fatal(err, out)
+	}
+	return out
+}
+
 func findContainerIP(c *check.C, id string, network string) string {
 	out, _ := dockerCmd(c, "inspect", fmt.Sprintf("--format='{{ .NetworkSettings.Networks.%s.IPAddress }}'", network), id)
 	return strings.Trim(out, " \r\n'")

man/docker-run.1.md

@@ -508,6 +508,38 @@ running binaries within a container is the root directory (/). The developer can
 set a different default with the Dockerfile WORKDIR instruction. The operator
 can override the working directory by using the **-w** option.
 
+# Exit Status
+
+The exit code from `docker run` gives information about why the container
+failed to run or why it exited.  When `docker run` exits with a non-zero code,
+the exit codes follow the `chroot` standard, see below:
+
+**_125_** if the error is with Docker daemon **_itself_** 
+
+    $ docker run --foo busybox; echo $?
+    # flag provided but not defined: --foo
+      See 'docker run --help'.
+      125
+
+**_126_** if the **_contained command_** cannot be invoked
+
+    $ docker run busybox /etc; echo $?
+    # exec: "/etc": permission denied
+      docker: Error response from daemon: Contained command could not be invoked
+      126
+
+**_127_** if the **_contained command_** cannot be found
+
+    $ docker run busybox foo; echo $?
+    # exec: "foo": executable file not found in $PATH
+      docker: Error response from daemon: Contained command not found or does not exist
+      127
+
+**_Exit code_** of **_contained command_** otherwise 
+    
+    $ docker run busybox /bin/sh -c 'exit 3' 
+    # 3
+
 # EXAMPLES
 
 ## Exposing log messages from the container to the host's log
@@ -732,3 +764,4 @@ April 2014, Originally compiled by William Henry (whenry at redhat dot com)
 based on docker.com source material and internal work.
 June 2014, updated by Sven Dowideit <SvenDowideit@home.org.au>
 July 2014, updated by Sven Dowideit <SvenDowideit@home.org.au>
+November 2015, updated by Sally O'Malley <somalley@redhat.com>

pkg/mflag/flag.go

@@ -1102,7 +1102,7 @@ func (fs *FlagSet) Parse(arguments []string) error {
 		case ContinueOnError:
 			return err
 		case ExitOnError:
-			os.Exit(2)
+			os.Exit(125)
 		case PanicOnError:
 			panic(err)
 		}