Merge pull request #32406 from ijc25/docker-build-iidfile

Add `docker build --iidfile=FILE`

api/server/router/build/build_routes.go

@@ -132,7 +132,10 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
 }
 
 func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	var notVerboseBuffer = bytes.NewBuffer(nil)
+	var (
+		notVerboseBuffer = bytes.NewBuffer(nil)
+		version          = httputils.VersionFromContext(ctx)
+	)
 
 	w.Header().Set("Content-Type", "application/json")
 
@@ -177,10 +180,12 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
 		return progress.NewProgressReader(in, progressOutput, r.ContentLength, "Downloading context", buildOptions.RemoteContext)
 	}
 
+	wantAux := versions.GreaterThanOrEqualTo(version, "1.30")
+
 	imgID, err := br.backend.Build(ctx, backend.BuildConfig{
 		Source:         r.Body,
 		Options:        buildOptions,
-		ProgressWriter: buildProgressWriter(out, createProgressReader),
+		ProgressWriter: buildProgressWriter(out, wantAux, createProgressReader),
 	})
 	if err != nil {
 		return errf(err)
@@ -221,13 +226,19 @@ func (s *syncWriter) Write(b []byte) (count int, err error) {
 	return
 }
 
-func buildProgressWriter(out io.Writer, createProgressReader func(io.ReadCloser) io.ReadCloser) backend.ProgressWriter {
+func buildProgressWriter(out io.Writer, wantAux bool, createProgressReader func(io.ReadCloser) io.ReadCloser) backend.ProgressWriter {
 	out = &syncWriter{w: out}
 
+	var aux *streamformatter.AuxFormatter
+	if wantAux {
+		aux = &streamformatter.AuxFormatter{Writer: out}
+	}
+
 	return backend.ProgressWriter{
 		Output:             out,
 		StdoutFormatter:    streamformatter.NewStdoutWriter(out),
 		StderrFormatter:    streamformatter.NewStderrWriter(out),
+		AuxFormatter:       aux,
 		ProgressReaderFunc: createProgressReader,
 	}
 }

api/types/backend/build.go

@@ -4,6 +4,7 @@ import (
 	"io"
 
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/pkg/streamformatter"
 )
 
 // ProgressWriter is a data object to transport progress streams to the client
@@ -11,6 +12,7 @@ type ProgressWriter struct {
 	Output             io.Writer
 	StdoutFormatter    io.Writer
 	StderrFormatter    io.Writer
+	AuxFormatter       *streamformatter.AuxFormatter
 	ProgressReaderFunc func(io.ReadCloser) io.ReadCloser
 }
 

api/types/types.go

@@ -530,3 +530,8 @@ type PushResult struct {
 	Digest string
 	Size   int
 }
+
+// BuildResult contains the image id of a successful build
+type BuildResult struct {
+	ID string
+}

builder/dockerfile/builder.go

@@ -15,6 +15,7 @@ import (
 	"github.com/docker/docker/builder/dockerfile/command"
 	"github.com/docker/docker/builder/dockerfile/parser"
 	"github.com/docker/docker/builder/remotecontext"
+	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
@@ -89,6 +90,7 @@ type Builder struct {
 
 	Stdout io.Writer
 	Stderr io.Writer
+	Aux    *streamformatter.AuxFormatter
 	Output io.Writer
 
 	docker    builder.Backend
@@ -114,6 +116,7 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
 		options:       config,
 		Stdout:        options.ProgressWriter.StdoutFormatter,
 		Stderr:        options.ProgressWriter.StderrFormatter,
+		Aux:           options.ProgressWriter.AuxFormatter,
 		Output:        options.ProgressWriter.Output,
 		docker:        options.Backend,
 		tmpContainers: map[string]struct{}{},
@@ -161,6 +164,13 @@ func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*buil
 	return &builder.Result{ImageID: dispatchState.imageID, FromImage: dispatchState.baseImage}, nil
 }
 
+func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error {
+	if aux == nil || state.imageID == "" {
+		return nil
+	}
+	return aux.Emit(types.BuildResult{ID: state.imageID})
+}
+
 func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result) (*dispatchState, error) {
 	shlex := NewShellLex(dockerfile.EscapeToken)
 	state := newDispatchState()
@@ -176,6 +186,15 @@ func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result)
 			// Not cancelled yet, keep going...
 		}
 
+		// If this is a FROM and we have a previous image then
+		// emit an aux message for that image since it is the
+		// end of the previous stage
+		if n.Value == command.From {
+			if err := emitImageID(b.Aux, state); err != nil {
+				return nil, err
+			}
+		}
+
 		if n.Value == command.From && state.isCurrentStage(b.options.Target) {
 			break
 		}
@@ -198,6 +217,12 @@ func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result)
 			b.clearTmp()
 		}
 	}
+
+	// Emit a final aux message for the final image
+	if err := emitImageID(b.Aux, state); err != nil {
+		return nil, err
+	}
+
 	return state, nil
 }
 

cli/command/image/build.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bufio"
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -65,6 +66,7 @@ type buildOptions struct {
 	networkMode    string
 	squash         bool
 	target         string
+	imageIDFile    string
 }
 
 // NewBuildCommand creates a new `docker build` command
@@ -117,6 +119,7 @@ func NewBuildCommand(dockerCli *command.DockerCli) *cobra.Command {
 	flags.SetAnnotation("network", "version", []string{"1.25"})
 	flags.Var(&options.extraHosts, "add-host", "Add a custom host-to-IP mapping (host:ip)")
 	flags.StringVar(&options.target, "target", "", "Set the target build stage to build.")
+	flags.StringVar(&options.imageIDFile, "iidfile", "", "Write the image ID to the file")
 
 	command.AddTrustVerificationFlags(flags)
 
@@ -162,6 +165,12 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 		progBuff = bytes.NewBuffer(nil)
 		buildBuff = bytes.NewBuffer(nil)
 	}
+	if options.imageIDFile != "" {
+		// Avoid leaving a stale file if we eventually fail
+		if err := os.Remove(options.imageIDFile); err != nil && !os.IsNotExist(err) {
+			return errors.Wrap(err, "Removing image ID file")
+		}
+	}
 
 	if options.dockerfileName == "-" {
 		if specifiedContext == "-" {
@@ -316,7 +325,17 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 	}
 	defer response.Body.Close()
 
-	err = jsonmessage.DisplayJSONMessagesStream(response.Body, buildBuff, dockerCli.Out().FD(), dockerCli.Out().IsTerminal(), nil)
+	imageID := ""
+	aux := func(auxJSON *json.RawMessage) {
+		var result types.BuildResult
+		if err := json.Unmarshal(*auxJSON, &result); err != nil {
+			fmt.Fprintf(dockerCli.Err(), "Failed to parse aux message: %s", err)
+		} else {
+			imageID = result.ID
+		}
+	}
+
+	err = jsonmessage.DisplayJSONMessagesStream(response.Body, buildBuff, dockerCli.Out().FD(), dockerCli.Out().IsTerminal(), aux)
 	if err != nil {
 		if jerr, ok := err.(*jsonmessage.JSONError); ok {
 			// If no error code is set, default to 1
@@ -344,9 +363,18 @@ func runBuild(dockerCli *command.DockerCli, options buildOptions) error {
 	// Everything worked so if -q was provided the output from the daemon
 	// should be just the image ID and we'll print that to stdout.
 	if options.quiet {
-		fmt.Fprintf(dockerCli.Out(), "%s", buildBuff)
+		imageID = fmt.Sprintf("%s", buildBuff)
+		fmt.Fprintf(dockerCli.Out(), imageID)
 	}
 
+	if options.imageIDFile != "" {
+		if imageID == "" {
+			return errors.Errorf("Server did not provide an image ID. Cannot write %s", options.imageIDFile)
+		}
+		if err := ioutil.WriteFile(options.imageIDFile, []byte(imageID), 0666); err != nil {
+			return err
+		}
+	}
 	if command.IsTrusted() {
 		// Since the build was successful, now we must tag any of the resolved
 		// images from the above Dockerfile rewrite.

docs/api/version-history.md

@@ -18,6 +18,7 @@ keywords: "API, Docker, rcli, REST, documentation"
 [Docker Engine API v1.30](https://docs.docker.com/engine/api/v1.30/) documentation
 
 * `GET /info` now returns the list of supported logging drivers, including plugins.
+* `POST /build/` now (when not silent) produces an `Aux` message in the JSON output stream with payload `types.BuildResult` for each image produced. The final such message will reference the image resulting from the build.
 
 ## v1.29 API changes
 

docs/reference/commandline/build.md

@@ -35,6 +35,7 @@ Options:
   -f, --file string             Name of the Dockerfile (Default is 'PATH/Dockerfile')
       --force-rm                Always remove intermediate containers
       --help                    Print usage
+      --iidfile string          Write the image ID to the file
       --isolation string        Container isolation technology
       --label value             Set metadata for an image (default [])
   -m, --memory string           Memory limit

integration-cli/docker_cli_build_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/docker/docker/pkg/testutil"
 	icmd "github.com/docker/docker/pkg/testutil/cmd"
 	"github.com/go-check/check"
+	"github.com/opencontainers/go-digest"
 )
 
 func (s *DockerSuite) TestBuildJSONEmptyRun(c *check.C) {
@@ -6398,3 +6399,49 @@ CMD echo foo
 	out, _ := dockerCmd(c, "inspect", "--format", "{{ json .Config.Cmd }}", "build2")
 	c.Assert(strings.TrimSpace(out), checker.Equals, `["/bin/sh","-c","echo foo"]`)
 }
+
+func (s *DockerSuite) TestBuildIidFile(c *check.C) {
+	tmpDir, err := ioutil.TempDir("", "TestBuildIidFile")
+	if err != nil {
+		c.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+	tmpIidFile := filepath.Join(tmpDir, "iid")
+
+	name := "testbuildiidfile"
+	// Use a Dockerfile with multiple stages to ensure we get the last one
+	cli.BuildCmd(c, name,
+		build.WithDockerfile(`FROM `+minimalBaseImage()+` AS stage1
+ENV FOO FOO
+FROM `+minimalBaseImage()+`
+ENV BAR BAZ`),
+		cli.WithFlags("--iidfile", tmpIidFile))
+
+	id, err := ioutil.ReadFile(tmpIidFile)
+	c.Assert(err, check.IsNil)
+	d, err := digest.Parse(string(id))
+	c.Assert(err, check.IsNil)
+	c.Assert(d.String(), checker.Equals, getIDByName(c, name))
+}
+
+func (s *DockerSuite) TestBuildIidFileCleanupOnFail(c *check.C) {
+	tmpDir, err := ioutil.TempDir("", "TestBuildIidFileCleanupOnFail")
+	if err != nil {
+		c.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+	tmpIidFile := filepath.Join(tmpDir, "iid")
+
+	err = ioutil.WriteFile(tmpIidFile, []byte("Dummy"), 0666)
+	c.Assert(err, check.IsNil)
+
+	cli.Docker(cli.Build("testbuildiidfilecleanuponfail"),
+		build.WithDockerfile(`FROM `+minimalBaseImage()+`
+	RUN /non/existing/command`),
+		cli.WithFlags("--iidfile", tmpIidFile)).Assert(c, icmd.Expected{
+		ExitCode: 1,
+	})
+	_, err = os.Stat(tmpIidFile)
+	c.Assert(err, check.NotNil)
+	c.Assert(os.IsNotExist(err), check.Equals, true)
+}

man/docker-build.1.md

@@ -11,6 +11,7 @@ docker-build - Build an image from a Dockerfile
 [**--cpu-shares**[=*0*]]
 [**--cgroup-parent**[=*CGROUP-PARENT*]]
 [**--help**]
+[**--iidfile**[=*CIDFILE*]]
 [**-f**|**--file**[=*PATH/Dockerfile*]]
 [**-squash**] *Experimental*
 [**--force-rm**]
@@ -104,6 +105,9 @@ option can be set multiple times.
 **--no-cache**=*true*|*false*
    Do not use cache when building the image. The default is *false*.
 
+**--iidfile**=""
+   Write the image ID to the file
+
 **--help**
   Print usage statement
 

pkg/jsonmessage/jsonmessage.go

@@ -109,7 +109,7 @@ type JSONMessage struct {
 	TimeNano        int64         `json:"timeNano,omitempty"`
 	Error           *JSONError    `json:"errorDetail,omitempty"`
 	ErrorMessage    string        `json:"error,omitempty"` //deprecated
-	// Aux contains out-of-band data, such as digests for push signing.
+	// Aux contains out-of-band data, such as digests for push signing and image id after building.
 	Aux *json.RawMessage `json:"aux,omitempty"`
 }
 

pkg/streamformatter/streamformatter.go

@@ -132,3 +132,28 @@ func (out *progressOutput) WriteProgress(prog progress.Progress) error {
 
 	return nil
 }
+
+// AuxFormatter is a streamFormatter that writes aux progress messages
+type AuxFormatter struct {
+	io.Writer
+}
+
+// Emit emits the given interface as an aux progress message
+func (sf *AuxFormatter) Emit(aux interface{}) error {
+	auxJSONBytes, err := json.Marshal(aux)
+	if err != nil {
+		return err
+	}
+	auxJSON := new(json.RawMessage)
+	*auxJSON = auxJSONBytes
+	msgJSON, err := json.Marshal(&jsonmessage.JSONMessage{Aux: auxJSON})
+	if err != nil {
+		return err
+	}
+	msgJSON = appendNewline(msgJSON)
+	n, err := sf.Writer.Write(msgJSON)
+	if n != len(msgJSON) {
+		return io.ErrShortWrite
+	}
+	return err
+}