Merge pull request #32610 from dnephin/builder-move-tag-and-squash

Extract squash and tagging from the Dockerfile builder

api/server/backend/build/backend.go

@@ -0,0 +1,70 @@
+package build
+
+import (
+	"fmt"
+
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/api/types/backend"
+	"github.com/docker/docker/builder"
+	"github.com/docker/docker/builder/dockerfile"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+)
+
+// ImageComponent provides an interface for working with images
+type ImageComponent interface {
+	SquashImage(from string, to string) (string, error)
+	TagImageWithReference(image.ID, reference.Named) error
+}
+
+// Backend provides build functionality to the API router
+type Backend struct {
+	manager        *dockerfile.BuildManager
+	imageComponent ImageComponent
+}
+
+// NewBackend creates a new build backend from components
+func NewBackend(components ImageComponent, builderBackend builder.Backend) *Backend {
+	manager := dockerfile.NewBuildManager(builderBackend)
+	return &Backend{imageComponent: components, manager: manager}
+}
+
+// Build builds an image from a Source
+func (b *Backend) Build(ctx context.Context, config backend.BuildConfig) (string, error) {
+	options := config.Options
+	tagger, err := NewTagger(b.imageComponent, config.ProgressWriter.StdoutFormatter, options.Tags)
+	if err != nil {
+		return "", err
+	}
+
+	build, err := b.manager.Build(ctx, config)
+	if err != nil {
+		return "", err
+	}
+
+	var imageID = build.ImageID
+	if options.Squash {
+		if imageID, err = squashBuild(build, b.imageComponent); err != nil {
+			return "", err
+		}
+	}
+
+	stdout := config.ProgressWriter.StdoutFormatter
+	fmt.Fprintf(stdout, "Successfully built %s\n", stringid.TruncateID(imageID))
+	err = tagger.TagImages(image.ID(imageID))
+	return imageID, err
+}
+
+func squashBuild(build *builder.Result, imageComponent ImageComponent) (string, error) {
+	var fromID string
+	if build.FromImage != nil {
+		fromID = build.FromImage.ImageID()
+	}
+	imageID, err := imageComponent.SquashImage(build.ImageID, fromID)
+	if err != nil {
+		return "", errors.Wrap(err, "error squashing image")
+	}
+	return imageID, nil
+}

api/server/backend/build/tag.go

@@ -0,0 +1,77 @@
+package build
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/image"
+	"github.com/pkg/errors"
+)
+
+// Tagger is responsible for tagging an image created by a builder
+type Tagger struct {
+	imageComponent ImageComponent
+	stdout         io.Writer
+	repoAndTags    []reference.Named
+}
+
+// NewTagger returns a new Tagger for tagging the images of a build.
+// If any of the names are invalid tags an error is returned.
+func NewTagger(backend ImageComponent, stdout io.Writer, names []string) (*Tagger, error) {
+	reposAndTags, err := sanitizeRepoAndTags(names)
+	if err != nil {
+		return nil, err
+	}
+	return &Tagger{
+		imageComponent: backend,
+		stdout:         stdout,
+		repoAndTags:    reposAndTags,
+	}, nil
+}
+
+// TagImages creates image tags for the imageID
+func (bt *Tagger) TagImages(imageID image.ID) error {
+	for _, rt := range bt.repoAndTags {
+		if err := bt.imageComponent.TagImageWithReference(imageID, rt); err != nil {
+			return err
+		}
+		fmt.Fprintf(bt.stdout, "Successfully tagged %s\n", reference.FamiliarString(rt))
+	}
+	return nil
+}
+
+// sanitizeRepoAndTags parses the raw "t" parameter received from the client
+// to a slice of repoAndTag.
+// It also validates each repoName and tag.
+func sanitizeRepoAndTags(names []string) ([]reference.Named, error) {
+	var (
+		repoAndTags []reference.Named
+		// This map is used for deduplicating the "-t" parameter.
+		uniqNames = make(map[string]struct{})
+	)
+	for _, repo := range names {
+		if repo == "" {
+			continue
+		}
+
+		ref, err := reference.ParseNormalizedNamed(repo)
+		if err != nil {
+			return nil, err
+		}
+
+		if _, isCanonical := ref.(reference.Canonical); isCanonical {
+			return nil, errors.New("build tag cannot contain a digest")
+		}
+
+		ref = reference.TagNameOnly(ref)
+
+		nameWithTag := ref.String()
+
+		if _, exists := uniqNames[nameWithTag]; !exists {
+			uniqNames[nameWithTag] = struct{}{}
+			repoAndTags = append(repoAndTags, ref)
+		}
+	}
+	return repoAndTags, nil
+}

api/server/router/build/backend.go

@@ -1,20 +1,17 @@
 package build
 
 import (
-	"io"
-
-	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"golang.org/x/net/context"
 )
 
 // Backend abstracts an image builder whose only purpose is to build an image referenced by an imageID.
 type Backend interface {
-	// BuildFromContext builds a Docker image referenced by an imageID string.
+	// Build a Docker image returning the id of the image
-	//
-	// Note: Tagging an image should not be done by a Builder, it should instead be done
-	// by the caller.
-	//
 	// TODO: make this return a reference instead of string
-	BuildFromContext(ctx context.Context, src io.ReadCloser, buildOptions *types.ImageBuildOptions, pg backend.ProgressWriter) (string, error)
+	Build(context.Context, backend.BuildConfig) (string, error)
+}
+
+type experimentalProvider interface {
+	HasExperimental() bool
 }

api/server/router/build/build.go

@@ -5,14 +5,13 @@ import "github.com/docker/docker/api/server/router"
 // buildRouter is a router to talk with the build controller
 type buildRouter struct {
 	backend Backend
+	daemon  experimentalProvider
 	routes  []router.Route
 }
 
 // NewRouter initializes a new build router
-func NewRouter(b Backend) router.Router {
+func NewRouter(b Backend, d experimentalProvider) router.Router {
-	r := &buildRouter{
+	r := &buildRouter{backend: b, daemon: d}
-		backend: b,
-	}
 	r.initRoutes()
 	return r
 }

api/server/router/build/build_routes.go

@@ -13,6 +13,7 @@ import (
 	"sync"
 
 	"github.com/Sirupsen/logrus"
+	apierrors "github.com/docker/docker/api/errors"
 	"github.com/docker/docker/api/server/httputils"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
@@ -22,6 +23,7 @@ import (
 	"github.com/docker/docker/pkg/progress"
 	"github.com/docker/docker/pkg/streamformatter"
 	units "github.com/docker/go-units"
+	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 )
 
@@ -87,9 +89,6 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
 		options.Ulimits = buildUlimits
 	}
 
-	var buildArgs = map[string]*string{}
-	buildArgsJSON := r.FormValue("buildargs")
-
 	// Note that there are two ways a --build-arg might appear in the
 	// json of the query param:
 	//     "foo":"bar"
@@ -102,25 +101,27 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
 	// the fact they mentioned it, we need to pass that along to the builder
 	// so that it can print a warning about "foo" being unused if there is
 	// no "ARG foo" in the Dockerfile.
+	buildArgsJSON := r.FormValue("buildargs")
 	if buildArgsJSON != "" {
+		var buildArgs = map[string]*string{}
 		if err := json.Unmarshal([]byte(buildArgsJSON), &buildArgs); err != nil {
 			return nil, err
 		}
 		options.BuildArgs = buildArgs
 	}
 
-	var labels = map[string]string{}
 	labelsJSON := r.FormValue("labels")
 	if labelsJSON != "" {
+		var labels = map[string]string{}
 		if err := json.Unmarshal([]byte(labelsJSON), &labels); err != nil {
 			return nil, err
 		}
 		options.Labels = labels
 	}
 
-	var cacheFrom = []string{}
 	cacheFromJSON := r.FormValue("cachefrom")
 	if cacheFromJSON != "" {
+		var cacheFrom = []string{}
 		if err := json.Unmarshal([]byte(cacheFromJSON), &cacheFrom); err != nil {
 			return nil, err
 		}
@@ -130,33 +131,8 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
 	return options, nil
 }
 
-type syncWriter struct {
-	w  io.Writer
-	mu sync.Mutex
-}
-
-func (s *syncWriter) Write(b []byte) (count int, err error) {
-	s.mu.Lock()
-	count, err = s.w.Write(b)
-	s.mu.Unlock()
-	return
-}
-
 func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
-	var (
+	var notVerboseBuffer = bytes.NewBuffer(nil)
-		authConfigs        = map[string]types.AuthConfig{}
-		authConfigsEncoded = r.Header.Get("X-Registry-Config")
-		notVerboseBuffer   = bytes.NewBuffer(nil)
-	)
-
-	if authConfigsEncoded != "" {
-		authConfigsJSON := base64.NewDecoder(base64.URLEncoding, strings.NewReader(authConfigsEncoded))
-		if err := json.NewDecoder(authConfigsJSON).Decode(&authConfigs); err != nil {
-			// for a pull it is not an error if no auth was given
-			// to increase compatibility with the existing api it is defaulting
-			// to be empty.
-		}
-	}
 
 	w.Header().Set("Content-Type", "application/json")
 
@@ -183,7 +159,12 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
 	if err != nil {
 		return errf(err)
 	}
-	buildOptions.AuthConfigs = authConfigs
+	buildOptions.AuthConfigs = getAuthConfigs(r.Header)
+
+	if buildOptions.Squash && !br.daemon.HasExperimental() {
+		return apierrors.NewBadRequestError(
+			errors.New("squash is only supported with experimental mode"))
+	}
 
 	// Currently, only used if context is from a remote url.
 	// Look at code in DetectContextFromRemoteURL for more information.
@@ -199,18 +180,12 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
 	if buildOptions.SuppressOutput {
 		out = notVerboseBuffer
 	}
-	out = &syncWriter{w: out}
-	stdout := &streamformatter.StdoutFormatter{Writer: out, StreamFormatter: sf}
-	stderr := &streamformatter.StderrFormatter{Writer: out, StreamFormatter: sf}
 
-	pg := backend.ProgressWriter{
+	imgID, err := br.backend.Build(ctx, backend.BuildConfig{
-		Output:             out,
+		Source:         r.Body,
-		StdoutFormatter:    stdout,
+		Options:        buildOptions,
-		StderrFormatter:    stderr,
+		ProgressWriter: buildProgressWriter(out, sf, createProgressReader),
-		ProgressReaderFunc: createProgressReader,
+	})
-	}
-
-	imgID, err := br.backend.BuildFromContext(ctx, r.Body, buildOptions, pg)
 	if err != nil {
 		return errf(err)
 	}
@@ -219,8 +194,47 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
 	// should be just the image ID and we'll print that to stdout.
 	if buildOptions.SuppressOutput {
 		stdout := &streamformatter.StdoutFormatter{Writer: output, StreamFormatter: sf}
-		fmt.Fprintf(stdout, "%s\n", string(imgID))
+		fmt.Fprintln(stdout, imgID)
 	}
-
 	return nil
 }
+
+func getAuthConfigs(header http.Header) map[string]types.AuthConfig {
+	authConfigs := map[string]types.AuthConfig{}
+	authConfigsEncoded := header.Get("X-Registry-Config")
+
+	if authConfigsEncoded == "" {
+		return authConfigs
+	}
+
+	authConfigsJSON := base64.NewDecoder(base64.URLEncoding, strings.NewReader(authConfigsEncoded))
+	// Pulling an image does not error when no auth is provided so to remain
+	// consistent with the existing api decode errors are ignored
+	json.NewDecoder(authConfigsJSON).Decode(&authConfigs)
+	return authConfigs
+}
+
+type syncWriter struct {
+	w  io.Writer
+	mu sync.Mutex
+}
+
+func (s *syncWriter) Write(b []byte) (count int, err error) {
+	s.mu.Lock()
+	count, err = s.w.Write(b)
+	s.mu.Unlock()
+	return
+}
+
+func buildProgressWriter(out io.Writer, sf *streamformatter.StreamFormatter, createProgressReader func(io.ReadCloser) io.ReadCloser) backend.ProgressWriter {
+	out = &syncWriter{w: out}
+	stdout := &streamformatter.StdoutFormatter{Writer: out, StreamFormatter: sf}
+	stderr := &streamformatter.StderrFormatter{Writer: out, StreamFormatter: sf}
+
+	return backend.ProgressWriter{
+		Output:             out,
+		StdoutFormatter:    stdout,
+		StderrFormatter:    stderr,
+		ProgressReaderFunc: createProgressReader,
+	}
+}

api/types/backend/backend.go

@@ -6,7 +6,6 @@ import (
 	"time"
 
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/pkg/streamformatter"
 )
 
 // ContainerAttachConfig holds the streams to use when connecting to a container to view logs.
@@ -99,11 +98,3 @@ type ContainerCommitConfig struct {
 	types.ContainerCommitConfig
 	Changes []string
 }
-
-// ProgressWriter is a data object to transport progress streams to the client
-type ProgressWriter struct {
-	Output             io.Writer
-	StdoutFormatter    *streamformatter.StdoutFormatter
-	StderrFormatter    *streamformatter.StderrFormatter
-	ProgressReaderFunc func(io.ReadCloser) io.ReadCloser
-}

api/types/backend/build.go

@@ -0,0 +1,23 @@
+package backend
+
+import (
+	"io"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/pkg/streamformatter"
+)
+
+// ProgressWriter is a data object to transport progress streams to the client
+type ProgressWriter struct {
+	Output             io.Writer
+	StdoutFormatter    *streamformatter.StdoutFormatter
+	StderrFormatter    *streamformatter.StderrFormatter
+	ProgressReaderFunc func(io.ReadCloser) io.ReadCloser
+}
+
+// BuildConfig is the configuration used by a BuildManager to start a build
+type BuildConfig struct {
+	Source         io.ReadCloser
+	ProgressWriter ProgressWriter
+	Options        *types.ImageBuildOptions
+}

builder/builder.go

@@ -8,11 +8,9 @@ import (
 	"io"
 	"time"
 
-	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/container"
-	"github.com/docker/docker/image"
 	"golang.org/x/net/context"
 )
 
@@ -40,8 +38,6 @@ type Backend interface {
 
 	// GetImageOnBuild looks up a Docker image referenced by `name`.
 	GetImageOnBuild(name string) (Image, error)
-	// TagImageWithReference tags an image with newTag
-	TagImageWithReference(image.ID, reference.Named) error
 	// PullOnBuild tells Docker to pull image referenced by `name`.
 	PullOnBuild(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer) (Image, error)
 	// ContainerAttachRaw attaches to container.
@@ -69,12 +65,6 @@ type Backend interface {
 	// TODO: use containerd/fs.changestream instead as a source
 	CopyOnBuild(containerID string, destPath string, srcRoot string, srcPath string, decompress bool) error
 
-	// HasExperimental checks if the backend supports experimental features
-	HasExperimental() bool
-
-	// SquashImage squashes the fs layers from the provided image down to the specified `to` image
-	SquashImage(from string, to string) (string, error)
-
 	// MountImage returns mounted path with rootfs of an image.
 	MountImage(name string) (string, func() error, error)
 }
@@ -85,6 +75,12 @@ type Image interface {
 	RunConfig() *container.Config
 }
 
+// Result is the output produced by a Builder
+type Result struct {
+	ImageID   string
+	FromImage Image
+}
+
 // ImageCacheBuilder represents a generator for stateful image cache.
 type ImageCacheBuilder interface {
 	// MakeImageCache creates a stateful image cache.

builder/dockerfile/builder.go

@@ -5,12 +5,9 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
-	"os"
 	"strings"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/docker/distribution/reference"
-	apierrors "github.com/docker/docker/api/errors"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/container"
@@ -18,10 +15,10 @@ import (
 	"github.com/docker/docker/builder/dockerfile/command"
 	"github.com/docker/docker/builder/dockerfile/parser"
 	"github.com/docker/docker/builder/remotecontext"
-	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
+	"golang.org/x/sync/syncmap"
 )
 
 var validCommitCommands = map[string]bool{
@@ -39,6 +36,52 @@ var validCommitCommands = map[string]bool{
 
 var defaultLogConfig = container.LogConfig{Type: "none"}
 
+// BuildManager is shared across all Builder objects
+type BuildManager struct {
+	backend   builder.Backend
+	pathCache pathCache // TODO: make this persistent
+}
+
+// NewBuildManager creates a BuildManager
+func NewBuildManager(b builder.Backend) *BuildManager {
+	return &BuildManager{backend: b, pathCache: &syncmap.Map{}}
+}
+
+// Build starts a new build from a BuildConfig
+func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (*builder.Result, error) {
+	if config.Options.Dockerfile == "" {
+		config.Options.Dockerfile = builder.DefaultDockerfileName
+	}
+
+	source, dockerfile, err := remotecontext.Detect(config)
+	if err != nil {
+		return nil, err
+	}
+	if source != nil {
+		defer func() {
+			if err := source.Close(); err != nil {
+				logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
+			}
+		}()
+	}
+
+	builderOptions := builderOptions{
+		Options:        config.Options,
+		ProgressWriter: config.ProgressWriter,
+		Backend:        bm.backend,
+		PathCache:      bm.pathCache,
+	}
+	return newBuilder(ctx, builderOptions).build(source, dockerfile)
+}
+
+// builderOptions are the dependencies required by the builder
+type builderOptions struct {
+	Options        *types.ImageBuildOptions
+	Backend        builder.Backend
+	ProgressWriter backend.ProgressWriter
+	PathCache      pathCache
+}
+
 // Builder is a Dockerfile builder
 // It implements the builder.Backend interface.
 type Builder struct {
@@ -68,65 +111,25 @@ type Builder struct {
 	from        builder.Image
 }
 
-// BuildManager implements builder.Backend and is shared across all Builder objects.
+// newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options.
-type BuildManager struct {
+func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
-	backend   builder.Backend
+	config := options.Options
-	pathCache *pathCache // TODO: make this persistent
-}
-
-// NewBuildManager creates a BuildManager.
-func NewBuildManager(b builder.Backend) (bm *BuildManager) {
-	return &BuildManager{backend: b, pathCache: &pathCache{}}
-}
-
-// BuildFromContext builds a new image from a given context.
-func (bm *BuildManager) BuildFromContext(ctx context.Context, src io.ReadCloser, buildOptions *types.ImageBuildOptions, pg backend.ProgressWriter) (string, error) {
-	if buildOptions.Squash && !bm.backend.HasExperimental() {
-		return "", apierrors.NewBadRequestError(errors.New("squash is only supported with experimental mode"))
-	}
-	if buildOptions.Dockerfile == "" {
-		buildOptions.Dockerfile = builder.DefaultDockerfileName
-	}
-
-	source, dockerfile, err := remotecontext.Detect(ctx, buildOptions.RemoteContext, buildOptions.Dockerfile, src, pg.ProgressReaderFunc)
-	if err != nil {
-		return "", err
-	}
-	if source != nil {
-		defer func() {
-			if err := source.Close(); err != nil {
-				logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
-			}
-		}()
-	}
-	b, err := NewBuilder(ctx, buildOptions, bm.backend, source)
-	if err != nil {
-		return "", err
-	}
-	b.imageContexts.cache = bm.pathCache
-	return b.build(dockerfile, pg.StdoutFormatter, pg.StderrFormatter, pg.Output)
-}
-
-// NewBuilder creates a new Dockerfile builder from an optional dockerfile and a Config.
-// If dockerfile is nil, the Dockerfile specified by Config.DockerfileName,
-// will be read from the Context passed to Build().
-func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, backend builder.Backend, source builder.Source) (b *Builder, err error) {
 	if config == nil {
 		config = new(types.ImageBuildOptions)
 	}
-	b = &Builder{
+	b := &Builder{
 		clientCtx:     clientCtx,
 		options:       config,
-		Stdout:        os.Stdout,
+		Stdout:        options.ProgressWriter.StdoutFormatter,
-		Stderr:        os.Stderr,
+		Stderr:        options.ProgressWriter.StderrFormatter,
-		docker:        backend,
+		Output:        options.ProgressWriter.Output,
-		source:        source,
+		docker:        options.Backend,
 		runConfig:     new(container.Config),
 		tmpContainers: map[string]struct{}{},
 		buildArgs:     newBuildArgs(config.BuildArgs),
 	}
-	b.imageContexts = &imageContexts{b: b}
+	b.imageContexts = &imageContexts{b: b, cache: options.PathCache}
-	return b, nil
+	return b
 }
 
 func (b *Builder) resetImageCache() {
@@ -137,83 +140,31 @@ func (b *Builder) resetImageCache() {
 	b.cacheBusted = false
 }
 
-// sanitizeRepoAndTags parses the raw "t" parameter received from the client
+// Build runs the Dockerfile builder by parsing the Dockerfile and executing
-// to a slice of repoAndTag.
+// the instructions from the file.
-// It also validates each repoName and tag.
+func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*builder.Result, error) {
-func sanitizeRepoAndTags(names []string) ([]reference.Named, error) {
-	var (
-		repoAndTags []reference.Named
-		// This map is used for deduplicating the "-t" parameter.
-		uniqNames = make(map[string]struct{})
-	)
-	for _, repo := range names {
-		if repo == "" {
-			continue
-		}
-
-		ref, err := reference.ParseNormalizedNamed(repo)
-		if err != nil {
-			return nil, err
-		}
-
-		if _, isCanonical := ref.(reference.Canonical); isCanonical {
-			return nil, errors.New("build tag cannot contain a digest")
-		}
-
-		ref = reference.TagNameOnly(ref)
-
-		nameWithTag := ref.String()
-
-		if _, exists := uniqNames[nameWithTag]; !exists {
-			uniqNames[nameWithTag] = struct{}{}
-			repoAndTags = append(repoAndTags, ref)
-		}
-	}
-	return repoAndTags, nil
-}
-
-// build runs the Dockerfile builder from a context and a docker object that allows to make calls
-// to Docker.
-func (b *Builder) build(dockerfile *parser.Result, stdout io.Writer, stderr io.Writer, out io.Writer) (string, error) {
 	defer b.imageContexts.unmount()
 
-	b.Stdout = stdout
+	// TODO: Remove source field from Builder
-	b.Stderr = stderr
+	b.source = source
-	b.Output = out
-
-	repoAndTags, err := sanitizeRepoAndTags(b.options.Tags)
-	if err != nil {
-		return "", err
-	}
 
 	addNodesForLabelOption(dockerfile.AST, b.options.Labels)
 
 	if err := checkDispatchDockerfile(dockerfile.AST); err != nil {
-		return "", err
+		return nil, err
 	}
 
 	imageID, err := b.dispatchDockerfileWithCancellation(dockerfile)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	b.warnOnUnusedBuildArgs()
 
 	if imageID == "" {
-		return "", errors.New("No image was generated. Is your Dockerfile empty?")
+		return nil, errors.New("No image was generated. Is your Dockerfile empty?")
 	}
-
+	return &builder.Result{ImageID: imageID, FromImage: b.from}, nil
-	if b.options.Squash {
-		if imageID, err = b.squashBuild(imageID); err != nil {
-			return "", err
-		}
-	}
-
-	fmt.Fprintf(b.Stdout, "Successfully built %s\n", stringid.TruncateID(imageID))
-	if err := b.tagImages(imageID, repoAndTags); err != nil {
-		return "", err
-	}
-	return imageID, nil
 }
 
 func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result) (string, error) {
@@ -258,19 +209,6 @@ func (b *Builder) dispatchDockerfileWithCancellation(dockerfile *parser.Result)
 	return imageID, nil
 }
 
-func (b *Builder) squashBuild(imageID string) (string, error) {
-	var fromID string
-	var err error
-	if b.from != nil {
-		fromID = b.from.ImageID()
-	}
-	imageID, err = b.docker.SquashImage(imageID, fromID)
-	if err != nil {
-		return "", errors.Wrap(err, "error squashing image")
-	}
-	return imageID, nil
-}
-
 func addNodesForLabelOption(dockerfile *parser.Node, labels map[string]string) {
 	if len(labels) == 0 {
 		return
@@ -289,17 +227,6 @@ func (b *Builder) warnOnUnusedBuildArgs() {
 	}
 }
 
-func (b *Builder) tagImages(id string, repoAndTags []reference.Named) error {
-	imageID := image.ID(id)
-	for _, rt := range repoAndTags {
-		if err := b.docker.TagImageWithReference(imageID, rt); err != nil {
-			return err
-		}
-		fmt.Fprintf(b.Stdout, "Successfully tagged %s\n", reference.FamiliarString(rt))
-	}
-	return nil
-}
-
 // hasFromImage returns true if the builder has processed a `FROM <image>` line
 // TODO: move to DispatchState
 func (b *Builder) hasFromImage() bool {
@@ -316,10 +243,7 @@ func (b *Builder) hasFromImage() bool {
 //
 // TODO: Remove?
 func BuildFromConfig(config *container.Config, changes []string) (*container.Config, error) {
-	b, err := NewBuilder(context.Background(), nil, nil, nil)
+	b := newBuilder(context.Background(), builderOptions{})
-	if err != nil {
-		return nil, err
-	}
 
 	result, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n")))
 	if err != nil {

builder/dockerfile/dispatchers.go

@@ -595,7 +595,7 @@ func healthcheck(req dispatchRequest) error {
 // to /usr/sbin/nginx. Uses the default shell if not in JSON format.
 //
 // Handles command processing similar to CMD and RUN, only req.runConfig.Entrypoint
-// is initialized at NewBuilder time instead of through argument parsing.
+// is initialized at newBuilder time instead of through argument parsing.
 //
 func entrypoint(req dispatchRequest) error {
 	if err := req.flags.Parse(); err != nil {

builder/dockerfile/evaluator.go

@@ -2,7 +2,7 @@
 //
 // It incorporates a dispatch table based on the parser.Node values (see the
 // parser package for more information) that are yielded from the parser itself.
-// Calling NewBuilder with the BuildOpts struct can be used to customize the
+// Calling newBuilder with the BuildOpts struct can be used to customize the
 // experience for execution purposes only. Parsing is controlled in the parser
 // package, and this division of responsibility should be respected.
 //

builder/dockerfile/imagecontext.go

@@ -3,7 +3,6 @@ package dockerfile
 import (
 	"strconv"
 	"strings"
-	"sync"
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/api/types/container"
@@ -12,13 +11,18 @@ import (
 	"github.com/pkg/errors"
 )
 
+type pathCache interface {
+	Load(key interface{}) (value interface{}, ok bool)
+	Store(key, value interface{})
+}
+
 // imageContexts is a helper for stacking up built image rootfs and reusing
 // them as contexts
 type imageContexts struct {
 	b           *Builder
 	list        []*imageMount
 	byName      map[string]*imageMount
-	cache       *pathCache
+	cache       pathCache
 	currentName string
 }
 
@@ -104,14 +108,14 @@ func (ic *imageContexts) getCache(id, path string) (interface{}, bool) {
 		if id == "" {
 			return nil, false
 		}
-		return ic.cache.get(id + path)
+		return ic.cache.Load(id + path)
 	}
 	return nil, false
 }
 
 func (ic *imageContexts) setCache(id, path string, v interface{}) {
 	if ic.cache != nil {
-		ic.cache.set(id+path, v)
+		ic.cache.Store(id+path, v)
 	}
 }
 
@@ -160,28 +164,3 @@ func (im *imageMount) ImageID() string {
 func (im *imageMount) RunConfig() *container.Config {
 	return im.runConfig
 }
-
-type pathCache struct {
-	mu    sync.Mutex
-	items map[string]interface{}
-}
-
-func (c *pathCache) set(k string, v interface{}) {
-	c.mu.Lock()
-	if c.items == nil {
-		c.items = make(map[string]interface{})
-	}
-	c.items[k] = v
-	c.mu.Unlock()
-}
-
-func (c *pathCache) get(k string) (interface{}, bool) {
-	c.mu.Lock()
-	if c.items == nil {
-		c.mu.Unlock()
-		return nil, false
-	}
-	v, ok := c.items[k]
-	c.mu.Unlock()
-	return v, ok
-}

builder/dockerfile/internals_test.go

@@ -1,10 +1,11 @@
 package dockerfile
 
 import (
-	"context"
 	"fmt"
 	"testing"
 
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/remotecontext"
@@ -69,7 +70,11 @@ func readAndCheckDockerfile(t *testing.T, testName, contextDir, dockerfilePath,
 		dockerfilePath = builder.DefaultDockerfileName
 	}
 
-	_, _, err = remotecontext.Detect(context.Background(), "", dockerfilePath, tarStream, nil)
+	config := backend.BuildConfig{
+		Options: &types.ImageBuildOptions{Dockerfile: dockerfilePath},
+		Source:  tarStream,
+	}
+	_, _, err = remotecontext.Detect(config)
 	assert.EqualError(t, err, expectedError)
 }
 

builder/remotecontext/detect.go

@@ -2,7 +2,6 @@ package remotecontext
 
 import (
 	"bufio"
-	"context"
 	"fmt"
 	"io"
 	"os"
@@ -10,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/dockerfile/parser"
 	"github.com/docker/docker/builder/dockerignore"
@@ -23,14 +23,17 @@ import (
 // Detect returns a context and dockerfile from remote location or local
 // archive. progressReader is only used if remoteURL is actually a URL
 // (not empty, and not a Git endpoint).
-func Detect(ctx context.Context, remoteURL string, dockerfilePath string, r io.ReadCloser, progressReader func(in io.ReadCloser) io.ReadCloser) (remote builder.Source, dockerfile *parser.Result, err error) {
+func Detect(config backend.BuildConfig) (remote builder.Source, dockerfile *parser.Result, err error) {
+	remoteURL := config.Options.RemoteContext
+	dockerfilePath := config.Options.Dockerfile
+
 	switch {
 	case remoteURL == "":
-		remote, dockerfile, err = newArchiveRemote(r, dockerfilePath)
+		remote, dockerfile, err = newArchiveRemote(config.Source, dockerfilePath)
 	case urlutil.IsGitURL(remoteURL):
 		remote, dockerfile, err = newGitRemote(remoteURL, dockerfilePath)
 	case urlutil.IsURL(remoteURL):
-		remote, dockerfile, err = newURLRemote(remoteURL, dockerfilePath, progressReader)
+		remote, dockerfile, err = newURLRemote(remoteURL, dockerfilePath, config.ProgressWriter.ProgressReaderFunc)
 	default:
 		err = fmt.Errorf("remoteURL (%s) could not be recognized as URL", remoteURL)
 	}

cmd/dockerd/daemon.go

@@ -14,6 +14,7 @@ import (
 	"github.com/docker/distribution/uuid"
 	"github.com/docker/docker/api"
 	apiserver "github.com/docker/docker/api/server"
+	buildbackend "github.com/docker/docker/api/server/backend/build"
 	"github.com/docker/docker/api/server/middleware"
 	"github.com/docker/docker/api/server/router"
 	"github.com/docker/docker/api/server/router/build"
@@ -25,7 +26,6 @@ import (
 	swarmrouter "github.com/docker/docker/api/server/router/swarm"
 	systemrouter "github.com/docker/docker/api/server/router/system"
 	"github.com/docker/docker/api/server/router/volume"
-	"github.com/docker/docker/builder/dockerfile"
 	cliconfig "github.com/docker/docker/cli/config"
 	"github.com/docker/docker/cli/debug"
 	cliflags "github.com/docker/docker/cli/flags"
@@ -484,7 +484,7 @@ func initRouter(s *apiserver.Server, d *daemon.Daemon, c *cluster.Cluster) {
 		image.NewRouter(d, decoder),
 		systemrouter.NewRouter(d, c),
 		volume.NewRouter(d),
-		build.NewRouter(dockerfile.NewBuildManager(d)),
+		build.NewRouter(buildbackend.NewBackend(d, d), d),
 		swarmrouter.NewRouter(c),
 		pluginrouter.NewRouter(d.PluginManager()),
 	}

vendor.conf

@@ -22,6 +22,7 @@ github.com/stretchr/testify 4d4bfba8f1d1027c4fdbe371823030df51419987
 
 github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5
 github.com/imdario/mergo 0.2.1
+golang.org/x/sync/syncmap de49d9dcd27d4f764488181bea099dfe6179bcf0
 
 #get libnetwork packages
 github.com/docker/libnetwork cace103704768d39bd88a23d0df76df125a0e39a

vendor/golang.org/x/sync/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/golang.org/x/sync/PATENTS

@@ -0,0 +1,22 @@
+Additional IP Rights Grant (Patents)
+
+"This implementation" means the copyrightable works distributed by
+Google as part of the Go project.
+
+Google hereby grants to You a perpetual, worldwide, non-exclusive,
+no-charge, royalty-free, irrevocable (except as stated in this section)
+patent license to make, have made, use, offer to sell, sell, import,
+transfer and otherwise run, modify and propagate the contents of this
+implementation of Go, where such license applies only to those patent
+claims, both currently owned or controlled by Google and acquired in
+the future, licensable by Google that are necessarily infringed by this
+implementation of Go.  This grant does not include claims that would be
+infringed only as a consequence of further modification of this
+implementation.  If you or your agent or exclusive licensee institute or
+order or agree to the institution of patent litigation against any
+entity (including a cross-claim or counterclaim in a lawsuit) alleging
+that this implementation of Go or any code incorporated within this
+implementation of Go constitutes direct or contributory patent
+infringement, or inducement of patent infringement, then any patent
+rights granted to you under this License for this implementation of Go
+shall terminate as of the date such litigation is filed.

vendor/golang.org/x/sync/README

@@ -0,0 +1,2 @@
+This repository provides Go concurrency primitives in addition to the
+ones provided by the language and "sync" and "sync/atomic" packages.

vendor/golang.org/x/sync/syncmap/map.go

@@ -0,0 +1,372 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package syncmap provides a concurrent map implementation.
+// It is a prototype for a proposed addition to the sync package
+// in the standard library.
+// (https://golang.org/issue/18177)
+package syncmap
+
+import (
+	"sync"
+	"sync/atomic"
+	"unsafe"
+)
+
+// Map is a concurrent map with amortized-constant-time loads, stores, and deletes.
+// It is safe for multiple goroutines to call a Map's methods concurrently.
+//
+// The zero Map is valid and empty.
+//
+// A Map must not be copied after first use.
+type Map struct {
+	mu sync.Mutex
+
+	// read contains the portion of the map's contents that are safe for
+	// concurrent access (with or without mu held).
+	//
+	// The read field itself is always safe to load, but must only be stored with
+	// mu held.
+	//
+	// Entries stored in read may be updated concurrently without mu, but updating
+	// a previously-expunged entry requires that the entry be copied to the dirty
+	// map and unexpunged with mu held.
+	read atomic.Value // readOnly
+
+	// dirty contains the portion of the map's contents that require mu to be
+	// held. To ensure that the dirty map can be promoted to the read map quickly,
+	// it also includes all of the non-expunged entries in the read map.
+	//
+	// Expunged entries are not stored in the dirty map. An expunged entry in the
+	// clean map must be unexpunged and added to the dirty map before a new value
+	// can be stored to it.
+	//
+	// If the dirty map is nil, the next write to the map will initialize it by
+	// making a shallow copy of the clean map, omitting stale entries.
+	dirty map[interface{}]*entry
+
+	// misses counts the number of loads since the read map was last updated that
+	// needed to lock mu to determine whether the key was present.
+	//
+	// Once enough misses have occurred to cover the cost of copying the dirty
+	// map, the dirty map will be promoted to the read map (in the unamended
+	// state) and the next store to the map will make a new dirty copy.
+	misses int
+}
+
+// readOnly is an immutable struct stored atomically in the Map.read field.
+type readOnly struct {
+	m       map[interface{}]*entry
+	amended bool // true if the dirty map contains some key not in m.
+}
+
+// expunged is an arbitrary pointer that marks entries which have been deleted
+// from the dirty map.
+var expunged = unsafe.Pointer(new(interface{}))
+
+// An entry is a slot in the map corresponding to a particular key.
+type entry struct {
+	// p points to the interface{} value stored for the entry.
+	//
+	// If p == nil, the entry has been deleted and m.dirty == nil.
+	//
+	// If p == expunged, the entry has been deleted, m.dirty != nil, and the entry
+	// is missing from m.dirty.
+	//
+	// Otherwise, the entry is valid and recorded in m.read.m[key] and, if m.dirty
+	// != nil, in m.dirty[key].
+	//
+	// An entry can be deleted by atomic replacement with nil: when m.dirty is
+	// next created, it will atomically replace nil with expunged and leave
+	// m.dirty[key] unset.
+	//
+	// An entry's associated value can be updated by atomic replacement, provided
+	// p != expunged. If p == expunged, an entry's associated value can be updated
+	// only after first setting m.dirty[key] = e so that lookups using the dirty
+	// map find the entry.
+	p unsafe.Pointer // *interface{}
+}
+
+func newEntry(i interface{}) *entry {
+	return &entry{p: unsafe.Pointer(&i)}
+}
+
+// Load returns the value stored in the map for a key, or nil if no
+// value is present.
+// The ok result indicates whether value was found in the map.
+func (m *Map) Load(key interface{}) (value interface{}, ok bool) {
+	read, _ := m.read.Load().(readOnly)
+	e, ok := read.m[key]
+	if !ok && read.amended {
+		m.mu.Lock()
+		// Avoid reporting a spurious miss if m.dirty got promoted while we were
+		// blocked on m.mu. (If further loads of the same key will not miss, it's
+		// not worth copying the dirty map for this key.)
+		read, _ = m.read.Load().(readOnly)
+		e, ok = read.m[key]
+		if !ok && read.amended {
+			e, ok = m.dirty[key]
+			// Regardless of whether the entry was present, record a miss: this key
+			// will take the slow path until the dirty map is promoted to the read
+			// map.
+			m.missLocked()
+		}
+		m.mu.Unlock()
+	}
+	if !ok {
+		return nil, false
+	}
+	return e.load()
+}
+
+func (e *entry) load() (value interface{}, ok bool) {
+	p := atomic.LoadPointer(&e.p)
+	if p == nil || p == expunged {
+		return nil, false
+	}
+	return *(*interface{})(p), true
+}
+
+// Store sets the value for a key.
+func (m *Map) Store(key, value interface{}) {
+	read, _ := m.read.Load().(readOnly)
+	if e, ok := read.m[key]; ok && e.tryStore(&value) {
+		return
+	}
+
+	m.mu.Lock()
+	read, _ = m.read.Load().(readOnly)
+	if e, ok := read.m[key]; ok {
+		if e.unexpungeLocked() {
+			// The entry was previously expunged, which implies that there is a
+			// non-nil dirty map and this entry is not in it.
+			m.dirty[key] = e
+		}
+		e.storeLocked(&value)
+	} else if e, ok := m.dirty[key]; ok {
+		e.storeLocked(&value)
+	} else {
+		if !read.amended {
+			// We're adding the first new key to the dirty map.
+			// Make sure it is allocated and mark the read-only map as incomplete.
+			m.dirtyLocked()
+			m.read.Store(readOnly{m: read.m, amended: true})
+		}
+		m.dirty[key] = newEntry(value)
+	}
+	m.mu.Unlock()
+}
+
+// tryStore stores a value if the entry has not been expunged.
+//
+// If the entry is expunged, tryStore returns false and leaves the entry
+// unchanged.
+func (e *entry) tryStore(i *interface{}) bool {
+	p := atomic.LoadPointer(&e.p)
+	if p == expunged {
+		return false
+	}
+	for {
+		if atomic.CompareAndSwapPointer(&e.p, p, unsafe.Pointer(i)) {
+			return true
+		}
+		p = atomic.LoadPointer(&e.p)
+		if p == expunged {
+			return false
+		}
+	}
+}
+
+// unexpungeLocked ensures that the entry is not marked as expunged.
+//
+// If the entry was previously expunged, it must be added to the dirty map
+// before m.mu is unlocked.
+func (e *entry) unexpungeLocked() (wasExpunged bool) {
+	return atomic.CompareAndSwapPointer(&e.p, expunged, nil)
+}
+
+// storeLocked unconditionally stores a value to the entry.
+//
+// The entry must be known not to be expunged.
+func (e *entry) storeLocked(i *interface{}) {
+	atomic.StorePointer(&e.p, unsafe.Pointer(i))
+}
+
+// LoadOrStore returns the existing value for the key if present.
+// Otherwise, it stores and returns the given value.
+// The loaded result is true if the value was loaded, false if stored.
+func (m *Map) LoadOrStore(key, value interface{}) (actual interface{}, loaded bool) {
+	// Avoid locking if it's a clean hit.
+	read, _ := m.read.Load().(readOnly)
+	if e, ok := read.m[key]; ok {
+		actual, loaded, ok := e.tryLoadOrStore(value)
+		if ok {
+			return actual, loaded
+		}
+	}
+
+	m.mu.Lock()
+	read, _ = m.read.Load().(readOnly)
+	if e, ok := read.m[key]; ok {
+		if e.unexpungeLocked() {
+			m.dirty[key] = e
+		}
+		actual, loaded, _ = e.tryLoadOrStore(value)
+	} else if e, ok := m.dirty[key]; ok {
+		actual, loaded, _ = e.tryLoadOrStore(value)
+		m.missLocked()
+	} else {
+		if !read.amended {
+			// We're adding the first new key to the dirty map.
+			// Make sure it is allocated and mark the read-only map as incomplete.
+			m.dirtyLocked()
+			m.read.Store(readOnly{m: read.m, amended: true})
+		}
+		m.dirty[key] = newEntry(value)
+		actual, loaded = value, false
+	}
+	m.mu.Unlock()
+
+	return actual, loaded
+}
+
+// tryLoadOrStore atomically loads or stores a value if the entry is not
+// expunged.
+//
+// If the entry is expunged, tryLoadOrStore leaves the entry unchanged and
+// returns with ok==false.
+func (e *entry) tryLoadOrStore(i interface{}) (actual interface{}, loaded, ok bool) {
+	p := atomic.LoadPointer(&e.p)
+	if p == expunged {
+		return nil, false, false
+	}
+	if p != nil {
+		return *(*interface{})(p), true, true
+	}
+
+	// Copy the interface after the first load to make this method more amenable
+	// to escape analysis: if we hit the "load" path or the entry is expunged, we
+	// shouldn't bother heap-allocating.
+	ic := i
+	for {
+		if atomic.CompareAndSwapPointer(&e.p, nil, unsafe.Pointer(&ic)) {
+			return i, false, true
+		}
+		p = atomic.LoadPointer(&e.p)
+		if p == expunged {
+			return nil, false, false
+		}
+		if p != nil {
+			return *(*interface{})(p), true, true
+		}
+	}
+}
+
+// Delete deletes the value for a key.
+func (m *Map) Delete(key interface{}) {
+	read, _ := m.read.Load().(readOnly)
+	e, ok := read.m[key]
+	if !ok && read.amended {
+		m.mu.Lock()
+		read, _ = m.read.Load().(readOnly)
+		e, ok = read.m[key]
+		if !ok && read.amended {
+			delete(m.dirty, key)
+		}
+		m.mu.Unlock()
+	}
+	if ok {
+		e.delete()
+	}
+}
+
+func (e *entry) delete() (hadValue bool) {
+	for {
+		p := atomic.LoadPointer(&e.p)
+		if p == nil || p == expunged {
+			return false
+		}
+		if atomic.CompareAndSwapPointer(&e.p, p, nil) {
+			return true
+		}
+	}
+}
+
+// Range calls f sequentially for each key and value present in the map.
+// If f returns false, range stops the iteration.
+//
+// Range does not necessarily correspond to any consistent snapshot of the Map's
+// contents: no key will be visited more than once, but if the value for any key
+// is stored or deleted concurrently, Range may reflect any mapping for that key
+// from any point during the Range call.
+//
+// Range may be O(N) with the number of elements in the map even if f returns
+// false after a constant number of calls.
+func (m *Map) Range(f func(key, value interface{}) bool) {
+	// We need to be able to iterate over all of the keys that were already
+	// present at the start of the call to Range.
+	// If read.amended is false, then read.m satisfies that property without
+	// requiring us to hold m.mu for a long time.
+	read, _ := m.read.Load().(readOnly)
+	if read.amended {
+		// m.dirty contains keys not in read.m. Fortunately, Range is already O(N)
+		// (assuming the caller does not break out early), so a call to Range
+		// amortizes an entire copy of the map: we can promote the dirty copy
+		// immediately!
+		m.mu.Lock()
+		read, _ = m.read.Load().(readOnly)
+		if read.amended {
+			read = readOnly{m: m.dirty}
+			m.read.Store(read)
+			m.dirty = nil
+			m.misses = 0
+		}
+		m.mu.Unlock()
+	}
+
+	for k, e := range read.m {
+		v, ok := e.load()
+		if !ok {
+			continue
+		}
+		if !f(k, v) {
+			break
+		}
+	}
+}
+
+func (m *Map) missLocked() {
+	m.misses++
+	if m.misses < len(m.dirty) {
+		return
+	}
+	m.read.Store(readOnly{m: m.dirty})
+	m.dirty = nil
+	m.misses = 0
+}
+
+func (m *Map) dirtyLocked() {
+	if m.dirty != nil {
+		return
+	}
+
+	read, _ := m.read.Load().(readOnly)
+	m.dirty = make(map[interface{}]*entry, len(read.m))
+	for k, e := range read.m {
+		if !e.tryExpungeLocked() {
+			m.dirty[k] = e
+		}
+	}
+}
+
+func (e *entry) tryExpungeLocked() (isExpunged bool) {
+	p := atomic.LoadPointer(&e.p)
+	for p == nil {
+		if atomic.CompareAndSwapPointer(&e.p, nil, expunged) {
+			return true
+		}
+		p = atomic.LoadPointer(&e.p)
+	}
+	return p == expunged
+}