From 9c332b164f1aefa2407706adf59d50495d6e02cb Mon Sep 17 00:00:00 2001 From: Anusha Ragunathan Date: Wed, 20 Jan 2016 15:32:02 -0800 Subject: [PATCH] Remove package daemonbuilder. Currently, daemonbuilder package (part of daemon) implemented the builder backend. However, it was a very thin wrapper around daemon methods and caused an implementation dependency for api/server build endpoint. api/server buildrouter should only know about the backend implementing the /build API endpoint. Removing daemonbuilder involved moving build specific methods to respective files in the daemon, where they fit naturally. Signed-off-by: Anusha Ragunathan --- api/server/router/build/backend.go | 8 +- api/server/router/build/build.go | 5 +- api/server/router/build/build_routes.go | 106 ++--------- api/server/server.go | 3 +- builder/builder.go | 22 ++- builder/dockerfile/builder.go | 102 +++++++++- builder/dockerfile/dispatchers.go | 4 +- builder/dockerfile/internals.go | 14 +- builder/image.go | 9 - builder/remote.go | 38 ++++ daemon/archive.go | 99 ++++++++++ daemon/archive_unix.go | 38 +++- daemon/archive_windows.go | 5 + daemon/attach.go | 10 + daemon/daemon.go | 53 +++++- daemon/daemonbuilder/builder.go | 235 ------------------------ daemon/daemonbuilder/builder_unix.go | 40 ---- daemon/daemonbuilder/builder_windows.go | 8 - daemon/daemonbuilder/image.go | 18 -- daemon/update.go | 11 ++ image/image.go | 10 + 21 files changed, 411 insertions(+), 427 deletions(-) delete mode 100644 builder/image.go delete mode 100644 daemon/daemonbuilder/builder.go delete mode 100644 daemon/daemonbuilder/builder_unix.go delete mode 100644 daemon/daemonbuilder/builder_windows.go delete mode 100644 daemon/daemonbuilder/image.go diff --git a/api/server/router/build/backend.go b/api/server/router/build/backend.go index fd9e314a03..6d8fca3c62 100644 --- a/api/server/router/build/backend.go +++ b/api/server/router/build/backend.go @@ -1,5 +1,11 @@ package build +import ( + "github.com/docker/docker/builder" + "github.com/docker/engine-api/types" + "io" +) + // Backend abstracts an image builder whose only purpose is to build an image referenced by an imageID. type Backend interface { // Build builds a Docker image referenced by an imageID string. @@ -8,5 +14,5 @@ type Backend interface { // by the caller. // // TODO: make this return a reference instead of string - Build() (imageID string) + Build(config *types.ImageBuildOptions, context builder.Context, stdout io.Writer, stderr io.Writer, out io.Writer, clientGone <-chan bool) (string, error) } diff --git a/api/server/router/build/build.go b/api/server/router/build/build.go index e21b634609..166f97bc66 100644 --- a/api/server/router/build/build.go +++ b/api/server/router/build/build.go @@ -3,17 +3,16 @@ package build import ( "github.com/docker/docker/api/server/router" "github.com/docker/docker/api/server/router/local" - "github.com/docker/docker/daemon" ) // buildRouter is a router to talk with the build controller type buildRouter struct { - backend *daemon.Daemon + backend Backend routes []router.Route } // NewRouter initializes a new build router -func NewRouter(b *daemon.Daemon) router.Router { +func NewRouter(b Backend) router.Router { r := &buildRouter{ backend: b, } diff --git a/api/server/router/build/build_routes.go b/api/server/router/build/build_routes.go index 5cda79c3cc..152dfff328 100644 --- a/api/server/router/build/build_routes.go +++ b/api/server/router/build/build_routes.go @@ -14,12 +14,9 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/docker/api/server/httputils" "github.com/docker/docker/builder" - "github.com/docker/docker/builder/dockerfile" - "github.com/docker/docker/daemon/daemonbuilder" "github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/streamformatter" - "github.com/docker/docker/reference" "github.com/docker/docker/utils" "github.com/docker/engine-api/types" "github.com/docker/engine-api/types/container" @@ -27,45 +24,6 @@ import ( "golang.org/x/net/context" ) -// sanitizeRepoAndTags parses the raw "t" parameter received from the client -// to a slice of repoAndTag. -// It also validates each repoName and tag. -func sanitizeRepoAndTags(names []string) ([]reference.Named, error) { - var ( - repoAndTags []reference.Named - // This map is used for deduplicating the "-t" parameter. - uniqNames = make(map[string]struct{}) - ) - for _, repo := range names { - if repo == "" { - continue - } - - ref, err := reference.ParseNamed(repo) - if err != nil { - return nil, err - } - - ref = reference.WithDefaultTag(ref) - - if _, isCanonical := ref.(reference.Canonical); isCanonical { - return nil, errors.New("build tag cannot contain a digest") - } - - if _, isTagged := ref.(reference.NamedTagged); !isTagged { - ref, err = reference.WithTag(ref, reference.DefaultTag) - } - - nameWithTag := ref.String() - - if _, exists := uniqNames[nameWithTag]; !exists { - uniqNames[nameWithTag] = struct{}{} - repoAndTags = append(repoAndTags, ref) - } - } - return repoAndTags, nil -} - func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBuildOptions, error) { version := httputils.VersionFromContext(ctx) options := &types.ImageBuildOptions{} @@ -92,6 +50,7 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui options.CPUSetCPUs = r.FormValue("cpusetcpus") options.CPUSetMems = r.FormValue("cpusetmems") options.CgroupParent = r.FormValue("cgroupparent") + options.Tags = r.Form["t"] if r.Form.Get("shmsize") != "" { shmSize, err := strconv.ParseInt(r.Form.Get("shmsize"), 10, 64) @@ -170,11 +129,6 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r * return errf(err) } - repoAndTags, err := sanitizeRepoAndTags(r.Form["t"]) - if err != nil { - return errf(err) - } - remoteURL := r.FormValue("remote") // Currently, only used if context is from a remote url. @@ -190,8 +144,9 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r * var ( context builder.ModifiableContext dockerfileName string + out io.Writer ) - context, dockerfileName, err = daemonbuilder.DetectContextFromRemoteURL(r.Body, remoteURL, createProgressReader) + context, dockerfileName, err = builder.DetectContextFromRemoteURL(r.Body, remoteURL, createProgressReader) if err != nil { return errf(err) } @@ -204,50 +159,25 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r * buildOptions.Dockerfile = dockerfileName } - b, err := dockerfile.NewBuilder( - buildOptions, // result of newBuildConfig - &daemonbuilder.Docker{br.backend}, + out = output + if buildOptions.SuppressOutput { + out = notVerboseBuffer + } + stdout := &streamformatter.StdoutFormatter{Writer: out, StreamFormatter: sf} + stderr := &streamformatter.StderrFormatter{Writer: out, StreamFormatter: sf} + + closeNotifier := make(<-chan bool) + if notifier, ok := w.(http.CloseNotifier); ok { + closeNotifier = notifier.CloseNotify() + } + + imgID, err := br.backend.Build(buildOptions, builder.DockerIgnoreContext{ModifiableContext: context}, - nil) + stdout, stderr, out, + closeNotifier) if err != nil { return errf(err) } - if buildOptions.SuppressOutput { - b.Output = notVerboseBuffer - } else { - b.Output = output - } - b.Stdout = &streamformatter.StdoutFormatter{Writer: output, StreamFormatter: sf} - b.Stderr = &streamformatter.StderrFormatter{Writer: output, StreamFormatter: sf} - if buildOptions.SuppressOutput { - b.Stdout = &streamformatter.StdoutFormatter{Writer: notVerboseBuffer, StreamFormatter: sf} - b.Stderr = &streamformatter.StderrFormatter{Writer: notVerboseBuffer, StreamFormatter: sf} - } - - if closeNotifier, ok := w.(http.CloseNotifier); ok { - finished := make(chan struct{}) - defer close(finished) - clientGone := closeNotifier.CloseNotify() - go func() { - select { - case <-finished: - case <-clientGone: - logrus.Infof("Client disconnected, cancelling job: build") - b.Cancel() - } - }() - } - - imgID, err := b.Build() - if err != nil { - return errf(err) - } - - for _, rt := range repoAndTags { - if err := br.backend.TagImage(rt, imgID); err != nil { - return errf(err) - } - } // Everything worked so if -q was provided the output from the daemon // should be just the image ID and we'll print that to stdout. diff --git a/api/server/server.go b/api/server/server.go index aa7905b437..a42cd1720b 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -15,6 +15,7 @@ import ( "github.com/docker/docker/api/server/router/network" "github.com/docker/docker/api/server/router/system" "github.com/docker/docker/api/server/router/volume" + "github.com/docker/docker/builder/dockerfile" "github.com/docker/docker/daemon" "github.com/docker/docker/pkg/authorization" "github.com/docker/docker/utils" @@ -180,7 +181,7 @@ func (s *Server) InitRouters(d *daemon.Daemon) { s.addRouter(network.NewRouter(d)) s.addRouter(system.NewRouter(d)) s.addRouter(volume.NewRouter(d)) - s.addRouter(build.NewRouter(d)) + s.addRouter(build.NewRouter(dockerfile.NewBuildManager(d))) } // addRouter adds a new router to the server. diff --git a/builder/builder.go b/builder/builder.go index 5ef8692950..da7aaf251e 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -9,6 +9,7 @@ import ( "os" "time" + "github.com/docker/docker/reference" "github.com/docker/engine-api/types" "github.com/docker/engine-api/types/container" ) @@ -99,11 +100,13 @@ type Backend interface { // TODO: use digest reference instead of name // GetImage looks up a Docker image referenced by `name`. - GetImage(name string) (Image, error) + GetImageOnBuild(name string) (Image, error) + // Tag an image with newTag + TagImage(newTag reference.Named, imageName string) error // Pull tells Docker to pull image referenced by `name`. - Pull(name string, authConfigs map[string]types.AuthConfig, output io.Writer) (Image, error) + PullOnBuild(name string, authConfigs map[string]types.AuthConfig, output io.Writer) (Image, error) // ContainerAttach attaches to container. - ContainerAttach(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool) error + ContainerAttachOnBuild(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool) error // ContainerCreate creates a new Docker container and returns potential warnings ContainerCreate(types.ContainerCreateConfig) (types.ContainerCreateResponse, error) // ContainerRm removes a container specified by `id`. @@ -116,9 +119,8 @@ type Backend interface { ContainerStart(containerID string, hostConfig *container.HostConfig) error // ContainerWait stops processing until the given container is stopped. ContainerWait(containerID string, timeout time.Duration) (int, error) - // ContainerUpdateCmd updates container.Path and container.Args - ContainerUpdateCmd(containerID string, cmd []string) error + ContainerUpdateCmdOnBuild(containerID string, cmd []string) error // ContainerCopy copies/extracts a source FileInfo to a destination path inside a container // specified by a container object. @@ -127,7 +129,13 @@ type Backend interface { // with Context.Walk //ContainerCopy(name string, res string) (io.ReadCloser, error) // TODO: use copyBackend api - BuilderCopy(containerID string, destPath string, src FileInfo, decompress bool) error + CopyOnBuild(containerID string, destPath string, src FileInfo, decompress bool) error +} + +// Image represents a Docker image used by the builder. +type Image interface { + ImageID() string + RunConfig() *container.Config } // ImageCache abstracts an image cache store. @@ -135,5 +143,5 @@ type Backend interface { type ImageCache interface { // GetCachedImage returns a reference to a cached image whose parent equals `parent` // and runconfig equals `cfg`. A cache miss is expected to return an empty ID and a nil error. - GetCachedImage(parentID string, cfg *container.Config) (imageID string, err error) + GetCachedImageOnBuild(parentID string, cfg *container.Config) (imageID string, err error) } diff --git a/builder/dockerfile/builder.go b/builder/dockerfile/builder.go index bbf9c7a578..f961a6ab60 100644 --- a/builder/dockerfile/builder.go +++ b/builder/dockerfile/builder.go @@ -2,6 +2,7 @@ package dockerfile import ( "bytes" + "errors" "fmt" "io" "io/ioutil" @@ -13,6 +14,7 @@ import ( "github.com/docker/docker/builder" "github.com/docker/docker/builder/dockerfile/parser" "github.com/docker/docker/pkg/stringid" + "github.com/docker/docker/reference" "github.com/docker/engine-api/types" "github.com/docker/engine-api/types/container" ) @@ -48,6 +50,7 @@ type Builder struct { Stdout io.Writer Stderr io.Writer + Output io.Writer docker builder.Backend context builder.Context @@ -67,8 +70,17 @@ type Builder struct { allowedBuildArgs map[string]bool // list of build-time args that are allowed for expansion/substitution and passing to commands in 'run'. // TODO: remove once docker.Commit can receive a tag - id string - Output io.Writer + id string +} + +// BuildManager implements builder.Backend and is shared across all Builder objects. +type BuildManager struct { + backend builder.Backend +} + +// NewBuildManager creates a BuildManager. +func NewBuildManager(b builder.Backend) (bm *BuildManager) { + return &BuildManager{backend: b} } // NewBuilder creates a new Dockerfile builder from an optional dockerfile and a Config. @@ -103,7 +115,57 @@ func NewBuilder(config *types.ImageBuildOptions, backend builder.Backend, contex return b, nil } -// Build runs the Dockerfile builder from a context and a docker object that allows to make calls +// sanitizeRepoAndTags parses the raw "t" parameter received from the client +// to a slice of repoAndTag. +// It also validates each repoName and tag. +func sanitizeRepoAndTags(names []string) ([]reference.Named, error) { + var ( + repoAndTags []reference.Named + // This map is used for deduplicating the "-t" parameter. + uniqNames = make(map[string]struct{}) + ) + for _, repo := range names { + if repo == "" { + continue + } + + ref, err := reference.ParseNamed(repo) + if err != nil { + return nil, err + } + + ref = reference.WithDefaultTag(ref) + + if _, isCanonical := ref.(reference.Canonical); isCanonical { + return nil, errors.New("build tag cannot contain a digest") + } + + if _, isTagged := ref.(reference.NamedTagged); !isTagged { + ref, err = reference.WithTag(ref, reference.DefaultTag) + } + + nameWithTag := ref.String() + + if _, exists := uniqNames[nameWithTag]; !exists { + uniqNames[nameWithTag] = struct{}{} + repoAndTags = append(repoAndTags, ref) + } + } + return repoAndTags, nil +} + +// Build creates a NewBuilder, which builds the image. +func (bm *BuildManager) Build(config *types.ImageBuildOptions, context builder.Context, stdout io.Writer, stderr io.Writer, out io.Writer, clientGone <-chan bool) (string, error) { + b, err := NewBuilder(config, bm.backend, context, nil) + if err != nil { + return "", err + } + img, err := b.build(config, context, stdout, stderr, out, clientGone) + return img, err + +} + +// build runs the Dockerfile builder from a context and a docker object that allows to make calls // to Docker. // // This will (barring errors): @@ -113,10 +175,16 @@ func NewBuilder(config *types.ImageBuildOptions, backend builder.Backend, contex // * walk the AST and execute it by dispatching to handlers. If Remove // or ForceRemove is set, additional cleanup around containers happens after // processing. +// * Tag image, if applicable. // * Print a happy message and return the image ID. -// * NOT tag the image, that is responsibility of the caller. // -func (b *Builder) Build() (string, error) { +func (b *Builder) build(config *types.ImageBuildOptions, context builder.Context, stdout io.Writer, stderr io.Writer, out io.Writer, clientGone <-chan bool) (string, error) { + b.options = config + b.context = context + b.Stdout = stdout + b.Stderr = stderr + b.Output = out + // If Dockerfile was not parsed yet, extract it from the Context if b.dockerfile == nil { if err := b.readDockerfile(); err != nil { @@ -124,6 +192,24 @@ func (b *Builder) Build() (string, error) { } } + finished := make(chan struct{}) + defer close(finished) + go func() { + select { + case <-finished: + case <-clientGone: + b.cancelOnce.Do(func() { + close(b.cancelled) + }) + } + + }() + + repoAndTags, err := sanitizeRepoAndTags(config.Tags) + if err != nil { + return "", err + } + var shortImgID string for i, n := range b.dockerfile.Children { select { @@ -163,6 +249,12 @@ func (b *Builder) Build() (string, error) { return "", fmt.Errorf("No image was generated. Is your Dockerfile empty?") } + for _, rt := range repoAndTags { + if err := b.docker.TagImage(rt, b.image); err != nil { + return "", err + } + } + fmt.Fprintf(b.Stdout, "Successfully built %s\n", shortImgID) return b.image, nil } diff --git a/builder/dockerfile/dispatchers.go b/builder/dockerfile/dispatchers.go index 7094cfb927..f800acbe8e 100644 --- a/builder/dockerfile/dispatchers.go +++ b/builder/dockerfile/dispatchers.go @@ -208,11 +208,11 @@ func from(b *Builder, args []string, attributes map[string]bool, original string } else { // TODO: don't use `name`, instead resolve it to a digest if !b.options.PullParent { - image, err = b.docker.GetImage(name) + image, err = b.docker.GetImageOnBuild(name) // TODO: shouldn't we error out if error is different from "not found" ? } if image == nil { - image, err = b.docker.Pull(name, b.options.AuthConfigs, b.Output) + image, err = b.docker.PullOnBuild(name, b.options.AuthConfigs, b.Output) if err != nil { return err } diff --git a/builder/dockerfile/internals.go b/builder/dockerfile/internals.go index bf5880f093..c5549820e7 100644 --- a/builder/dockerfile/internals.go +++ b/builder/dockerfile/internals.go @@ -205,7 +205,7 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalD } for _, info := range infos { - if err := b.docker.BuilderCopy(container.ID, dest, info.FileInfo, info.decompress); err != nil { + if err := b.docker.CopyOnBuild(container.ID, dest, info.FileInfo, info.decompress); err != nil { return err } } @@ -396,10 +396,10 @@ func containsWildcards(name string) bool { func (b *Builder) processImageFrom(img builder.Image) error { if img != nil { - b.image = img.ID() + b.image = img.ImageID() - if img.Config() != nil { - b.runConfig = img.Config() + if img.RunConfig() != nil { + b.runConfig = img.RunConfig() } } @@ -469,7 +469,7 @@ func (b *Builder) probeCache() (bool, error) { if !ok || b.options.NoCache || b.cacheBusted { return false, nil } - cache, err := c.GetCachedImage(b.image, b.runConfig) + cache, err := c.GetCachedImageOnBuild(b.image, b.runConfig) if err != nil { return false, err } @@ -530,7 +530,7 @@ func (b *Builder) create() (string, error) { if config.Cmd.Len() > 0 { // override the entry point that may have been picked up from the base image - if err := b.docker.ContainerUpdateCmd(c.ID, config.Cmd.Slice()); err != nil { + if err := b.docker.ContainerUpdateCmdOnBuild(c.ID, config.Cmd.Slice()); err != nil { return "", err } } @@ -541,7 +541,7 @@ func (b *Builder) create() (string, error) { func (b *Builder) run(cID string) (err error) { errCh := make(chan error) go func() { - errCh <- b.docker.ContainerAttach(cID, nil, b.Stdout, b.Stderr, true) + errCh <- b.docker.ContainerAttachOnBuild(cID, nil, b.Stdout, b.Stderr, true) }() finished := make(chan struct{}) diff --git a/builder/image.go b/builder/image.go deleted file mode 100644 index 2e545cfa9d..0000000000 --- a/builder/image.go +++ /dev/null @@ -1,9 +0,0 @@ -package builder - -import "github.com/docker/engine-api/types/container" - -// Image represents a Docker image used by the builder. -type Image interface { - ID() string - Config() *container.Config -} diff --git a/builder/remote.go b/builder/remote.go index b43bd32841..3ab5923109 100644 --- a/builder/remote.go +++ b/builder/remote.go @@ -8,7 +8,10 @@ import ( "io/ioutil" "regexp" + "github.com/docker/docker/api" + "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/httputils" + "github.com/docker/docker/pkg/urlutil" ) // When downloading remote contexts, limit the amount (in bytes) @@ -65,6 +68,41 @@ func MakeRemoteContext(remoteURL string, contentTypeHandlers map[string]func(io. return MakeTarSumContext(contextReader) } +// DetectContextFromRemoteURL returns a context and in certain cases the name of the dockerfile to be used +// irrespective of user input. +// progressReader is only used if remoteURL is actually a URL (not empty, and not a Git endpoint). +func DetectContextFromRemoteURL(r io.ReadCloser, remoteURL string, createProgressReader func(in io.ReadCloser) io.ReadCloser) (context ModifiableContext, dockerfileName string, err error) { + switch { + case remoteURL == "": + context, err = MakeTarSumContext(r) + case urlutil.IsGitURL(remoteURL): + context, err = MakeGitContext(remoteURL) + case urlutil.IsURL(remoteURL): + context, err = MakeRemoteContext(remoteURL, map[string]func(io.ReadCloser) (io.ReadCloser, error){ + httputils.MimeTypes.TextPlain: func(rc io.ReadCloser) (io.ReadCloser, error) { + dockerfile, err := ioutil.ReadAll(rc) + if err != nil { + return nil, err + } + + // dockerfileName is set to signal that the remote was interpreted as a single Dockerfile, in which case the caller + // should use dockerfileName as the new name for the Dockerfile, irrespective of any other user input. + dockerfileName = api.DefaultDockerfileName + + // TODO: return a context without tarsum + return archive.Generate(dockerfileName, string(dockerfile)) + }, + // fallback handler (tar context) + "": func(rc io.ReadCloser) (io.ReadCloser, error) { + return createProgressReader(rc), nil + }, + }) + default: + err = fmt.Errorf("remoteURL (%s) could not be recognized as URL", remoteURL) + } + return +} + // inspectResponse looks into the http response data at r to determine whether its // content-type is on the list of acceptable content types for remote build contexts. // This function returns: diff --git a/daemon/archive.go b/daemon/archive.go index 4ac667d261..5d0e9e0b52 100644 --- a/daemon/archive.go +++ b/daemon/archive.go @@ -7,9 +7,11 @@ import ( "path/filepath" "strings" + "github.com/docker/docker/builder" "github.com/docker/docker/container" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/chrootarchive" + "github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/ioutils" "github.com/docker/engine-api/types" ) @@ -328,3 +330,100 @@ func (daemon *Daemon) containerCopy(container *container.Container, resource str daemon.LogContainerEvent(container, "copy") return reader, nil } + +// CopyOnBuild copies/extracts a source FileInfo to a destination path inside a container +// specified by a container object. +// TODO: make sure callers don't unnecessarily convert destPath with filepath.FromSlash (Copy does it already). +// CopyOnBuild should take in abstract paths (with slashes) and the implementation should convert it to OS-specific paths. +func (daemon *Daemon) CopyOnBuild(cID string, destPath string, src builder.FileInfo, decompress bool) error { + srcPath := src.Path() + destExists := true + destDir := false + rootUID, rootGID := daemon.GetRemappedUIDGID() + + // Work in daemon-local OS specific file paths + destPath = filepath.FromSlash(destPath) + + c, err := daemon.GetContainer(cID) + if err != nil { + return err + } + err = daemon.Mount(c) + if err != nil { + return err + } + defer daemon.Unmount(c) + + dest, err := c.GetResourcePath(destPath) + if err != nil { + return err + } + + // Preserve the trailing slash + // TODO: why are we appending another path separator if there was already one? + if strings.HasSuffix(destPath, string(os.PathSeparator)) || destPath == "." { + destDir = true + dest += string(os.PathSeparator) + } + + destPath = dest + + destStat, err := os.Stat(destPath) + if err != nil { + if !os.IsNotExist(err) { + //logrus.Errorf("Error performing os.Stat on %s. %s", destPath, err) + return err + } + destExists = false + } + + uidMaps, gidMaps := daemon.GetUIDGIDMaps() + archiver := &archive.Archiver{ + Untar: chrootarchive.Untar, + UIDMaps: uidMaps, + GIDMaps: gidMaps, + } + + if src.IsDir() { + // copy as directory + if err := archiver.CopyWithTar(srcPath, destPath); err != nil { + return err + } + return fixPermissions(srcPath, destPath, rootUID, rootGID, destExists) + } + if decompress && archive.IsArchivePath(srcPath) { + // Only try to untar if it is a file and that we've been told to decompress (when ADD-ing a remote file) + + // First try to unpack the source as an archive + // to support the untar feature we need to clean up the path a little bit + // because tar is very forgiving. First we need to strip off the archive's + // filename from the path but this is only added if it does not end in slash + tarDest := destPath + if strings.HasSuffix(tarDest, string(os.PathSeparator)) { + tarDest = filepath.Dir(destPath) + } + + // try to successfully untar the orig + err := archiver.UntarPath(srcPath, tarDest) + /* + if err != nil { + logrus.Errorf("Couldn't untar to %s: %v", tarDest, err) + } + */ + return err + } + + // only needed for fixPermissions, but might as well put it before CopyFileWithTar + if destDir || (destExists && destStat.IsDir()) { + destPath = filepath.Join(destPath, src.Name()) + } + + if err := idtools.MkdirAllNewAs(filepath.Dir(destPath), 0755, rootUID, rootGID); err != nil { + return err + } + if err := archiver.CopyFileWithTar(srcPath, destPath); err != nil { + return err + } + + return fixPermissions(srcPath, destPath, rootUID, rootGID, destExists) +} diff --git a/daemon/archive_unix.go b/daemon/archive_unix.go index e51c9ebbcc..fcea13c929 100644 --- a/daemon/archive_unix.go +++ b/daemon/archive_unix.go @@ -2,7 +2,11 @@ package daemon -import "github.com/docker/docker/container" +import ( + "github.com/docker/docker/container" + "os" + "path/filepath" +) // checkIfPathIsInAVolume checks if the path is in a volume. If it is, it // cannot be in a read-only volume. If it is not in a volume, the container @@ -19,3 +23,35 @@ func checkIfPathIsInAVolume(container *container.Container, absPath string) (boo } return toVolume, nil } + +func fixPermissions(source, destination string, uid, gid int, destExisted bool) error { + // If the destination didn't already exist, or the destination isn't a + // directory, then we should Lchown the destination. Otherwise, we shouldn't + // Lchown the destination. + destStat, err := os.Stat(destination) + if err != nil { + // This should *never* be reached, because the destination must've already + // been created while untar-ing the context. + return err + } + doChownDestination := !destExisted || !destStat.IsDir() + + // We Walk on the source rather than on the destination because we don't + // want to change permissions on things we haven't created or modified. + return filepath.Walk(source, func(fullpath string, info os.FileInfo, err error) error { + // Do not alter the walk root iff. it existed before, as it doesn't fall under + // the domain of "things we should chown". + if !doChownDestination && (source == fullpath) { + return nil + } + + // Path is prefixed by source: substitute with destination instead. + cleaned, err := filepath.Rel(source, fullpath) + if err != nil { + return err + } + + fullpath = filepath.Join(destination, cleaned) + return os.Lchown(fullpath, uid, gid) + }) +} diff --git a/daemon/archive_windows.go b/daemon/archive_windows.go index feebc84fe9..4cefb8de54 100644 --- a/daemon/archive_windows.go +++ b/daemon/archive_windows.go @@ -11,3 +11,8 @@ import "github.com/docker/docker/container" func checkIfPathIsInAVolume(container *container.Container, absPath string) (bool, error) { return false, nil } + +func fixPermissions(source, destination string, uid, gid int, destExisted bool) error { + // chown is not supported on Windows + return nil +} diff --git a/daemon/attach.go b/daemon/attach.go index a0374097ec..f0caf16996 100644 --- a/daemon/attach.go +++ b/daemon/attach.go @@ -100,6 +100,16 @@ func (daemon *Daemon) ContainerWsAttachWithLogs(prefixOrName string, c *Containe return daemon.attachWithLogs(container, c.InStream, c.OutStream, c.ErrStream, c.Logs, c.Stream, c.DetachKeys) } +// ContainerAttachOnBuild attaches streams to the container cID. If stream is true, it streams the output. +func (daemon *Daemon) ContainerAttachOnBuild(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool) error { + return daemon.ContainerWsAttachWithLogs(cID, &ContainerWsAttachWithLogsConfig{ + InStream: stdin, + OutStream: stdout, + ErrStream: stderr, + Stream: stream, + }) +} + func (daemon *Daemon) attachWithLogs(container *container.Container, stdin io.ReadCloser, stdout, stderr io.Writer, logs, stream bool, keys []byte) error { if logs { logDriver, err := daemon.getLogger(container) diff --git a/daemon/daemon.go b/daemon/daemon.go index 2528a5f386..ddda4103f1 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -22,6 +22,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/digest" "github.com/docker/docker/api" + "github.com/docker/docker/builder" "github.com/docker/docker/container" "github.com/docker/docker/daemon/events" "github.com/docker/docker/daemon/exec" @@ -1035,6 +1036,35 @@ func (daemon *Daemon) PullImage(ref reference.Named, metaHeaders map[string][]st return err } +// PullOnBuild tells Docker to pull image referenced by `name`. +func (daemon *Daemon) PullOnBuild(name string, authConfigs map[string]types.AuthConfig, output io.Writer) (builder.Image, error) { + ref, err := reference.ParseNamed(name) + if err != nil { + return nil, err + } + ref = reference.WithDefaultTag(ref) + + pullRegistryAuth := &types.AuthConfig{} + if len(authConfigs) > 0 { + // The request came with a full auth config file, we prefer to use that + repoInfo, err := daemon.RegistryService.ResolveRepository(ref) + if err != nil { + return nil, err + } + + resolvedConfig := registry.ResolveAuthConfig( + authConfigs, + repoInfo.Index, + ) + pullRegistryAuth = &resolvedConfig + } + + if err := daemon.PullImage(ref, nil, pullRegistryAuth, output); err != nil { + return nil, err + } + return daemon.GetImage(name) +} + // ExportImage exports a list of images to the given output stream. The // exported images are archived into a tar when written to the output // stream. All images with the given tag and all versions containing @@ -1275,6 +1305,15 @@ func (daemon *Daemon) GetImage(refOrID string) (*image.Image, error) { return daemon.imageStore.Get(imgID) } +// GetImageOnBuild looks up a Docker image referenced by `name`. +func (daemon *Daemon) GetImageOnBuild(name string) (builder.Image, error) { + img, err := daemon.GetImage(name) + if err != nil { + return nil, err + } + return img, nil +} + // GraphDriverName returns the name of the graph driver used by the layer.Store func (daemon *Daemon) GraphDriverName() string { return daemon.layerStore.DriverName() @@ -1301,11 +1340,11 @@ func (daemon *Daemon) GetRemappedUIDGID() (int, int) { return uid, gid } -// ImageGetCached returns the most recent created image that is a child +// GetCachedImage returns the most recent created image that is a child // of the image with imgID, that had the same config when it was // created. nil is returned if a child cannot be found. An error is // returned if the parent image cannot be found. -func (daemon *Daemon) ImageGetCached(imgID image.ID, config *containertypes.Config) (*image.Image, error) { +func (daemon *Daemon) GetCachedImage(imgID image.ID, config *containertypes.Config) (*image.Image, error) { // Loop on the children of the given image and check the config getMatch := func(siblings []image.ID) (*image.Image, error) { var match *image.Image @@ -1342,6 +1381,16 @@ func (daemon *Daemon) ImageGetCached(imgID image.ID, config *containertypes.Conf return getMatch(siblings) } +// GetCachedImageOnBuild returns a reference to a cached image whose parent equals `parent` +// and runconfig equals `cfg`. A cache miss is expected to return an empty ID and a nil error. +func (daemon *Daemon) GetCachedImageOnBuild(imgID string, cfg *containertypes.Config) (string, error) { + cache, err := daemon.GetCachedImage(image.ID(imgID), cfg) + if cache == nil || err != nil { + return "", err + } + return cache.ID().String(), nil +} + // tempDir returns the default directory to use for temporary files. func tempDir(rootDir string, rootUID, rootGID int) (string, error) { var tmpDir string diff --git a/daemon/daemonbuilder/builder.go b/daemon/daemonbuilder/builder.go deleted file mode 100644 index 6e29447b24..0000000000 --- a/daemon/daemonbuilder/builder.go +++ /dev/null @@ -1,235 +0,0 @@ -package daemonbuilder - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - - "github.com/Sirupsen/logrus" - "github.com/docker/docker/api" - "github.com/docker/docker/builder" - "github.com/docker/docker/daemon" - "github.com/docker/docker/image" - "github.com/docker/docker/pkg/archive" - "github.com/docker/docker/pkg/chrootarchive" - "github.com/docker/docker/pkg/httputils" - "github.com/docker/docker/pkg/idtools" - "github.com/docker/docker/pkg/ioutils" - "github.com/docker/docker/pkg/urlutil" - "github.com/docker/docker/reference" - "github.com/docker/docker/registry" - "github.com/docker/engine-api/types" - "github.com/docker/engine-api/types/container" -) - -// Docker implements builder.Backend for the docker Daemon object. -type Docker struct { - *daemon.Daemon -} - -// ensure Docker implements builder.Backend -var _ builder.Backend = Docker{} - -// Pull tells Docker to pull image referenced by `name`. -func (d Docker) Pull(name string, authConfigs map[string]types.AuthConfig, output io.Writer) (builder.Image, error) { - ref, err := reference.ParseNamed(name) - if err != nil { - return nil, err - } - ref = reference.WithDefaultTag(ref) - - pullRegistryAuth := &types.AuthConfig{} - if len(authConfigs) > 0 { - // The request came with a full auth config file, we prefer to use that - repoInfo, err := d.Daemon.RegistryService.ResolveRepository(ref) - if err != nil { - return nil, err - } - - resolvedConfig := registry.ResolveAuthConfig( - authConfigs, - repoInfo.Index, - ) - pullRegistryAuth = &resolvedConfig - } - - if err := d.Daemon.PullImage(ref, nil, pullRegistryAuth, ioutils.NopWriteCloser(output)); err != nil { - return nil, err - } - return d.GetImage(name) -} - -// GetImage looks up a Docker image referenced by `name`. -func (d Docker) GetImage(name string) (builder.Image, error) { - img, err := d.Daemon.GetImage(name) - if err != nil { - return nil, err - } - return imgWrap{img}, nil -} - -// ContainerUpdateCmd updates Path and Args for the container with ID cID. -func (d Docker) ContainerUpdateCmd(cID string, cmd []string) error { - c, err := d.Daemon.GetContainer(cID) - if err != nil { - return err - } - c.Path = cmd[0] - c.Args = cmd[1:] - return nil -} - -// ContainerAttach attaches streams to the container cID. If stream is true, it streams the output. -func (d Docker) ContainerAttach(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool) error { - return d.Daemon.ContainerWsAttachWithLogs(cID, &daemon.ContainerWsAttachWithLogsConfig{ - InStream: stdin, - OutStream: stdout, - ErrStream: stderr, - Stream: stream, - }) -} - -// BuilderCopy copies/extracts a source FileInfo to a destination path inside a container -// specified by a container object. -// TODO: make sure callers don't unnecessarily convert destPath with filepath.FromSlash (Copy does it already). -// BuilderCopy should take in abstract paths (with slashes) and the implementation should convert it to OS-specific paths. -func (d Docker) BuilderCopy(cID string, destPath string, src builder.FileInfo, decompress bool) error { - srcPath := src.Path() - destExists := true - destDir := false - rootUID, rootGID := d.Daemon.GetRemappedUIDGID() - - // Work in daemon-local OS specific file paths - destPath = filepath.FromSlash(destPath) - - c, err := d.Daemon.GetContainer(cID) - if err != nil { - return err - } - err = d.Daemon.Mount(c) - if err != nil { - return err - } - defer d.Daemon.Unmount(c) - - dest, err := c.GetResourcePath(destPath) - if err != nil { - return err - } - - // Preserve the trailing slash - // TODO: why are we appending another path separator if there was already one? - if strings.HasSuffix(destPath, string(os.PathSeparator)) || destPath == "." { - destDir = true - dest += string(os.PathSeparator) - } - - destPath = dest - - destStat, err := os.Stat(destPath) - if err != nil { - if !os.IsNotExist(err) { - logrus.Errorf("Error performing os.Stat on %s. %s", destPath, err) - return err - } - destExists = false - } - - uidMaps, gidMaps := d.Daemon.GetUIDGIDMaps() - archiver := &archive.Archiver{ - Untar: chrootarchive.Untar, - UIDMaps: uidMaps, - GIDMaps: gidMaps, - } - - if src.IsDir() { - // copy as directory - if err := archiver.CopyWithTar(srcPath, destPath); err != nil { - return err - } - return fixPermissions(srcPath, destPath, rootUID, rootGID, destExists) - } - if decompress && archive.IsArchivePath(srcPath) { - // Only try to untar if it is a file and that we've been told to decompress (when ADD-ing a remote file) - - // First try to unpack the source as an archive - // to support the untar feature we need to clean up the path a little bit - // because tar is very forgiving. First we need to strip off the archive's - // filename from the path but this is only added if it does not end in slash - tarDest := destPath - if strings.HasSuffix(tarDest, string(os.PathSeparator)) { - tarDest = filepath.Dir(destPath) - } - - // try to successfully untar the orig - err := archiver.UntarPath(srcPath, tarDest) - if err != nil { - logrus.Errorf("Couldn't untar to %s: %v", tarDest, err) - } - return err - } - - // only needed for fixPermissions, but might as well put it before CopyFileWithTar - if destDir || (destExists && destStat.IsDir()) { - destPath = filepath.Join(destPath, src.Name()) - } - - if err := idtools.MkdirAllNewAs(filepath.Dir(destPath), 0755, rootUID, rootGID); err != nil { - return err - } - if err := archiver.CopyFileWithTar(srcPath, destPath); err != nil { - return err - } - - return fixPermissions(srcPath, destPath, rootUID, rootGID, destExists) -} - -// GetCachedImage returns a reference to a cached image whose parent equals `parent` -// and runconfig equals `cfg`. A cache miss is expected to return an empty ID and a nil error. -func (d Docker) GetCachedImage(imgID string, cfg *container.Config) (string, error) { - cache, err := d.Daemon.ImageGetCached(image.ID(imgID), cfg) - if cache == nil || err != nil { - return "", err - } - return cache.ID().String(), nil -} - -// Following is specific to builder contexts - -// DetectContextFromRemoteURL returns a context and in certain cases the name of the dockerfile to be used -// irrespective of user input. -// progressReader is only used if remoteURL is actually a URL (not empty, and not a Git endpoint). -func DetectContextFromRemoteURL(r io.ReadCloser, remoteURL string, createProgressReader func(in io.ReadCloser) io.ReadCloser) (context builder.ModifiableContext, dockerfileName string, err error) { - switch { - case remoteURL == "": - context, err = builder.MakeTarSumContext(r) - case urlutil.IsGitURL(remoteURL): - context, err = builder.MakeGitContext(remoteURL) - case urlutil.IsURL(remoteURL): - context, err = builder.MakeRemoteContext(remoteURL, map[string]func(io.ReadCloser) (io.ReadCloser, error){ - httputils.MimeTypes.TextPlain: func(rc io.ReadCloser) (io.ReadCloser, error) { - dockerfile, err := ioutil.ReadAll(rc) - if err != nil { - return nil, err - } - - // dockerfileName is set to signal that the remote was interpreted as a single Dockerfile, in which case the caller - // should use dockerfileName as the new name for the Dockerfile, irrespective of any other user input. - dockerfileName = api.DefaultDockerfileName - - // TODO: return a context without tarsum - return archive.Generate(dockerfileName, string(dockerfile)) - }, - // fallback handler (tar context) - "": func(rc io.ReadCloser) (io.ReadCloser, error) { - return createProgressReader(rc), nil - }, - }) - default: - err = fmt.Errorf("remoteURL (%s) could not be recognized as URL", remoteURL) - } - return -} diff --git a/daemon/daemonbuilder/builder_unix.go b/daemon/daemonbuilder/builder_unix.go deleted file mode 100644 index aa63b33b4c..0000000000 --- a/daemon/daemonbuilder/builder_unix.go +++ /dev/null @@ -1,40 +0,0 @@ -// +build freebsd linux - -package daemonbuilder - -import ( - "os" - "path/filepath" -) - -func fixPermissions(source, destination string, uid, gid int, destExisted bool) error { - // If the destination didn't already exist, or the destination isn't a - // directory, then we should Lchown the destination. Otherwise, we shouldn't - // Lchown the destination. - destStat, err := os.Stat(destination) - if err != nil { - // This should *never* be reached, because the destination must've already - // been created while untar-ing the context. - return err - } - doChownDestination := !destExisted || !destStat.IsDir() - - // We Walk on the source rather than on the destination because we don't - // want to change permissions on things we haven't created or modified. - return filepath.Walk(source, func(fullpath string, info os.FileInfo, err error) error { - // Do not alter the walk root iff. it existed before, as it doesn't fall under - // the domain of "things we should chown". - if !doChownDestination && (source == fullpath) { - return nil - } - - // Path is prefixed by source: substitute with destination instead. - cleaned, err := filepath.Rel(source, fullpath) - if err != nil { - return err - } - - fullpath = filepath.Join(destination, cleaned) - return os.Lchown(fullpath, uid, gid) - }) -} diff --git a/daemon/daemonbuilder/builder_windows.go b/daemon/daemonbuilder/builder_windows.go deleted file mode 100644 index dded0e400d..0000000000 --- a/daemon/daemonbuilder/builder_windows.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build windows - -package daemonbuilder - -func fixPermissions(source, destination string, uid, gid int, destExisted bool) error { - // chown is not supported on Windows - return nil -} diff --git a/daemon/daemonbuilder/image.go b/daemon/daemonbuilder/image.go deleted file mode 100644 index 0683a08275..0000000000 --- a/daemon/daemonbuilder/image.go +++ /dev/null @@ -1,18 +0,0 @@ -package daemonbuilder - -import ( - "github.com/docker/docker/image" - "github.com/docker/engine-api/types/container" -) - -type imgWrap struct { - inner *image.Image -} - -func (img imgWrap) ID() string { - return string(img.inner.ID()) -} - -func (img imgWrap) Config() *container.Config { - return img.inner.Config -} diff --git a/daemon/update.go b/daemon/update.go index 6f07e92584..600d022452 100644 --- a/daemon/update.go +++ b/daemon/update.go @@ -22,6 +22,17 @@ func (daemon *Daemon) ContainerUpdate(name string, hostConfig *container.HostCon return warnings, nil } +// ContainerUpdateCmdOnBuild updates Path and Args for the container with ID cID. +func (daemon *Daemon) ContainerUpdateCmdOnBuild(cID string, cmd []string) error { + c, err := daemon.GetContainer(cID) + if err != nil { + return err + } + c.Path = cmd[0] + c.Args = cmd[1:] + return nil +} + func (daemon *Daemon) update(name string, hostConfig *container.HostConfig) error { if hostConfig == nil { return nil diff --git a/image/image.go b/image/image.go index b33b0b8402..b153a8231a 100644 --- a/image/image.go +++ b/image/image.go @@ -70,6 +70,16 @@ func (img *Image) ID() ID { return img.computedID } +// ImageID stringizes ID. +func (img *Image) ImageID() string { + return string(img.ID()) +} + +// RunConfig returns the image's container config. +func (img *Image) RunConfig() *container.Config { + return img.Config +} + // MarshalJSON serializes the image to JSON. It sorts the top-level keys so // that JSON that's been manipulated by a push/pull cycle with a legacy // registry won't end up with a different key order.