package builder // internals for handling commands. Covers many areas and a lot of // non-contiguous functionality. Please read the comments. import ( "crypto/sha256" "encoding/hex" "fmt" "io" "io/ioutil" "net/url" "os" "path" "path/filepath" "sort" "strings" "syscall" "time" "github.com/docker/docker/archive" "github.com/docker/docker/daemon" imagepkg "github.com/docker/docker/image" "github.com/docker/docker/pkg/log" "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/pkg/symlink" "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/registry" "github.com/docker/docker/utils" ) func (b *Builder) readContext(context io.Reader) error { tmpdirPath, err := ioutil.TempDir("", "docker-build") if err != nil { return err } decompressedStream, err := archive.DecompressStream(context) if err != nil { return err } if b.context, err = tarsum.NewTarSum(decompressedStream, true, tarsum.Version0); err != nil { return err } if err := archive.Untar(b.context, tmpdirPath, nil); err != nil { return err } b.contextPath = tmpdirPath return nil } func (b *Builder) commit(id string, autoCmd []string, comment string) error { if b.image == "" { return fmt.Errorf("Please provide a source image with `from` prior to commit") } b.Config.Image = b.image if id == "" { cmd := b.Config.Cmd b.Config.Cmd = []string{"/bin/sh", "-c", "#(nop) " + comment} defer func(cmd []string) { b.Config.Cmd = cmd }(cmd) hit, err := b.probeCache() if err != nil { return err } if hit { return nil } container, err := b.create() if err != nil { return err } id = container.ID if err := container.Mount(); err != nil { return err } defer container.Unmount() } container := b.Daemon.Get(id) if container == nil { return fmt.Errorf("An error occured while creating the container") } // Note: Actually copy the struct autoConfig := *b.Config autoConfig.Cmd = autoCmd // Commit the container image, err := b.Daemon.Commit(container, "", "", "", b.maintainer, true, &autoConfig) if err != nil { return err } b.image = image.ID return nil } type copyInfo struct { origPath string destPath string hashPath string decompress bool tmpDir string } func (b *Builder) runContextCommand(args []string, allowRemote bool, allowDecompression bool, cmdName string) error { if b.context == nil { return fmt.Errorf("No context given. Impossible to use %s", cmdName) } if len(args) < 2 { return fmt.Errorf("Invalid %s format - at least two arguments required", cmdName) } dest := args[len(args)-1] // last one is always the dest if len(args) > 2 && dest[len(dest)-1] != '/' { return fmt.Errorf("When using %s with more than one source file, the destination must be a directory and end with a /", cmdName) } copyInfos := make([]copyInfo, len(args)-1) hasHash := false srcPaths := "" origPaths := "" b.Config.Image = b.image defer func() { for _, ci := range copyInfos { if ci.tmpDir != "" { os.RemoveAll(ci.tmpDir) } } }() // Loop through each src file and calculate the info we need to // do the copy (e.g. hash value if cached). Don't actually do // the copy until we've looked at all src files for i, orig := range args[0 : len(args)-1] { ci := ©Infos[i] ci.origPath = orig ci.destPath = dest ci.decompress = true err := calcCopyInfo(b, cmdName, ci, allowRemote, allowDecompression) if err != nil { return err } origPaths += " " + ci.origPath // will have leading space if ci.hashPath == "" { srcPaths += " " + ci.origPath // note leading space } else { srcPaths += " " + ci.hashPath // note leading space hasHash = true } } cmd := b.Config.Cmd b.Config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) %s%s in %s", cmdName, srcPaths, dest)} defer func(cmd []string) { b.Config.Cmd = cmd }(cmd) hit, err := b.probeCache() if err != nil { return err } // If we do not have at least one hash, never use the cache if hit && hasHash { return nil } container, _, err := b.Daemon.Create(b.Config, "") if err != nil { return err } b.TmpContainers[container.ID] = struct{}{} if err := container.Mount(); err != nil { return err } defer container.Unmount() for _, ci := range copyInfos { if err := b.addContext(container, ci.origPath, ci.destPath, ci.decompress); err != nil { return err } } if err := b.commit(container.ID, cmd, fmt.Sprintf("%s%s in %s", cmdName, origPaths, dest)); err != nil { return err } return nil } func calcCopyInfo(b *Builder, cmdName string, ci *copyInfo, allowRemote bool, allowDecompression bool) error { var ( remoteHash string isRemote bool ) saveOrig := ci.origPath isRemote = utils.IsURL(ci.origPath) if isRemote && !allowRemote { return fmt.Errorf("Source can't be an URL for %s", cmdName) } else if isRemote { // Initiate the download resp, err := utils.Download(ci.origPath) if err != nil { return err } // Create a tmp dir tmpDirName, err := ioutil.TempDir(b.contextPath, "docker-remote") if err != nil { return err } ci.tmpDir = tmpDirName // Create a tmp file within our tmp dir tmpFileName := path.Join(tmpDirName, "tmp") tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } // Download and dump result to tmp file if _, err := io.Copy(tmpFile, utils.ProgressReader(resp.Body, int(resp.ContentLength), b.OutOld, b.StreamFormatter, true, "", "Downloading")); err != nil { tmpFile.Close() return err } fmt.Fprintf(b.OutStream, "\n") tmpFile.Close() // Remove the mtime of the newly created tmp file if err := system.UtimesNano(tmpFileName, make([]syscall.Timespec, 2)); err != nil { return err } ci.origPath = path.Join(filepath.Base(tmpDirName), filepath.Base(tmpFileName)) // Process the checksum r, err := archive.Tar(tmpFileName, archive.Uncompressed) if err != nil { return err } tarSum, err := tarsum.NewTarSum(r, true, tarsum.Version0) if err != nil { return err } if _, err := io.Copy(ioutil.Discard, tarSum); err != nil { return err } remoteHash = tarSum.Sum(nil) r.Close() // If the destination is a directory, figure out the filename. if strings.HasSuffix(ci.destPath, "/") { u, err := url.Parse(saveOrig) if err != nil { return err } path := u.Path if strings.HasSuffix(path, "/") { path = path[:len(path)-1] } parts := strings.Split(path, "/") filename := parts[len(parts)-1] if filename == "" { return fmt.Errorf("cannot determine filename from url: %s", u) } ci.destPath = ci.destPath + filename } } if err := b.checkPathForAddition(ci.origPath); err != nil { return err } // Hash path and check the cache if b.UtilizeCache { var ( sums = b.context.GetSums() ) if remoteHash != "" { ci.hashPath = remoteHash } else if fi, err := os.Stat(path.Join(b.contextPath, ci.origPath)); err != nil { return err } else if fi.IsDir() { var subfiles []string for _, fileInfo := range sums { absFile := path.Join(b.contextPath, fileInfo.Name()) absOrigPath := path.Join(b.contextPath, ci.origPath) if strings.HasPrefix(absFile, absOrigPath) { subfiles = append(subfiles, fileInfo.Sum()) } } sort.Strings(subfiles) hasher := sha256.New() hasher.Write([]byte(strings.Join(subfiles, ","))) ci.hashPath = "dir:" + hex.EncodeToString(hasher.Sum(nil)) } else { if ci.origPath[0] == '/' && len(ci.origPath) > 1 { ci.origPath = ci.origPath[1:] } ci.origPath = strings.TrimPrefix(ci.origPath, "./") // This will match on the first file in sums of the archive if fis := sums.GetFile(ci.origPath); fis != nil { ci.hashPath = "file:" + fis.Sum() } } } if !allowDecompression || isRemote { ci.decompress = false } return nil } func (b *Builder) pullImage(name string) (*imagepkg.Image, error) { remote, tag := parsers.ParseRepositoryTag(name) if tag == "" { tag = "latest" } pullRegistryAuth := b.AuthConfig if len(b.AuthConfigFile.Configs) > 0 { // The request came with a full auth config file, we prefer to use that endpoint, _, err := registry.ResolveRepositoryName(remote) if err != nil { return nil, err } resolvedAuth := b.AuthConfigFile.ResolveAuthConfig(endpoint) pullRegistryAuth = &resolvedAuth } job := b.Engine.Job("pull", remote, tag) job.SetenvBool("json", b.StreamFormatter.Json()) job.SetenvBool("parallel", true) job.SetenvJson("authConfig", pullRegistryAuth) job.Stdout.Add(b.OutOld) if err := job.Run(); err != nil { return nil, err } image, err := b.Daemon.Repositories().LookupImage(name) if err != nil { return nil, err } return image, nil } func (b *Builder) processImageFrom(img *imagepkg.Image) error { b.image = img.ID if img.Config != nil { b.Config = img.Config } if len(b.Config.Env) == 0 { b.Config.Env = append(b.Config.Env, "PATH="+daemon.DefaultPathEnv) } // Process ONBUILD triggers if they exist if nTriggers := len(b.Config.OnBuild); nTriggers != 0 { fmt.Fprintf(b.ErrStream, "# Executing %d build triggers\n", nTriggers) } // Copy the ONBUILD triggers, and remove them from the config, since the config will be commited. onBuildTriggers := b.Config.OnBuild b.Config.OnBuild = []string{} // FIXME rewrite this so that builder/parser is used; right now steps in // onbuild are muted because we have no good way to represent the step // number for _, step := range onBuildTriggers { splitStep := strings.Split(step, " ") stepInstruction := strings.ToUpper(strings.Trim(splitStep[0], " ")) switch stepInstruction { case "ONBUILD": return fmt.Errorf("Source image contains forbidden chained `ONBUILD ONBUILD` trigger: %s", step) case "MAINTAINER", "FROM": return fmt.Errorf("Source image contains forbidden %s trigger: %s", stepInstruction, step) } // FIXME we have to run the evaluator manually here. This does not belong // in this function. Once removed, the init() in evaluator.go should no // longer be necessary. if f, ok := evaluateTable[strings.ToLower(stepInstruction)]; ok { if err := f(b, splitStep[1:], nil); err != nil { return err } } else { return fmt.Errorf("%s doesn't appear to be a valid Dockerfile instruction", splitStep[0]) } } return nil } // probeCache checks to see if image-caching is enabled (`b.UtilizeCache`) // and if so attempts to look up the current `b.image` and `b.Config` pair // in the current server `b.Daemon`. If an image is found, probeCache returns // `(true, nil)`. If no image is found, it returns `(false, nil)`. If there // is any error, it returns `(false, err)`. func (b *Builder) probeCache() (bool, error) { if b.UtilizeCache { if cache, err := b.Daemon.ImageGetCached(b.image, b.Config); err != nil { return false, err } else if cache != nil { fmt.Fprintf(b.OutStream, " ---> Using cache\n") log.Debugf("[BUILDER] Use cached version") b.image = cache.ID return true, nil } else { log.Debugf("[BUILDER] Cache miss") } } return false, nil } func (b *Builder) create() (*daemon.Container, error) { if b.image == "" { return nil, fmt.Errorf("Please provide a source image with `from` prior to run") } b.Config.Image = b.image config := *b.Config // Create the container c, warnings, err := b.Daemon.Create(b.Config, "") if err != nil { return nil, err } for _, warning := range warnings { fmt.Fprintf(b.OutStream, " ---> [Warning] %s\n", warning) } b.TmpContainers[c.ID] = struct{}{} fmt.Fprintf(b.OutStream, " ---> Running in %s\n", utils.TruncateID(c.ID)) // override the entry point that may have been picked up from the base image c.Path = config.Cmd[0] c.Args = config.Cmd[1:] return c, nil } func (b *Builder) run(c *daemon.Container) error { var errCh chan error if b.Verbose { errCh = utils.Go(func() error { // FIXME: call the 'attach' job so that daemon.Attach can be made private // // FIXME (LK4D4): Also, maybe makes sense to call "logs" job, it is like attach // but without hijacking for stdin. Also, with attach there can be race // condition because of some output already was printed before it. return <-b.Daemon.Attach(&c.StreamConfig, c.Config.OpenStdin, c.Config.StdinOnce, c.Config.Tty, nil, nil, b.OutStream, b.ErrStream) }) } //start the container if err := c.Start(); err != nil { return err } if errCh != nil { if err := <-errCh; err != nil { return err } } // Wait for it to finish if ret, _ := c.WaitStop(-1 * time.Second); ret != 0 { err := &utils.JSONError{ Message: fmt.Sprintf("The command %v returned a non-zero code: %d", b.Config.Cmd, ret), Code: ret, } return err } return nil } func (b *Builder) checkPathForAddition(orig string) error { origPath := path.Join(b.contextPath, orig) origPath, err := filepath.EvalSymlinks(origPath) if err != nil { if os.IsNotExist(err) { return fmt.Errorf("%s: no such file or directory", orig) } return err } if !strings.HasPrefix(origPath, b.contextPath) { return fmt.Errorf("Forbidden path outside the build context: %s (%s)", orig, origPath) } if _, err := os.Stat(origPath); err != nil { if os.IsNotExist(err) { return fmt.Errorf("%s: no such file or directory", orig) } return err } return nil } func (b *Builder) addContext(container *daemon.Container, orig, dest string, decompress bool) error { var ( err error destExists = true origPath = path.Join(b.contextPath, orig) destPath = path.Join(container.RootfsPath(), dest) ) if destPath != container.RootfsPath() { destPath, err = symlink.FollowSymlinkInScope(destPath, container.RootfsPath()) if err != nil { return err } } // Preserve the trailing '/' if strings.HasSuffix(dest, "/") || dest == "." { destPath = destPath + "/" } destStat, err := os.Stat(destPath) if err != nil { if !os.IsNotExist(err) { return err } destExists = false } fi, err := os.Stat(origPath) if err != nil { if os.IsNotExist(err) { return fmt.Errorf("%s: no such file or directory", orig) } return err } if fi.IsDir() { return copyAsDirectory(origPath, destPath, destExists) } // If we are adding a remote file (or we've been told not to decompress), do not try to untar it if decompress { // First try to unpack the source as an archive // to support the untar feature we need to clean up the path a little bit // because tar is very forgiving. First we need to strip off the archive's // filename from the path but this is only added if it does not end in / . tarDest := destPath if strings.HasSuffix(tarDest, "/") { tarDest = filepath.Dir(destPath) } // try to successfully untar the orig if err := archive.UntarPath(origPath, tarDest); err == nil { return nil } else if err != io.EOF { log.Debugf("Couldn't untar %s to %s: %s", origPath, tarDest, err) } } if err := os.MkdirAll(path.Dir(destPath), 0755); err != nil { return err } if err := archive.CopyWithTar(origPath, destPath); err != nil { return err } resPath := destPath if destExists && destStat.IsDir() { resPath = path.Join(destPath, path.Base(origPath)) } return fixPermissions(resPath, 0, 0) } func copyAsDirectory(source, destination string, destinationExists bool) error { if err := archive.CopyWithTar(source, destination); err != nil { return err } if destinationExists { files, err := ioutil.ReadDir(source) if err != nil { return err } for _, file := range files { if err := fixPermissions(filepath.Join(destination, file.Name()), 0, 0); err != nil { return err } } return nil } return fixPermissions(destination, 0, 0) } func fixPermissions(destination string, uid, gid int) error { return filepath.Walk(destination, func(path string, info os.FileInfo, err error) error { if err := os.Lchown(path, uid, gid); err != nil && !os.IsNotExist(err) { return err } return nil }) } func (b *Builder) clearTmp() { for c := range b.TmpContainers { tmp := b.Daemon.Get(c) if err := b.Daemon.Destroy(tmp); err != nil { fmt.Fprintf(b.OutStream, "Error removing intermediate container %s: %s\n", utils.TruncateID(c), err.Error()) return } b.Daemon.DeleteVolumes(tmp.VolumePaths()) delete(b.TmpContainers, c) fmt.Fprintf(b.OutStream, "Removing intermediate container %s\n", utils.TruncateID(c)) } }