package server import ( "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "io/ioutil" "net/url" "os" "path" "path/filepath" "reflect" "regexp" "sort" "strings" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/daemon" "github.com/dotcloud/docker/nat" "github.com/dotcloud/docker/registry" "github.com/dotcloud/docker/runconfig" "github.com/dotcloud/docker/utils" ) var ( ErrDockerfileEmpty = errors.New("Dockerfile cannot be empty") ) type BuildFile interface { Build(io.Reader) (string, error) CmdFrom(string) error CmdRun(string) error } type buildFile struct { daemon *daemon.Daemon srv *Server image string maintainer string config *runconfig.Config contextPath string context *utils.TarSum verbose bool utilizeCache bool rm bool authConfig *registry.AuthConfig configFile *registry.ConfigFile tmpContainers map[string]struct{} tmpImages map[string]struct{} outStream io.Writer errStream io.Writer // Deprecated, original writer used for ImagePull. To be removed. outOld io.Writer sf *utils.StreamFormatter } func (b *buildFile) clearTmp(containers map[string]struct{}) { for c := range containers { tmp := b.daemon.Get(c) if err := b.daemon.Destroy(tmp); err != nil { fmt.Fprintf(b.outStream, "Error removing intermediate container %s: %s\n", utils.TruncateID(c), err.Error()) } else { delete(containers, c) fmt.Fprintf(b.outStream, "Removing intermediate container %s\n", utils.TruncateID(c)) } } } func (b *buildFile) CmdFrom(name string) error { image, err := b.daemon.Repositories().LookupImage(name) if err != nil { if b.daemon.Graph().IsNotExist(err) { remote, tag := utils.ParseRepositoryTag(name) pullRegistryAuth := b.authConfig if len(b.configFile.Configs) > 0 { // The request came with a full auth config file, we prefer to use that endpoint, _, err := registry.ResolveRepositoryName(remote) if err != nil { return err } resolvedAuth := b.configFile.ResolveAuthConfig(endpoint) pullRegistryAuth = &resolvedAuth } job := b.srv.Eng.Job("pull", remote, tag) job.SetenvBool("json", b.sf.Json()) job.SetenvBool("parallel", true) job.SetenvJson("authConfig", pullRegistryAuth) job.Stdout.Add(b.outOld) if err := job.Run(); err != nil { return err } image, err = b.daemon.Repositories().LookupImage(name) if err != nil { return err } } else { return err } } b.image = image.ID b.config = &runconfig.Config{} if image.Config != nil { b.config = image.Config } if b.config.Env == nil || len(b.config.Env) == 0 { b.config.Env = append(b.config.Env, "HOME=/", "PATH="+daemon.DefaultPathEnv) } // Process ONBUILD triggers if they exist if nTriggers := len(b.config.OnBuild); nTriggers != 0 { fmt.Fprintf(b.errStream, "# Executing %d build triggers\n", nTriggers) } for n, step := range b.config.OnBuild { splitStep := strings.Split(step, " ") stepInstruction := strings.ToUpper(strings.Trim(splitStep[0], " ")) switch stepInstruction { case "ONBUILD": return fmt.Errorf("Source image contains forbidden chained `ONBUILD ONBUILD` trigger: %s", step) case "MAINTAINER", "FROM": return fmt.Errorf("Source image contains forbidden %s trigger: %s", stepInstruction, step) } if err := b.BuildStep(fmt.Sprintf("onbuild-%d", n), step); err != nil { return err } } b.config.OnBuild = []string{} return nil } // The ONBUILD command declares a build instruction to be executed in any future build // using the current image as a base. func (b *buildFile) CmdOnbuild(trigger string) error { splitTrigger := strings.Split(trigger, " ") triggerInstruction := strings.ToUpper(strings.Trim(splitTrigger[0], " ")) switch triggerInstruction { case "ONBUILD": return fmt.Errorf("Chaining ONBUILD via `ONBUILD ONBUILD` isn't allowed") case "MAINTAINER", "FROM": return fmt.Errorf("%s isn't allowed as an ONBUILD trigger", triggerInstruction) } b.config.OnBuild = append(b.config.OnBuild, trigger) return b.commit("", b.config.Cmd, fmt.Sprintf("ONBUILD %s", trigger)) } func (b *buildFile) CmdMaintainer(name string) error { b.maintainer = name return b.commit("", b.config.Cmd, fmt.Sprintf("MAINTAINER %s", name)) } // probeCache checks to see if image-caching is enabled (`b.utilizeCache`) // and if so attempts to look up the current `b.image` and `b.config` pair // in the current server `b.srv`. If an image is found, probeCache returns // `(true, nil)`. If no image is found, it returns `(false, nil)`. If there // is any error, it returns `(false, err)`. func (b *buildFile) probeCache() (bool, error) { if b.utilizeCache { if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { return false, err } else if cache != nil { fmt.Fprintf(b.outStream, " ---> Using cache\n") utils.Debugf("[BUILDER] Use cached version") b.image = cache.ID return true, nil } else { utils.Debugf("[BUILDER] Cache miss") } } return false, nil } func (b *buildFile) CmdRun(args string) error { if b.image == "" { return fmt.Errorf("Please provide a source image with `from` prior to run") } config, _, _, err := runconfig.Parse(append([]string{b.image}, b.buildCmdFromJson(args)...), nil) if err != nil { return err } cmd := b.config.Cmd b.config.Cmd = nil runconfig.Merge(b.config, config) defer func(cmd []string) { b.config.Cmd = cmd }(cmd) utils.Debugf("Command to be executed: %v", b.config.Cmd) hit, err := b.probeCache() if err != nil { return err } if hit { return nil } c, err := b.create() if err != nil { return err } // Ensure that we keep the container mounted until the commit // to avoid unmounting and then mounting directly again c.Mount() defer c.Unmount() err = b.run(c) if err != nil { return err } if err := b.commit(c.ID, cmd, "run"); err != nil { return err } return nil } func (b *buildFile) FindEnvKey(key string) int { for k, envVar := range b.config.Env { envParts := strings.SplitN(envVar, "=", 2) if key == envParts[0] { return k } } return -1 } func (b *buildFile) ReplaceEnvMatches(value string) (string, error) { exp, err := regexp.Compile("(\\\\\\\\+|[^\\\\]|\\b|\\A)\\$({?)([[:alnum:]_]+)(}?)") if err != nil { return value, err } matches := exp.FindAllString(value, -1) for _, match := range matches { match = match[strings.Index(match, "$"):] matchKey := strings.Trim(match, "${}") for _, envVar := range b.config.Env { envParts := strings.SplitN(envVar, "=", 2) envKey := envParts[0] envValue := envParts[1] if envKey == matchKey { value = strings.Replace(value, match, envValue, -1) break } } } return value, nil } func (b *buildFile) CmdEnv(args string) error { tmp := strings.SplitN(args, " ", 2) if len(tmp) != 2 { return fmt.Errorf("Invalid ENV format") } key := strings.Trim(tmp[0], " \t") value := strings.Trim(tmp[1], " \t") envKey := b.FindEnvKey(key) replacedValue, err := b.ReplaceEnvMatches(value) if err != nil { return err } replacedVar := fmt.Sprintf("%s=%s", key, replacedValue) if envKey >= 0 { b.config.Env[envKey] = replacedVar } else { b.config.Env = append(b.config.Env, replacedVar) } return b.commit("", b.config.Cmd, fmt.Sprintf("ENV %s", replacedVar)) } func (b *buildFile) buildCmdFromJson(args string) []string { var cmd []string if err := json.Unmarshal([]byte(args), &cmd); err != nil { utils.Debugf("Error unmarshalling: %s, setting to /bin/sh -c", err) cmd = []string{"/bin/sh", "-c", args} } return cmd } func (b *buildFile) CmdCmd(args string) error { cmd := b.buildCmdFromJson(args) b.config.Cmd = cmd if err := b.commit("", b.config.Cmd, fmt.Sprintf("CMD %v", cmd)); err != nil { return err } return nil } func (b *buildFile) CmdEntrypoint(args string) error { entrypoint := b.buildCmdFromJson(args) b.config.Entrypoint = entrypoint if err := b.commit("", b.config.Cmd, fmt.Sprintf("ENTRYPOINT %v", entrypoint)); err != nil { return err } return nil } func (b *buildFile) CmdExpose(args string) error { portsTab := strings.Split(args, " ") if b.config.ExposedPorts == nil { b.config.ExposedPorts = make(nat.PortSet) } ports, _, err := nat.ParsePortSpecs(append(portsTab, b.config.PortSpecs...)) if err != nil { return err } for port := range ports { if _, exists := b.config.ExposedPorts[port]; !exists { b.config.ExposedPorts[port] = struct{}{} } } b.config.PortSpecs = nil return b.commit("", b.config.Cmd, fmt.Sprintf("EXPOSE %v", ports)) } func (b *buildFile) CmdUser(args string) error { b.config.User = args return b.commit("", b.config.Cmd, fmt.Sprintf("USER %v", args)) } func (b *buildFile) CmdInsert(args string) error { return fmt.Errorf("INSERT has been deprecated. Please use ADD instead") } func (b *buildFile) CmdCopy(args string) error { return fmt.Errorf("COPY has been deprecated. Please use ADD instead") } func (b *buildFile) CmdWorkdir(workdir string) error { if workdir[0] == '/' { b.config.WorkingDir = workdir } else { if b.config.WorkingDir == "" { b.config.WorkingDir = "/" } b.config.WorkingDir = filepath.Join(b.config.WorkingDir, workdir) } return b.commit("", b.config.Cmd, fmt.Sprintf("WORKDIR %v", workdir)) } func (b *buildFile) CmdVolume(args string) error { if args == "" { return fmt.Errorf("Volume cannot be empty") } var volume []string if err := json.Unmarshal([]byte(args), &volume); err != nil { volume = []string{args} } if b.config.Volumes == nil { b.config.Volumes = map[string]struct{}{} } for _, v := range volume { b.config.Volumes[v] = struct{}{} } if err := b.commit("", b.config.Cmd, fmt.Sprintf("VOLUME %s", args)); err != nil { return err } return nil } func (b *buildFile) checkPathForAddition(orig string) error { origPath := path.Join(b.contextPath, orig) if p, err := filepath.EvalSymlinks(origPath); err != nil { if os.IsNotExist(err) { return fmt.Errorf("%s: no such file or directory", orig) } return err } else { origPath = p } if !strings.HasPrefix(origPath, b.contextPath) { return fmt.Errorf("Forbidden path outside the build context: %s (%s)", orig, origPath) } _, err := os.Stat(origPath) if err != nil { if os.IsNotExist(err) { return fmt.Errorf("%s: no such file or directory", orig) } return err } return nil } func (b *buildFile) addContext(container *daemon.Container, orig, dest string, remote bool) error { var ( err error destExists = true origPath = path.Join(b.contextPath, orig) destPath = path.Join(container.RootfsPath(), dest) ) if destPath != container.RootfsPath() { destPath, err = utils.FollowSymlinkInScope(destPath, container.RootfsPath()) if err != nil { return err } } // Preserve the trailing '/' if strings.HasSuffix(dest, "/") { destPath = destPath + "/" } destStat, err := os.Stat(destPath) if err != nil { if os.IsNotExist(err) { destExists = false } else { return err } } fi, err := os.Stat(origPath) if err != nil { if os.IsNotExist(err) { return fmt.Errorf("%s: no such file or directory", orig) } return err } chownR := func(destPath string, uid, gid int) error { return filepath.Walk(destPath, func(path string, info os.FileInfo, err error) error { if err := os.Lchown(path, uid, gid); err != nil { return err } return nil }) } if fi.IsDir() { if err := archive.CopyWithTar(origPath, destPath); err != nil { return err } if destExists { files, err := ioutil.ReadDir(origPath) if err != nil { return err } for _, file := range files { if err := chownR(filepath.Join(destPath, file.Name()), 0, 0); err != nil { return err } } } else { if err := chownR(destPath, 0, 0); err != nil { return err } } return nil } // First try to unpack the source as an archive // to support the untar feature we need to clean up the path a little bit // because tar is very forgiving. First we need to strip off the archive's // filename from the path but this is only added if it does not end in / . tarDest := destPath if strings.HasSuffix(tarDest, "/") { tarDest = filepath.Dir(destPath) } // If we are adding a remote file, do not try to untar it if !remote { // try to successfully untar the orig if err := archive.UntarPath(origPath, tarDest); err == nil { return nil } utils.Debugf("Couldn't untar %s to %s: %s", origPath, destPath, err) } // If that fails, just copy it as a regular file // but do not use all the magic path handling for the tar path if err := os.MkdirAll(path.Dir(destPath), 0755); err != nil { return err } if err := archive.CopyWithTar(origPath, destPath); err != nil { return err } resPath := destPath if destExists && destStat.IsDir() { resPath = path.Join(destPath, path.Base(origPath)) } if err := chownR(resPath, 0, 0); err != nil { return err } return nil } func (b *buildFile) CmdAdd(args string) error { if b.context == nil { return fmt.Errorf("No context given. Impossible to use ADD") } tmp := strings.SplitN(args, " ", 2) if len(tmp) != 2 { return fmt.Errorf("Invalid ADD format") } orig, err := b.ReplaceEnvMatches(strings.Trim(tmp[0], " \t")) if err != nil { return err } dest, err := b.ReplaceEnvMatches(strings.Trim(tmp[1], " \t")) if err != nil { return err } cmd := b.config.Cmd b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)} b.config.Image = b.image var ( origPath = orig destPath = dest remoteHash string isRemote bool ) if utils.IsURL(orig) { // Initiate the download isRemote = true resp, err := utils.Download(orig) if err != nil { return err } // Create a tmp dir tmpDirName, err := ioutil.TempDir(b.contextPath, "docker-remote") if err != nil { return err } // Create a tmp file within our tmp dir tmpFileName := path.Join(tmpDirName, "tmp") tmpFile, err := os.OpenFile(tmpFileName, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } defer os.RemoveAll(tmpDirName) // Download and dump result to tmp file if _, err := io.Copy(tmpFile, resp.Body); err != nil { tmpFile.Close() return err } tmpFile.Close() origPath = path.Join(filepath.Base(tmpDirName), filepath.Base(tmpFileName)) // Process the checksum r, err := archive.Tar(tmpFileName, archive.Uncompressed) if err != nil { return err } tarSum := utils.TarSum{Reader: r, DisableCompression: true} remoteHash = tarSum.Sum(nil) r.Close() // If the destination is a directory, figure out the filename. if strings.HasSuffix(dest, "/") { u, err := url.Parse(orig) if err != nil { return err } path := u.Path if strings.HasSuffix(path, "/") { path = path[:len(path)-1] } parts := strings.Split(path, "/") filename := parts[len(parts)-1] if filename == "" { return fmt.Errorf("cannot determine filename from url: %s", u) } destPath = dest + filename } } if err := b.checkPathForAddition(origPath); err != nil { return err } // Hash path and check the cache if b.utilizeCache { var ( hash string sums = b.context.GetSums() ) if remoteHash != "" { hash = remoteHash } else if fi, err := os.Stat(path.Join(b.contextPath, origPath)); err != nil { return err } else if fi.IsDir() { var subfiles []string for file, sum := range sums { absFile := path.Join(b.contextPath, file) absOrigPath := path.Join(b.contextPath, origPath) if strings.HasPrefix(absFile, absOrigPath) { subfiles = append(subfiles, sum) } } sort.Strings(subfiles) hasher := sha256.New() hasher.Write([]byte(strings.Join(subfiles, ","))) hash = "dir:" + hex.EncodeToString(hasher.Sum(nil)) } else { if origPath[0] == '/' && len(origPath) > 1 { origPath = origPath[1:] } origPath = strings.TrimPrefix(origPath, "./") if h, ok := sums[origPath]; ok { hash = "file:" + h } } b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", hash, dest)} hit, err := b.probeCache() if err != nil { return err } // If we do not have a hash, never use the cache if hit && hash != "" { return nil } } // Create the container and start it container, _, err := b.daemon.Create(b.config, "") if err != nil { return err } b.tmpContainers[container.ID] = struct{}{} if err := container.Mount(); err != nil { return err } defer container.Unmount() if err := b.addContext(container, origPath, destPath, isRemote); err != nil { return err } if err := b.commit(container.ID, cmd, fmt.Sprintf("ADD %s in %s", orig, dest)); err != nil { return err } b.config.Cmd = cmd return nil } func (b *buildFile) create() (*daemon.Container, error) { if b.image == "" { return nil, fmt.Errorf("Please provide a source image with `from` prior to run") } b.config.Image = b.image // Create the container and start it c, _, err := b.daemon.Create(b.config, "") if err != nil { return nil, err } b.tmpContainers[c.ID] = struct{}{} fmt.Fprintf(b.outStream, " ---> Running in %s\n", utils.TruncateID(c.ID)) // override the entry point that may have been picked up from the base image c.Path = b.config.Cmd[0] c.Args = b.config.Cmd[1:] return c, nil } func (b *buildFile) run(c *daemon.Container) error { var errCh chan error if b.verbose { errCh = utils.Go(func() error { return <-b.daemon.Attach(c, nil, nil, b.outStream, b.errStream) }) } //start the container if err := c.Start(); err != nil { return err } if errCh != nil { if err := <-errCh; err != nil { return err } } // Wait for it to finish if ret := c.Wait(); ret != 0 { err := &utils.JSONError{ Message: fmt.Sprintf("The command %v returned a non-zero code: %d", b.config.Cmd, ret), Code: ret, } return err } return nil } // Commit the container with the autorun command func (b *buildFile) commit(id string, autoCmd []string, comment string) error { if b.image == "" { return fmt.Errorf("Please provide a source image with `from` prior to commit") } b.config.Image = b.image if id == "" { cmd := b.config.Cmd b.config.Cmd = []string{"/bin/sh", "-c", "#(nop) " + comment} defer func(cmd []string) { b.config.Cmd = cmd }(cmd) hit, err := b.probeCache() if err != nil { return err } if hit { return nil } container, warnings, err := b.daemon.Create(b.config, "") if err != nil { return err } for _, warning := range warnings { fmt.Fprintf(b.outStream, " ---> [Warning] %s\n", warning) } b.tmpContainers[container.ID] = struct{}{} fmt.Fprintf(b.outStream, " ---> Running in %s\n", utils.TruncateID(container.ID)) id = container.ID if err := container.Mount(); err != nil { return err } defer container.Unmount() } container := b.daemon.Get(id) if container == nil { return fmt.Errorf("An error occured while creating the container") } // Note: Actually copy the struct autoConfig := *b.config autoConfig.Cmd = autoCmd // Commit the container image, err := b.daemon.Commit(container, "", "", "", b.maintainer, &autoConfig) if err != nil { return err } b.tmpImages[image.ID] = struct{}{} b.image = image.ID return nil } // Long lines can be split with a backslash var lineContinuation = regexp.MustCompile(`\s*\\\s*\n`) func (b *buildFile) Build(context io.Reader) (string, error) { tmpdirPath, err := ioutil.TempDir("", "docker-build") if err != nil { return "", err } decompressedStream, err := archive.DecompressStream(context) if err != nil { return "", err } b.context = &utils.TarSum{Reader: decompressedStream, DisableCompression: true} if err := archive.Untar(b.context, tmpdirPath, nil); err != nil { return "", err } defer os.RemoveAll(tmpdirPath) b.contextPath = tmpdirPath filename := path.Join(tmpdirPath, "Dockerfile") if _, err := os.Stat(filename); os.IsNotExist(err) { return "", fmt.Errorf("Can't build a directory with no Dockerfile") } fileBytes, err := ioutil.ReadFile(filename) if err != nil { return "", err } if len(fileBytes) == 0 { return "", ErrDockerfileEmpty } var ( dockerfile = lineContinuation.ReplaceAllString(stripComments(fileBytes), "") stepN = 0 ) for _, line := range strings.Split(dockerfile, "\n") { line = strings.Trim(strings.Replace(line, "\t", " ", -1), " \t\r\n") if len(line) == 0 { continue } if err := b.BuildStep(fmt.Sprintf("%d", stepN), line); err != nil { return "", err } else if b.rm { b.clearTmp(b.tmpContainers) } stepN += 1 } if b.image != "" { fmt.Fprintf(b.outStream, "Successfully built %s\n", utils.TruncateID(b.image)) return b.image, nil } return "", fmt.Errorf("No image was generated. This may be because the Dockerfile does not, like, do anything.\n") } // BuildStep parses a single build step from `instruction` and executes it in the current context. func (b *buildFile) BuildStep(name, expression string) error { fmt.Fprintf(b.outStream, "Step %s : %s\n", name, expression) tmp := strings.SplitN(expression, " ", 2) if len(tmp) != 2 { return fmt.Errorf("Invalid Dockerfile format") } instruction := strings.ToLower(strings.Trim(tmp[0], " ")) arguments := strings.Trim(tmp[1], " ") method, exists := reflect.TypeOf(b).MethodByName("Cmd" + strings.ToUpper(instruction[:1]) + strings.ToLower(instruction[1:])) if !exists { fmt.Fprintf(b.errStream, "# Skipping unknown instruction %s\n", strings.ToUpper(instruction)) return nil } ret := method.Func.Call([]reflect.Value{reflect.ValueOf(b), reflect.ValueOf(arguments)})[0].Interface() if ret != nil { return ret.(error) } fmt.Fprintf(b.outStream, " ---> %s\n", utils.TruncateID(b.image)) return nil } func stripComments(raw []byte) string { var ( out []string lines = strings.Split(string(raw), "\n") ) for _, l := range lines { if len(l) == 0 || l[0] == '#' { continue } out = append(out, l) } return strings.Join(out, "\n") } func NewBuildFile(srv *Server, outStream, errStream io.Writer, verbose, utilizeCache, rm bool, outOld io.Writer, sf *utils.StreamFormatter, auth *registry.AuthConfig, authConfigFile *registry.ConfigFile) BuildFile { return &buildFile{ daemon: srv.daemon, srv: srv, config: &runconfig.Config{}, outStream: outStream, errStream: errStream, tmpContainers: make(map[string]struct{}), tmpImages: make(map[string]struct{}), verbose: verbose, utilizeCache: utilizeCache, rm: rm, sf: sf, authConfig: auth, configFile: authConfigFile, outOld: outOld, } }