kotovalexarian-likes-github/moby--moby
1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00
Code Releases Activity

Merge pull request #31984 from tonistiigi/remote-context

Browse source 
builder: Refactor remote context
This commit is contained in:
Brian Goff 2017-04-27 11:50:18 -04:00 • committed by GitHub
commit ae0f8c7ba1
22 changed files with 588 additions and 804 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
api/server/router/build/backend.go
View file

@ -16,5 +16,5 @@ type Backend interface {
// by the caller. // by the caller.
// //
// TODO: make this return a reference instead of string // TODO: make this return a reference instead of string
BuildFromContext(ctx context.Context, src io.ReadCloser, remote string, buildOptions *types.ImageBuildOptions, pg backend.ProgressWriter) (string, error) BuildFromContext(ctx context.Context, src io.ReadCloser, buildOptions *types.ImageBuildOptions, pg backend.ProgressWriter) (string, error)
} }

9
api/server/router/build/build_routes.go
View file

@ -21,7 +21,7 @@ import (
"github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/progress" "github.com/docker/docker/pkg/progress"
"github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/pkg/streamformatter"
"github.com/docker/go-units" units "github.com/docker/go-units"
"golang.org/x/net/context" "golang.org/x/net/context"
) )
@ -57,6 +57,7 @@ func newImageBuildOptions(ctx context.Context, r *http.Request) (*types.ImageBui
options.SecurityOpt = r.Form["securityopt"] options.SecurityOpt = r.Form["securityopt"]
options.Squash = httputils.BoolValue(r, "squash") options.Squash = httputils.BoolValue(r, "squash")
options.Target = r.FormValue("target") options.Target = r.FormValue("target")
options.RemoteContext = r.FormValue("remote")
if r.Form.Get("shmsize") != "" { if r.Form.Get("shmsize") != "" {
shmSize, err := strconv.ParseInt(r.Form.Get("shmsize"), 10, 64) shmSize, err := strconv.ParseInt(r.Form.Get("shmsize"), 10, 64)
@ -184,8 +185,6 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
} }
buildOptions.AuthConfigs = authConfigs buildOptions.AuthConfigs = authConfigs
remoteURL := r.FormValue("remote")
// Currently, only used if context is from a remote url. // Currently, only used if context is from a remote url.
// Look at code in DetectContextFromRemoteURL for more information. // Look at code in DetectContextFromRemoteURL for more information.
createProgressReader := func(in io.ReadCloser) io.ReadCloser { createProgressReader := func(in io.ReadCloser) io.ReadCloser {
@ -193,7 +192,7 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
if buildOptions.SuppressOutput { if buildOptions.SuppressOutput {
progressOutput = sf.NewProgressOutput(notVerboseBuffer, true) progressOutput = sf.NewProgressOutput(notVerboseBuffer, true)
} }
return progress.NewProgressReader(in, progressOutput, r.ContentLength, "Downloading context", remoteURL) return progress.NewProgressReader(in, progressOutput, r.ContentLength, "Downloading context", buildOptions.RemoteContext)
} }
out := io.Writer(output) out := io.Writer(output)
@ -211,7 +210,7 @@ func (br *buildRouter) postBuild(ctx context.Context, w http.ResponseWriter, r *
ProgressReaderFunc: createProgressReader, ProgressReaderFunc: createProgressReader,
} }
imgID, err := br.backend.BuildFromContext(ctx, r.Body, remoteURL, buildOptions, pg) imgID, err := br.backend.BuildFromContext(ctx, r.Body, buildOptions, pg)
if err != nil { if err != nil {
return errf(err) return errf(err)
} }

92
builder/builder.go
View file

@ -6,7 +6,6 @@ package builder
import ( import (
"io" "io"
"os"
"time" "time"
"github.com/docker/distribution/reference" "github.com/docker/distribution/reference"
@ -22,85 +21,17 @@ const (
DefaultDockerfileName string = "Dockerfile" DefaultDockerfileName string = "Dockerfile"
) )
// Context represents a file system tree. // Source defines a location that can be used as a source for the ADD/COPY
type Context interface { // instructions in the builder.
type Source interface {
// Root returns root path for accessing source
Root() string
// Close allows to signal that the filesystem tree won't be used anymore. // Close allows to signal that the filesystem tree won't be used anymore.
// For Context implementations using a temporary directory, it is recommended to // For Context implementations using a temporary directory, it is recommended to
// delete the temporary directory in Close(). // delete the temporary directory in Close().
Close() error Close() error
// Stat returns an entry corresponding to path if any. // Hash returns a checksum for a file
// It is recommended to return an error if path was not found. Hash(path string) (string, error)
// If path is a symlink it also returns the path to the target file.
Stat(path string) (string, FileInfo, error)
// Open opens path from the context and returns a readable stream of it.
Open(path string) (io.ReadCloser, error)
// Walk walks the tree of the context with the function passed to it.
Walk(root string, walkFn WalkFunc) error
}
// WalkFunc is the type of the function called for each file or directory visited by Context.Walk().
type WalkFunc func(path string, fi FileInfo, err error) error
// ModifiableContext represents a modifiable Context.
// TODO: remove this interface once we can get rid of Remove()
type ModifiableContext interface {
Context
// Remove deletes the entry specified by `path`.
// It is usual for directory entries to delete all its subentries.
Remove(path string) error
}
// FileInfo extends os.FileInfo to allow retrieving an absolute path to the file.
// TODO: remove this interface once pkg/archive exposes a walk function that Context can use.
type FileInfo interface {
os.FileInfo
Path() string
}
// PathFileInfo is a convenience struct that implements the FileInfo interface.
type PathFileInfo struct {
os.FileInfo
// FilePath holds the absolute path to the file.
FilePath string
// FileName holds the basename for the file.
FileName string
}
// Path returns the absolute path to the file.
func (fi PathFileInfo) Path() string {
return fi.FilePath
}
// Name returns the basename of the file.
func (fi PathFileInfo) Name() string {
if fi.FileName != "" {
return fi.FileName
}
return fi.FileInfo.Name()
}
// Hashed defines an extra method intended for implementations of os.FileInfo.
type Hashed interface {
// Hash returns the hash of a file.
Hash() string
SetHash(string)
}
// HashedFileInfo is a convenient struct that augments FileInfo with a field.
type HashedFileInfo struct {
FileInfo
// FileHash represents the hash of a file.
FileHash string
}
// Hash returns the hash of a file.
func (fi HashedFileInfo) Hash() string {
return fi.FileHash
}
// SetHash sets the hash of a file.
func (fi *HashedFileInfo) SetHash(h string) {
fi.FileHash = h
} }
// Backend abstracts calls to a Docker Daemon. // Backend abstracts calls to a Docker Daemon.
@ -134,12 +65,9 @@ type Backend interface {
// ContainerCopy copies/extracts a source FileInfo to a destination path inside a container // ContainerCopy copies/extracts a source FileInfo to a destination path inside a container
// specified by a container object. // specified by a container object.
// TODO: make an Extract method instead of passing `decompress` // TODO: extract in the builder instead of passing `decompress`
// TODO: do not pass a FileInfo, instead refactor the archive package to export a Walk function that can be used // TODO: use containerd/fs.changestream instead as a source
// with Context.Walk CopyOnBuild(containerID string, destPath string, srcRoot string, srcPath string, decompress bool) error
// ContainerCopy(name string, res string) (io.ReadCloser, error)
// TODO: use copyBackend api
CopyOnBuild(containerID string, destPath string, src FileInfo, decompress bool) error
// HasExperimental checks if the backend supports experimental features // HasExperimental checks if the backend supports experimental features
HasExperimental() bool HasExperimental() bool

32
builder/dockerfile/builder.go
View file

@ -17,6 +17,7 @@ import (
"github.com/docker/docker/builder" "github.com/docker/docker/builder"
"github.com/docker/docker/builder/dockerfile/command" "github.com/docker/docker/builder/dockerfile/command"
"github.com/docker/docker/builder/dockerfile/parser" "github.com/docker/docker/builder/dockerfile/parser"
"github.com/docker/docker/builder/remotecontext"
"github.com/docker/docker/image" "github.com/docker/docker/image"
"github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/stringid"
"github.com/pkg/errors" "github.com/pkg/errors"
@ -48,7 +49,7 @@ type Builder struct {
Output io.Writer Output io.Writer
docker builder.Backend docker builder.Backend
context builder.Context source builder.Source
clientCtx context.Context clientCtx context.Context
runConfig *container.Config // runconfig for cmd, run, entrypoint etc. runConfig *container.Config // runconfig for cmd, run, entrypoint etc.
@ -80,35 +81,37 @@ func NewBuildManager(b builder.Backend) (bm *BuildManager) {
} }
// BuildFromContext builds a new image from a given context. // BuildFromContext builds a new image from a given context.
func (bm *BuildManager) BuildFromContext(ctx context.Context, src io.ReadCloser, remote string, buildOptions *types.ImageBuildOptions, pg backend.ProgressWriter) (string, error) { func (bm *BuildManager) BuildFromContext(ctx context.Context, src io.ReadCloser, buildOptions *types.ImageBuildOptions, pg backend.ProgressWriter) (string, error) {
if buildOptions.Squash && !bm.backend.HasExperimental() { if buildOptions.Squash && !bm.backend.HasExperimental() {
return "", apierrors.NewBadRequestError(errors.New("squash is only supported with experimental mode")) return "", apierrors.NewBadRequestError(errors.New("squash is only supported with experimental mode"))
} }
buildContext, dockerfileName, err := builder.DetectContextFromRemoteURL(src, remote, pg.ProgressReaderFunc) if buildOptions.Dockerfile == "" {
buildOptions.Dockerfile = builder.DefaultDockerfileName
}
source, dockerfile, err := remotecontext.Detect(ctx, buildOptions.RemoteContext, buildOptions.Dockerfile, src, pg.ProgressReaderFunc)
if err != nil { if err != nil {
return "", err return "", err
} }
if source != nil {
defer func() { defer func() {
if err := buildContext.Close(); err != nil { if err := source.Close(); err != nil {
logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err) logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
} }
}() }()
if len(dockerfileName) > 0 {
buildOptions.Dockerfile = dockerfileName
} }
b, err := NewBuilder(ctx, buildOptions, bm.backend, builder.DockerIgnoreContext{ModifiableContext: buildContext}) b, err := NewBuilder(ctx, buildOptions, bm.backend, source)
if err != nil { if err != nil {
return "", err return "", err
} }
b.imageContexts.cache = bm.pathCache b.imageContexts.cache = bm.pathCache
return b.build(pg.StdoutFormatter, pg.StderrFormatter, pg.Output) return b.build(dockerfile, pg.StdoutFormatter, pg.StderrFormatter, pg.Output)
} }
// NewBuilder creates a new Dockerfile builder from an optional dockerfile and a Config. // NewBuilder creates a new Dockerfile builder from an optional dockerfile and a Config.
// If dockerfile is nil, the Dockerfile specified by Config.DockerfileName, // If dockerfile is nil, the Dockerfile specified by Config.DockerfileName,
// will be read from the Context passed to Build(). // will be read from the Context passed to Build().
func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, backend builder.Backend, buildContext builder.Context) (b *Builder, err error) { func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, backend builder.Backend, source builder.Source) (b *Builder, err error) {
if config == nil { if config == nil {
config = new(types.ImageBuildOptions) config = new(types.ImageBuildOptions)
} }
@ -118,7 +121,7 @@ func NewBuilder(clientCtx context.Context, config *types.ImageBuildOptions, back
Stdout: os.Stdout, Stdout: os.Stdout,
Stderr: os.Stderr, Stderr: os.Stderr,
docker: backend, docker: backend,
context: buildContext, source: source,
runConfig: new(container.Config), runConfig: new(container.Config),
tmpContainers: map[string]struct{}{}, tmpContainers: map[string]struct{}{},
buildArgs: newBuildArgs(config.BuildArgs), buildArgs: newBuildArgs(config.BuildArgs),
@ -173,18 +176,13 @@ func sanitizeRepoAndTags(names []string) ([]reference.Named, error) {
// build runs the Dockerfile builder from a context and a docker object that allows to make calls // build runs the Dockerfile builder from a context and a docker object that allows to make calls
// to Docker. // to Docker.
func (b *Builder) build(stdout io.Writer, stderr io.Writer, out io.Writer) (string, error) { func (b *Builder) build(dockerfile *parser.Result, stdout io.Writer, stderr io.Writer, out io.Writer) (string, error) {
defer b.imageContexts.unmount() defer b.imageContexts.unmount()
b.Stdout = stdout b.Stdout = stdout
b.Stderr = stderr b.Stderr = stderr
b.Output = out b.Output = out
dockerfile, err := b.readAndParseDockerfile()
if err != nil {
return "", err
}
repoAndTags, err := sanitizeRepoAndTags(b.options.Tags) repoAndTags, err := sanitizeRepoAndTags(b.options.Tags)
if err != nil { if err != nil {
return "", err return "", err

6
builder/dockerfile/evaluator_test.go
View file

@ -7,8 +7,8 @@ import (
"github.com/docker/docker/api/types" "github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/container"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/dockerfile/parser" "github.com/docker/docker/builder/dockerfile/parser"
"github.com/docker/docker/builder/remotecontext"
"github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/reexec" "github.com/docker/docker/pkg/reexec"
) )
@ -158,7 +158,7 @@ func executeTestCase(t *testing.T, testCase dispatchTestCase) {
} }
}() }()
context, err := builder.MakeTarSumContext(tarStream) context, err := remotecontext.MakeTarSumContext(tarStream)
if err != nil { if err != nil {
t.Fatalf("Error when creating tar context: %s", err) t.Fatalf("Error when creating tar context: %s", err)
@ -186,7 +186,7 @@ func executeTestCase(t *testing.T, testCase dispatchTestCase) {
runConfig: config, runConfig: config,
options: options, options: options,
Stdout: ioutil.Discard, Stdout: ioutil.Discard,
context: context, source: context,
buildArgs: newBuildArgs(options.BuildArgs), buildArgs: newBuildArgs(options.BuildArgs),
} }

12
builder/dockerfile/imagecontext.go
View file

@ -119,14 +119,14 @@ func (ic *imageContexts) setCache(id, path string, v interface{}) {
// by an existing image // by an existing image
type imageMount struct { type imageMount struct {
id string id string
ctx builder.Context source builder.Source
release func() error release func() error
ic *imageContexts ic *imageContexts
runConfig *container.Config runConfig *container.Config
} }
func (im *imageMount) context() (builder.Context, error) { func (im *imageMount) context() (builder.Source, error) {
if im.ctx == nil { if im.source == nil {
if im.id == "" { if im.id == "" {
return nil, errors.Errorf("could not copy from empty context") return nil, errors.Errorf("could not copy from empty context")
} }
@ -134,14 +134,14 @@ func (im *imageMount) context() (builder.Context, error) {
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "failed to mount %s", im.id) return nil, errors.Wrapf(err, "failed to mount %s", im.id)
} }
ctx, err := remotecontext.NewLazyContext(p) source, err := remotecontext.NewLazyContext(p)
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "failed to create lazycontext for %s", p) return nil, errors.Wrapf(err, "failed to create lazycontext for %s", p)
} }
im.release = release im.release = release
im.ctx = ctx im.source = source
} }
return im.ctx, nil return im.source, nil
} }
func (im *imageMount) unmount() error { func (im *imageMount) unmount() error {

180
builder/dockerfile/internals.go
View file

@ -8,7 +8,6 @@ import (
"encoding/hex" "encoding/hex"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
@ -25,7 +24,7 @@ import (
"github.com/docker/docker/api/types/strslice" "github.com/docker/docker/api/types/strslice"
"github.com/docker/docker/builder" "github.com/docker/docker/builder"
"github.com/docker/docker/builder/dockerfile/parser" "github.com/docker/docker/builder/dockerfile/parser"
"github.com/docker/docker/pkg/archive" "github.com/docker/docker/builder/remotecontext"
"github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/httputils"
"github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/jsonmessage" "github.com/docker/docker/pkg/jsonmessage"
@ -33,7 +32,6 @@ import (
"github.com/docker/docker/pkg/streamformatter" "github.com/docker/docker/pkg/streamformatter"
"github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/system"
"github.com/docker/docker/pkg/tarsum"
"github.com/docker/docker/pkg/urlutil" "github.com/docker/docker/pkg/urlutil"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -88,7 +86,9 @@ func (b *Builder) commit(id string, autoCmd strslice.StrSlice, comment string) e
} }
type copyInfo struct { type copyInfo struct {
builder.FileInfo root string
path string
hash string
decompress bool decompress bool
} }
@ -109,19 +109,23 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalD
// the copy until we've looked at all src files // the copy until we've looked at all src files
var err error var err error
for _, orig := range args[0 : len(args)-1] { for _, orig := range args[0 : len(args)-1] {
var fi builder.FileInfo
if urlutil.IsURL(orig) { if urlutil.IsURL(orig) {
if !allowRemote { if !allowRemote {
return fmt.Errorf("Source can't be a URL for %s", cmdName) return fmt.Errorf("Source can't be a URL for %s", cmdName)
} }
fi, err = b.download(orig) remote, path, err := b.download(orig)
if err != nil {
return err
}
defer os.RemoveAll(remote.Root())
h, err := remote.Hash(path)
if err != nil { if err != nil {
return err return err
} }
defer os.RemoveAll(filepath.Dir(fi.Path()))
infos = append(infos, copyInfo{ infos = append(infos, copyInfo{
FileInfo: fi, root: remote.Root(),
decompress: false, path: path,
hash: h,
}) })
continue continue
} }
@ -147,20 +151,15 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalD
var origPaths string var origPaths string
if len(infos) == 1 { if len(infos) == 1 {
fi := infos[0].FileInfo info := infos[0]
origPaths = fi.Name() origPaths = info.path
if hfi, ok := fi.(builder.Hashed); ok { srcHash = info.hash
srcHash = hfi.Hash()
}
} else { } else {
var hashs []string var hashs []string
var origs []string var origs []string
for _, info := range infos { for _, info := range infos {
fi := info.FileInfo origs = append(origs, info.path)
origs = append(origs, fi.Name()) hashs = append(hashs, info.hash)
if hfi, ok := fi.(builder.Hashed); ok {
hashs = append(hashs, hfi.Hash())
}
} }
hasher := sha256.New() hasher := sha256.New()
hasher.Write([]byte(strings.Join(hashs, ","))) hasher.Write([]byte(strings.Join(hashs, ",")))
@ -197,7 +196,7 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalD
} }
for _, info := range infos { for _, info := range infos {
if err := b.docker.CopyOnBuild(container.ID, dest, info.FileInfo, info.decompress); err != nil { if err := b.docker.CopyOnBuild(container.ID, dest, info.root, info.path, info.decompress); err != nil {
return err return err
} }
} }
@ -205,7 +204,7 @@ func (b *Builder) runContextCommand(args []string, allowRemote bool, allowLocalD
return b.commit(container.ID, cmd, comment) return b.commit(container.ID, cmd, comment)
} }
func (b *Builder) download(srcURL string) (fi builder.FileInfo, err error) { func (b *Builder) download(srcURL string) (remote builder.Source, p string, err error) {
// get filename from URL // get filename from URL
u, err := url.Parse(srcURL) u, err := url.Parse(srcURL)
if err != nil { if err != nil {
@ -248,17 +247,12 @@ func (b *Builder) download(srcURL string) (fi builder.FileInfo, err error) {
progressOutput := stdoutFormatter.StreamFormatter.NewProgressOutput(stdoutFormatter.Writer, true) progressOutput := stdoutFormatter.StreamFormatter.NewProgressOutput(stdoutFormatter.Writer, true)
progressReader := progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Downloading") progressReader := progress.NewProgressReader(resp.Body, progressOutput, resp.ContentLength, "", "Downloading")
// Download and dump result to tmp file // Download and dump result to tmp file
// TODO: add filehash directly
if _, err = io.Copy(tmpFile, progressReader); err != nil { if _, err = io.Copy(tmpFile, progressReader); err != nil {
tmpFile.Close() tmpFile.Close()
return return
} }
fmt.Fprintln(b.Stdout) fmt.Fprintln(b.Stdout)
// ignoring error because the file was already opened successfully
tmpFileSt, err := tmpFile.Stat()
if err != nil {
tmpFile.Close()
return
}
// Set the mtime to the Last-Modified header value if present // Set the mtime to the Last-Modified header value if present
// Otherwise just remove atime and mtime // Otherwise just remove atime and mtime
@ -279,21 +273,12 @@ func (b *Builder) download(srcURL string) (fi builder.FileInfo, err error) {
return return
} }
// Calc the checksum, even if we're using the cache lc, err := remotecontext.NewLazyContext(tmpDir)
r, err := archive.Tar(tmpFileName, archive.Uncompressed)
if err != nil { if err != nil {
return return
} }
tarSum, err := tarsum.NewTarSum(r, true, tarsum.Version1)
if err != nil { return lc, filename, nil
return
}
if _, err = io.Copy(ioutil.Discard, tarSum); err != nil {
return
}
hash := tarSum.Sum(nil)
r.Close()
return &builder.HashedFileInfo{FileInfo: builder.PathFileInfo{FileInfo: tmpFileSt, FilePath: tmpFileName}, FileHash: hash}, nil
} }
var windowsBlacklist = map[string]bool{ var windowsBlacklist = map[string]bool{
@ -328,37 +313,40 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
} }
origPath = strings.TrimPrefix(origPath, "."+string(os.PathSeparator)) origPath = strings.TrimPrefix(origPath, "."+string(os.PathSeparator))
context := b.context source := b.source
var err error var err error
if imageSource != nil { if imageSource != nil {
context, err = imageSource.context() source, err = imageSource.context()
if err != nil { if err != nil {
return nil, err return nil, err
} }
} }
if context == nil { if source == nil {
return nil, errors.Errorf("No context given. Impossible to use %s", cmdName) return nil, errors.Errorf("No context given. Impossible to use %s", cmdName)
} }
// Deal with wildcards // Deal with wildcards
if allowWildcards && containsWildcards(origPath) { if allowWildcards && containsWildcards(origPath) {
var copyInfos []copyInfo var copyInfos []copyInfo
if err := context.Walk("", func(path string, info builder.FileInfo, err error) error { if err := filepath.Walk(source.Root(), func(path string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
return err return err
} }
if info.Name() == "" { rel, err := remotecontext.Rel(source.Root(), path)
// Why are we doing this check? if err != nil {
return err
}
if rel == "." {
return nil return nil
} }
if match, _ := filepath.Match(origPath, path); !match { if match, _ := filepath.Match(origPath, rel); !match {
return nil return nil
} }
// Note we set allowWildcards to false in case the name has // Note we set allowWildcards to false in case the name has
// a * in it // a * in it
subInfos, err := b.calcCopyInfo(cmdName, path, allowLocalDecompression, false, imageSource) subInfos, err := b.calcCopyInfo(cmdName, rel, allowLocalDecompression, false, imageSource)
if err != nil { if err != nil {
return err return err
} }
@ -371,38 +359,56 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
} }
// Must be a dir or a file // Must be a dir or a file
statPath, fi, err := context.Stat(origPath) hash, err := source.Hash(origPath)
if err != nil { if err != nil {
return nil, err return nil, err
} }
copyInfos := []copyInfo{{FileInfo: fi, decompress: allowLocalDecompression}} fi, err := remotecontext.StatAt(source, origPath)
if err != nil {
hfi, handleHash := fi.(builder.Hashed) return nil, err
if !handleHash {
return copyInfos, nil
} }
// TODO: remove, handle dirs in Hash()
copyInfos := []copyInfo{{root: source.Root(), path: origPath, hash: hash, decompress: allowLocalDecompression}}
if imageSource != nil { if imageSource != nil {
// fast-cache based on imageID // fast-cache based on imageID
if h, ok := b.imageContexts.getCache(imageSource.id, origPath); ok { if h, ok := b.imageContexts.getCache(imageSource.id, origPath); ok {
hfi.SetHash(h.(string)) copyInfos[0].hash = h.(string)
return copyInfos, nil return copyInfos, nil
} }
} }
// Deal with the single file case // Deal with the single file case
if !fi.IsDir() { if !fi.IsDir() {
hfi.SetHash("file:" + hfi.Hash()) copyInfos[0].hash = "file:" + copyInfos[0].hash
return copyInfos, nil return copyInfos, nil
} }
fp, err := remotecontext.FullPath(source, origPath)
if err != nil {
return nil, err
}
// Must be a dir // Must be a dir
var subfiles []string var subfiles []string
err = context.Walk(statPath, func(path string, info builder.FileInfo, err error) error { err = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
return err return err
} }
rel, err := remotecontext.Rel(source.Root(), path)
if err != nil {
return err
}
if rel == "." {
return nil
}
hash, err := source.Hash(rel)
if err != nil {
return nil
}
// we already checked handleHash above // we already checked handleHash above
subfiles = append(subfiles, info.(builder.Hashed).Hash()) subfiles = append(subfiles, hash)
return nil return nil
}) })
if err != nil { if err != nil {
@ -412,9 +418,9 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
sort.Strings(subfiles) sort.Strings(subfiles)
hasher := sha256.New() hasher := sha256.New()
hasher.Write([]byte(strings.Join(subfiles, ","))) hasher.Write([]byte(strings.Join(subfiles, ",")))
hfi.SetHash("dir:" + hex.EncodeToString(hasher.Sum(nil))) copyInfos[0].hash = "dir:" + hex.EncodeToString(hasher.Sum(nil))
if imageSource != nil { if imageSource != nil {
b.imageContexts.setCache(imageSource.id, origPath, hfi.Hash()) b.imageContexts.setCache(imageSource.id, origPath, copyInfos[0].hash)
} }
return copyInfos, nil return copyInfos, nil
@ -655,59 +661,3 @@ func (b *Builder) clearTmp() {
fmt.Fprintf(b.Stdout, "Removing intermediate container %s\n", stringid.TruncateID(c)) fmt.Fprintf(b.Stdout, "Removing intermediate container %s\n", stringid.TruncateID(c))
} }
} }
// readAndParseDockerfile reads a Dockerfile from the current context.
func (b *Builder) readAndParseDockerfile() (*parser.Result, error) {
// If no -f was specified then look for 'Dockerfile'. If we can't find
// that then look for 'dockerfile'. If neither are found then default
// back to 'Dockerfile' and use that in the error message.
if b.options.Dockerfile == "" {
b.options.Dockerfile = builder.DefaultDockerfileName
if _, _, err := b.context.Stat(b.options.Dockerfile); os.IsNotExist(err) {
lowercase := strings.ToLower(b.options.Dockerfile)
if _, _, err := b.context.Stat(lowercase); err == nil {
b.options.Dockerfile = lowercase
}
}
}
result, err := b.parseDockerfile()
if err != nil {
return nil, err
}
// After the Dockerfile has been parsed, we need to check the .dockerignore
// file for either "Dockerfile" or ".dockerignore", and if either are
// present then erase them from the build context. These files should never
// have been sent from the client but we did send them to make sure that
// we had the Dockerfile to actually parse, and then we also need the
// .dockerignore file to know whether either file should be removed.
// Note that this assumes the Dockerfile has been read into memory and
// is now safe to be removed.
if dockerIgnore, ok := b.context.(builder.DockerIgnoreContext); ok {
dockerIgnore.Process([]string{b.options.Dockerfile})
}
return result, nil
}
func (b *Builder) parseDockerfile() (*parser.Result, error) {
f, err := b.context.Open(b.options.Dockerfile)
if err != nil {
if os.IsNotExist(err) {
return nil, fmt.Errorf("Cannot locate specified Dockerfile: %s", b.options.Dockerfile)
}
return nil, err
}
defer f.Close()
if f, ok := f.(*os.File); ok {
// ignoring error because Open already succeeded
fi, err := f.Stat()
if err != nil {
return nil, fmt.Errorf("Unexpected error reading Dockerfile: %v", err)
}
if fi.Size() == 0 {
return nil, fmt.Errorf("The Dockerfile (%s) cannot be empty", b.options.Dockerfile)
}
}
return parser.Parse(f)
}

22
builder/dockerfile/internals_test.go
View file

@ -1,11 +1,12 @@
package dockerfile package dockerfile
import ( import (
"context"
"fmt" "fmt"
"testing" "testing"
"github.com/docker/docker/api/types"
"github.com/docker/docker/builder" "github.com/docker/docker/builder"
"github.com/docker/docker/builder/remotecontext"
"github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/archive"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -17,7 +18,7 @@ func TestEmptyDockerfile(t *testing.T) {
createTestTempFile(t, contextDir, builder.DefaultDockerfileName, "", 0777) createTestTempFile(t, contextDir, builder.DefaultDockerfileName, "", 0777)
readAndCheckDockerfile(t, "emptyDockerfile", contextDir, "", "The Dockerfile (Dockerfile) cannot be empty") readAndCheckDockerfile(t, "emptyDockerfile", contextDir, "", "the Dockerfile (Dockerfile) cannot be empty")
} }
func TestSymlinkDockerfile(t *testing.T) { func TestSymlinkDockerfile(t *testing.T) {
@ -63,21 +64,10 @@ func readAndCheckDockerfile(t *testing.T, testName, contextDir, dockerfilePath,
} }
}() }()
context, err := builder.MakeTarSumContext(tarStream) if dockerfilePath == "" { // handled in BuildWithContext
require.NoError(t, err) dockerfilePath = builder.DefaultDockerfileName
defer func() {
if err = context.Close(); err != nil {
t.Fatalf("Error when closing tar context: %s", err)
}
}()
options := &types.ImageBuildOptions{
Dockerfile: dockerfilePath,
} }
b := &Builder{options: options, context: context} _, _, err = remotecontext.Detect(context.Background(), "", dockerfilePath, tarStream, nil)
_, err = b.readAndParseDockerfile()
assert.EqualError(t, err, expectedError) assert.EqualError(t, err, expectedError)
} }

2
builder/dockerfile/mockbackend_test.go
View file

@ -69,7 +69,7 @@ func (m *MockBackend) ContainerCreateWorkdir(containerID string) error {
return nil return nil
} }
func (m *MockBackend) CopyOnBuild(containerID string, destPath string, src builder.FileInfo, decompress bool) error { func (m *MockBackend) CopyOnBuild(containerID string, destPath string, srcRoot string, srcPath string, decompress bool) error {
return nil return nil
} }

48
builder/dockerignore.go
View file

@ -1,48 +0,0 @@
package builder
import (
"os"
"github.com/docker/docker/builder/dockerignore"
"github.com/docker/docker/pkg/fileutils"
)
// DockerIgnoreContext wraps a ModifiableContext to add a method
// for handling the .dockerignore file at the root of the context.
type DockerIgnoreContext struct {
ModifiableContext
}
// Process reads the .dockerignore file at the root of the embedded context.
// If .dockerignore does not exist in the context, then nil is returned.
//
// It can take a list of files to be removed after .dockerignore is removed.
// This is used for server-side implementations of builders that need to send
// the .dockerignore file as well as the special files specified in filesToRemove,
// but expect them to be excluded from the context after they were processed.
//
// For example, server-side Dockerfile builders are expected to pass in the name
// of the Dockerfile to be removed after it was parsed.
//
// TODO: Don't require a ModifiableContext (use Context instead) and don't remove
// files, instead handle a list of files to be excluded from the context.
func (c DockerIgnoreContext) Process(filesToRemove []string) error {
f, err := c.Open(".dockerignore")
// Note that a missing .dockerignore file isn't treated as an error
if err != nil {
if os.IsNotExist(err) {
return nil
}
return err
}
excludes, _ := dockerignore.ReadAll(f)
f.Close()
filesToRemove = append([]string{".dockerignore"}, filesToRemove...)
for _, fileToRemove := range filesToRemove {
rm, _ := fileutils.Matches(fileToRemove, excludes)
if rm {
c.Remove(fileToRemove)
}
}
return nil
}

172
builder/remotecontext/detect.go Normal file
View file

@ -0,0 +1,172 @@
package remotecontext
import (
"bufio"
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/docker/builder"
"github.com/docker/docker/builder/dockerfile/parser"
"github.com/docker/docker/builder/dockerignore"
"github.com/docker/docker/pkg/fileutils"
"github.com/docker/docker/pkg/httputils"
"github.com/docker/docker/pkg/symlink"
"github.com/docker/docker/pkg/urlutil"
"github.com/pkg/errors"
)
// Detect returns a context and dockerfile from remote location or local
// archive. progressReader is only used if remoteURL is actually a URL
// (not empty, and not a Git endpoint).
func Detect(ctx context.Context, remoteURL string, dockerfilePath string, r io.ReadCloser, progressReader func(in io.ReadCloser) io.ReadCloser) (remote builder.Source, dockerfile *parser.Result, err error) {
switch {
case remoteURL == "":
remote, dockerfile, err = newArchiveRemote(r, dockerfilePath)
case urlutil.IsGitURL(remoteURL):
remote, dockerfile, err = newGitRemote(remoteURL, dockerfilePath)
case urlutil.IsURL(remoteURL):
remote, dockerfile, err = newURLRemote(remoteURL, dockerfilePath, progressReader)
default:
err = fmt.Errorf("remoteURL (%s) could not be recognized as URL", remoteURL)
}
return
}
func newArchiveRemote(rc io.ReadCloser, dockerfilePath string) (builder.Source, *parser.Result, error) {
c, err := MakeTarSumContext(rc)
if err != nil {
return nil, nil, err
}
return withDockerfileFromContext(c.(modifiableContext), dockerfilePath)
}
func withDockerfileFromContext(c modifiableContext, dockerfilePath string) (builder.Source, *parser.Result, error) {
df, err := openAt(c, dockerfilePath)
if err != nil {
if os.IsNotExist(err) {
if dockerfilePath == builder.DefaultDockerfileName {
lowercase := strings.ToLower(dockerfilePath)
if _, err := StatAt(c, lowercase); err == nil {
return withDockerfileFromContext(c, lowercase)
}
}
return nil, nil, errors.Errorf("Cannot locate specified Dockerfile: %s", dockerfilePath) // backwards compatible error
}
c.Close()
return nil, nil, err
}
res, err := readAndParseDockerfile(dockerfilePath, df)
if err != nil {
return nil, nil, err
}
df.Close()
if err := removeDockerfile(c, dockerfilePath); err != nil {
c.Close()
return nil, nil, err
}
return c, res, nil
}
func newGitRemote(gitURL string, dockerfilePath string) (builder.Source, *parser.Result, error) {
c, err := MakeGitContext(gitURL) // TODO: change this to NewLazyContext
if err != nil {
return nil, nil, err
}
return withDockerfileFromContext(c.(modifiableContext), dockerfilePath)
}
func newURLRemote(url string, dockerfilePath string, progressReader func(in io.ReadCloser) io.ReadCloser) (builder.Source, *parser.Result, error) {
var dockerfile io.ReadCloser
dockerfileFoundErr := errors.New("found-dockerfile")
c, err := MakeRemoteContext(url, map[string]func(io.ReadCloser) (io.ReadCloser, error){
httputils.MimeTypes.TextPlain: func(rc io.ReadCloser) (io.ReadCloser, error) {
dockerfile = rc
return nil, dockerfileFoundErr
},
// fallback handler (tar context)
"": func(rc io.ReadCloser) (io.ReadCloser, error) {
return progressReader(rc), nil
},
})
if err != nil {
if err == dockerfileFoundErr {
res, err := parser.Parse(dockerfile)
if err != nil {
return nil, nil, err
}
return nil, res, nil
}
return nil, nil, err
}
return withDockerfileFromContext(c.(modifiableContext), dockerfilePath)
}
func removeDockerfile(c modifiableContext, filesToRemove ...string) error {
f, err := openAt(c, ".dockerignore")
// Note that a missing .dockerignore file isn't treated as an error
switch {
case os.IsNotExist(err):
return nil
case err != nil:
return err
}
excludes, err := dockerignore.ReadAll(f)
f.Close()
filesToRemove = append([]string{".dockerignore"}, filesToRemove...)
for _, fileToRemove := range filesToRemove {
if rm, _ := fileutils.Matches(fileToRemove, excludes); rm {
if err := c.Remove(fileToRemove); err != nil {
logrus.Errorf("failed to remove %s: %v", fileToRemove, err)
}
}
}
return nil
}
func readAndParseDockerfile(name string, rc io.Reader) (*parser.Result, error) {
br := bufio.NewReader(rc)
if _, err := br.Peek(1); err != nil {
if err == io.EOF {
return nil, errors.Errorf("the Dockerfile (%s) cannot be empty", name)
}
return nil, errors.Wrap(err, "unexpected error reading Dockerfile")
}
return parser.Parse(br)
}
func openAt(remote builder.Source, path string) (*os.File, error) {
fullPath, err := FullPath(remote, path)
if err != nil {
return nil, err
}
return os.Open(fullPath)
}
// StatAt is a helper for calling Stat on a path from a source
func StatAt(remote builder.Source, path string) (os.FileInfo, error) {
fullPath, err := FullPath(remote, path)
if err != nil {
return nil, err
}
return os.Stat(fullPath)
}
// FullPath is a helper for getting a full path for a path from a source
func FullPath(remote builder.Source, path string) (string, error) {
fullPath, err := symlink.FollowSymlinkInScope(filepath.Join(remote.Root(), path), remote.Root())
if err != nil {
return "", fmt.Errorf("Forbidden path outside the build context: %s (%s)", path, fullPath) // backwards compat with old error
}
return fullPath, nil
}

46
builder/dockerignore_test.go → builder/remotecontext/detect_test.go
View file

@ -1,11 +1,21 @@
package builder package remotecontext
import ( import (
"errors"
"io/ioutil" "io/ioutil"
"log" "log"
"os" "os"
"path/filepath"
"sort" "sort"
"testing" "testing"
"github.com/docker/docker/builder"
)
const (
dockerfileContents = "FROM busybox"
dockerignoreFilename = ".dockerignore"
testfileContents = "test"
) )
const shouldStayFilename = "should_stay" const shouldStayFilename = "should_stay"
@ -43,10 +53,9 @@ func checkDirectory(t *testing.T, dir string, expectedFiles []string) {
} }
func executeProcess(t *testing.T, contextDir string) { func executeProcess(t *testing.T, contextDir string) {
modifiableCtx := &tarSumContext{root: contextDir} modifiableCtx := &stubRemote{root: contextDir}
ctx := DockerIgnoreContext{ModifiableContext: modifiableCtx}
err := ctx.Process([]string{DefaultDockerfileName}) err := removeDockerfile(modifiableCtx, builder.DefaultDockerfileName)
if err != nil { if err != nil {
t.Fatalf("Error when executing Process: %s", err) t.Fatalf("Error when executing Process: %s", err)
@ -59,7 +68,7 @@ func TestProcessShouldRemoveDockerfileDockerignore(t *testing.T) {
createTestTempFile(t, contextDir, shouldStayFilename, testfileContents, 0777) createTestTempFile(t, contextDir, shouldStayFilename, testfileContents, 0777)
createTestTempFile(t, contextDir, dockerignoreFilename, "Dockerfile\n.dockerignore", 0777) createTestTempFile(t, contextDir, dockerignoreFilename, "Dockerfile\n.dockerignore", 0777)
createTestTempFile(t, contextDir, DefaultDockerfileName, dockerfileContents, 0777) createTestTempFile(t, contextDir, builder.DefaultDockerfileName, dockerfileContents, 0777)
executeProcess(t, contextDir) executeProcess(t, contextDir)
@ -72,11 +81,11 @@ func TestProcessNoDockerignore(t *testing.T) {
defer cleanup() defer cleanup()
createTestTempFile(t, contextDir, shouldStayFilename, testfileContents, 0777) createTestTempFile(t, contextDir, shouldStayFilename, testfileContents, 0777)
createTestTempFile(t, contextDir, DefaultDockerfileName, dockerfileContents, 0777) createTestTempFile(t, contextDir, builder.DefaultDockerfileName, dockerfileContents, 0777)
executeProcess(t, contextDir) executeProcess(t, contextDir)
checkDirectory(t, contextDir, []string{shouldStayFilename, DefaultDockerfileName}) checkDirectory(t, contextDir, []string{shouldStayFilename, builder.DefaultDockerfileName})
} }
@ -85,11 +94,30 @@ func TestProcessShouldLeaveAllFiles(t *testing.T) {
defer cleanup() defer cleanup()
createTestTempFile(t, contextDir, shouldStayFilename, testfileContents, 0777) createTestTempFile(t, contextDir, shouldStayFilename, testfileContents, 0777)
createTestTempFile(t, contextDir, DefaultDockerfileName, dockerfileContents, 0777) createTestTempFile(t, contextDir, builder.DefaultDockerfileName, dockerfileContents, 0777)
createTestTempFile(t, contextDir, dockerignoreFilename, "input1\ninput2", 0777) createTestTempFile(t, contextDir, dockerignoreFilename, "input1\ninput2", 0777)
executeProcess(t, contextDir) executeProcess(t, contextDir)
checkDirectory(t, contextDir, []string{shouldStayFilename, DefaultDockerfileName, dockerignoreFilename}) checkDirectory(t, contextDir, []string{shouldStayFilename, builder.DefaultDockerfileName, dockerignoreFilename})
} }
// TODO: remove after moving to a separate pkg
type stubRemote struct {
root string
}
func (r *stubRemote) Hash(path string) (string, error) {
return "", errors.New("not implemented")
}
func (r *stubRemote) Root() string {
return r.root
}
func (r *stubRemote) Close() error {
return errors.New("not implemented")
}
func (r *stubRemote) Remove(p string) error {
return os.Remove(filepath.Join(r.root, p))
}

5
builder/git.go → builder/remotecontext/git.go
View file

@ -1,14 +1,15 @@
package builder package remotecontext
import ( import (
"os" "os"
"github.com/docker/docker/builder"
"github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/gitutils" "github.com/docker/docker/pkg/gitutils"
) )
// MakeGitContext returns a Context from gitURL that is cloned in a temporary directory. // MakeGitContext returns a Context from gitURL that is cloned in a temporary directory.
func MakeGitContext(gitURL string) (ModifiableContext, error) { func MakeGitContext(gitURL string) (builder.Source, error) {
root, err := gitutils.Clone(gitURL) root, err := gitutils.Clone(gitURL)
if err != nil { if err != nil {
return nil, err return nil, err

99
builder/remotecontext/lazycontext.go
View file

@ -2,7 +2,6 @@ package remotecontext
import ( import (
"encoding/hex" "encoding/hex"
"io"
"os" "os"
"path/filepath" "path/filepath"
"runtime" "runtime"
@ -10,14 +9,13 @@ import (
"github.com/docker/docker/builder" "github.com/docker/docker/builder"
"github.com/docker/docker/pkg/pools" "github.com/docker/docker/pkg/pools"
"github.com/docker/docker/pkg/symlink"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
// NewLazyContext creates a new LazyContext. LazyContext defines a hashed build // NewLazyContext creates a new LazyContext. LazyContext defines a hashed build
// context based on a root directory. Individual files are hashed first time // context based on a root directory. Individual files are hashed first time
// they are asked. It is not safe to call methods of LazyContext concurrently. // they are asked. It is not safe to call methods of LazyContext concurrently.
func NewLazyContext(root string) (builder.Context, error) { func NewLazyContext(root string) (builder.Source, error) {
return &lazyContext{ return &lazyContext{
root: root, root: root,
sums: make(map[string]string), sums: make(map[string]string),
@ -29,86 +27,39 @@ type lazyContext struct {
sums map[string]string sums map[string]string
} }
func (c *lazyContext) Root() string {
return c.root
}
func (c *lazyContext) Close() error { func (c *lazyContext) Close() error {
return nil return nil
} }
func (c *lazyContext) Open(path string) (io.ReadCloser, error) { func (c *lazyContext) Hash(path string) (string, error) {
cleanPath, fullPath, err := c.normalize(path) cleanPath, fullPath, err := normalize(path, c.root)
if err != nil { if err != nil {
return nil, err return "", err
} }
r, err := os.Open(fullPath) fi, err := os.Lstat(fullPath)
if err != nil { if err != nil {
return nil, errors.WithStack(convertPathError(err, cleanPath)) return "", err
}
return r, nil
} }
func (c *lazyContext) Stat(path string) (string, builder.FileInfo, error) { relPath, err := Rel(c.root, fullPath)
// TODO: although stat returns builder.FileInfo it builder.Context actually requires Hashed
cleanPath, fullPath, err := c.normalize(path)
if err != nil { if err != nil {
return "", nil, err return "", errors.WithStack(convertPathError(err, cleanPath))
}
st, err := os.Lstat(fullPath)
if err != nil {
return "", nil, errors.WithStack(convertPathError(err, cleanPath))
}
relPath, err := rel(c.root, fullPath)
if err != nil {
return "", nil, errors.WithStack(convertPathError(err, cleanPath))
}
sum, ok := c.sums[relPath]
if !ok {
sum, err = c.prepareHash(relPath, st)
if err != nil {
return "", nil, err
}
}
fi := &builder.HashedFileInfo{
builder.PathFileInfo{st, fullPath, filepath.Base(cleanPath)},
sum,
}
return relPath, fi, nil
}
func (c *lazyContext) Walk(root string, walkFn builder.WalkFunc) error {
_, fullPath, err := c.normalize(root)
if err != nil {
return err
}
return filepath.Walk(fullPath, func(fullPath string, fi os.FileInfo, err error) error {
relPath, err := rel(c.root, fullPath)
if err != nil {
return errors.WithStack(err)
}
if relPath == "." {
return nil
} }
sum, ok := c.sums[relPath] sum, ok := c.sums[relPath]
if !ok { if !ok {
sum, err = c.prepareHash(relPath, fi) sum, err = c.prepareHash(relPath, fi)
if err != nil { if err != nil {
return err return "", err
} }
} }
hfi := &builder.HashedFileInfo{ return sum, nil
builder.PathFileInfo{FileInfo: fi, FilePath: fullPath},
sum,
}
if err := walkFn(relPath, hfi, nil); err != nil {
return err
}
return nil
})
} }
func (c *lazyContext) prepareHash(relPath string, fi os.FileInfo) (string, error) { func (c *lazyContext) prepareHash(relPath string, fi os.FileInfo) (string, error) {
@ -132,25 +83,9 @@ func (c *lazyContext) prepareHash(relPath string, fi os.FileInfo) (string, error
return sum, nil return sum, nil
} }
func (c *lazyContext) normalize(path string) (cleanPath, fullPath string, err error) { // Rel makes a path relative to base path. Same as `filepath.Rel` but can also
// todo: combine these helpers with tarsum after they are moved to same package // handle UUID paths in windows.
cleanPath = filepath.Clean(string(os.PathSeparator) + path)[1:] func Rel(basepath, targpath string) (string, error) {
fullPath, err = symlink.FollowSymlinkInScope(filepath.Join(c.root, path), c.root)
if err != nil {
return "", "", errors.Wrapf(err, "forbidden path outside the build context: %s (%s)", path, fullPath)
}
return
}
func convertPathError(err error, cleanpath string) error {
if err, ok := err.(*os.PathError); ok {
err.Path = cleanpath
return err
}
return err
}
func rel(basepath, targpath string) (string, error) {
// filepath.Rel can't handle UUID paths in windows // filepath.Rel can't handle UUID paths in windows
if runtime.GOOS == "windows" { if runtime.GOOS == "windows" {
pfx := basepath + `\` pfx := basepath + `\`

47
builder/remote.go → builder/remotecontext/remote.go
View file

@ -1,4 +1,4 @@
package builder package remotecontext
import ( import (
"bytes" "bytes"
@ -8,9 +8,8 @@ import (
"io/ioutil" "io/ioutil"
"regexp" "regexp"
"github.com/docker/docker/pkg/archive" "github.com/docker/docker/builder"
"github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/httputils"
"github.com/docker/docker/pkg/urlutil"
) )
// When downloading remote contexts, limit the amount (in bytes) // When downloading remote contexts, limit the amount (in bytes)
@ -30,7 +29,7 @@ var mimeRe = regexp.MustCompile(acceptableRemoteMIME)
// If a match is found, then the body is sent to the contentType handler and a (potentially compressed) tar stream is expected // If a match is found, then the body is sent to the contentType handler and a (potentially compressed) tar stream is expected
// to be returned. If no match is found, it is assumed the body is a tar stream (compressed or not). // to be returned. If no match is found, it is assumed the body is a tar stream (compressed or not).
// In either case, an (assumed) tar stream is passed to MakeTarSumContext whose result is returned. // In either case, an (assumed) tar stream is passed to MakeTarSumContext whose result is returned.
func MakeRemoteContext(remoteURL string, contentTypeHandlers map[string]func(io.ReadCloser) (io.ReadCloser, error)) (ModifiableContext, error) { func MakeRemoteContext(remoteURL string, contentTypeHandlers map[string]func(io.ReadCloser) (io.ReadCloser, error)) (builder.Source, error) {
f, err := httputils.Download(remoteURL) f, err := httputils.Download(remoteURL)
if err != nil { if err != nil {
return nil, fmt.Errorf("error downloading remote context %s: %v", remoteURL, err) return nil, fmt.Errorf("error downloading remote context %s: %v", remoteURL, err)
@ -67,46 +66,6 @@ func MakeRemoteContext(remoteURL string, contentTypeHandlers map[string]func(io.
return MakeTarSumContext(contextReader) return MakeTarSumContext(contextReader)
} }
// DetectContextFromRemoteURL returns a context and in certain cases the name of the dockerfile to be used
// irrespective of user input.
// progressReader is only used if remoteURL is actually a URL (not empty, and not a Git endpoint).
func DetectContextFromRemoteURL(r io.ReadCloser, remoteURL string, createProgressReader func(in io.ReadCloser) io.ReadCloser) (context ModifiableContext, dockerfileName string, err error) {
switch {
case remoteURL == "":
context, err = MakeTarSumContext(r)
case urlutil.IsGitURL(remoteURL):
context, err = MakeGitContext(remoteURL)
case urlutil.IsURL(remoteURL):
context, err = MakeRemoteContext(remoteURL, map[string]func(io.ReadCloser) (io.ReadCloser, error){
httputils.MimeTypes.TextPlain: func(rc io.ReadCloser) (io.ReadCloser, error) {
dockerfile, err := ioutil.ReadAll(rc)
if err != nil {
return nil, err
}
// dockerfileName is set to signal that the remote was interpreted as a single Dockerfile, in which case the caller
// should use dockerfileName as the new name for the Dockerfile, irrespective of any other user input.
dockerfileName = DefaultDockerfileName
// TODO: return a context without tarsum
r, err := archive.Generate(dockerfileName, string(dockerfile))
if err != nil {
return nil, err
}
return ioutil.NopCloser(r), nil
},
// fallback handler (tar context)
"": func(rc io.ReadCloser) (io.ReadCloser, error) {
return createProgressReader(rc), nil
},
})
default:
err = fmt.Errorf("remoteURL (%s) could not be recognized as URL", remoteURL)
}
return
}
// inspectResponse looks into the http response data at r to determine whether its // inspectResponse looks into the http response data at r to determine whether its
// content-type is on the list of acceptable content types for remote build contexts. // content-type is on the list of acceptable content types for remote build contexts.
// This function returns: // This function returns:

15
builder/remote_test.go → builder/remotecontext/remote_test.go
View file

@ -1,4 +1,4 @@
package builder package remotecontext
import ( import (
"bytes" "bytes"
@ -9,6 +9,7 @@ import (
"net/url" "net/url"
"testing" "testing"
"github.com/docker/docker/builder"
"github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/httputils" "github.com/docker/docker/pkg/httputils"
) )
@ -175,13 +176,13 @@ func TestMakeRemoteContext(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test") contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup() defer cleanup()
createTestTempFile(t, contextDir, DefaultDockerfileName, dockerfileContents, 0777) createTestTempFile(t, contextDir, builder.DefaultDockerfileName, dockerfileContents, 0777)
mux := http.NewServeMux() mux := http.NewServeMux()
server := httptest.NewServer(mux) server := httptest.NewServer(mux)
serverURL, _ := url.Parse(server.URL) serverURL, _ := url.Parse(server.URL)
serverURL.Path = "/" + DefaultDockerfileName serverURL.Path = "/" + builder.DefaultDockerfileName
remoteURL := serverURL.String() remoteURL := serverURL.String()
mux.Handle("/", http.FileServer(http.Dir(contextDir))) mux.Handle("/", http.FileServer(http.Dir(contextDir)))
@ -193,7 +194,7 @@ func TestMakeRemoteContext(t *testing.T) {
return nil, err return nil, err
} }
r, err := archive.Generate(DefaultDockerfileName, string(dockerfile)) r, err := archive.Generate(builder.DefaultDockerfileName, string(dockerfile))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -221,13 +222,13 @@ func TestMakeRemoteContext(t *testing.T) {
t.Fatalf("Size of file info sums should be 1, got: %d", fileInfoSums.Len()) t.Fatalf("Size of file info sums should be 1, got: %d", fileInfoSums.Len())
} }
fileInfo := fileInfoSums.GetFile(DefaultDockerfileName) fileInfo := fileInfoSums.GetFile(builder.DefaultDockerfileName)
if fileInfo == nil { if fileInfo == nil {
t.Fatalf("There should be file named %s in fileInfoSums", DefaultDockerfileName) t.Fatalf("There should be file named %s in fileInfoSums", builder.DefaultDockerfileName)
} }
if fileInfo.Pos() != 0 { if fileInfo.Pos() != 0 {
t.Fatalf("File %s should have position 0, got %d", DefaultDockerfileName, fileInfo.Pos()) t.Fatalf("File %s should have position 0, got %d", builder.DefaultDockerfileName, fileInfo.Pos())
} }
} }

121
builder/tarsum.go → builder/remotecontext/tarsum.go
View file

@ -1,16 +1,17 @@
package builder package remotecontext
import ( import (
"fmt"
"io" "io"
"os" "os"
"path/filepath" "path/filepath"
"github.com/docker/docker/builder"
"github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/chrootarchive" "github.com/docker/docker/pkg/chrootarchive"
"github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/symlink" "github.com/docker/docker/pkg/symlink"
"github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/pkg/tarsum"
"github.com/pkg/errors"
) )
type tarSumContext struct { type tarSumContext struct {
@ -30,44 +31,11 @@ func convertPathError(err error, cleanpath string) error {
return err return err
} }
func (c *tarSumContext) Open(path string) (io.ReadCloser, error) { type modifiableContext interface {
cleanpath, fullpath, err := c.normalize(path) builder.Source
if err != nil { // Remove deletes the entry specified by `path`.
return nil, err // It is usual for directory entries to delete all its subentries.
} Remove(path string) error
r, err := os.Open(fullpath)
if err != nil {
return nil, convertPathError(err, cleanpath)
}
return r, nil
}
func (c *tarSumContext) Stat(path string) (string, FileInfo, error) {
cleanpath, fullpath, err := c.normalize(path)
if err != nil {
return "", nil, err
}
st, err := os.Lstat(fullpath)
if err != nil {
return "", nil, convertPathError(err, cleanpath)
}
rel, err := filepath.Rel(c.root, fullpath)
if err != nil {
return "", nil, convertPathError(err, cleanpath)
}
// We set sum to path by default for the case where GetFile returns nil.
// The usual case is if relative path is empty.
sum := path
// Use the checksum of the followed path(not the possible symlink) because
// this is the file that is actually copied.
if tsInfo := c.sums.GetFile(filepath.ToSlash(rel)); tsInfo != nil {
sum = tsInfo.Sum()
}
fi := &HashedFileInfo{PathFileInfo{st, fullpath, filepath.Base(cleanpath)}, sum}
return rel, fi, nil
} }
// MakeTarSumContext returns a build Context from a tar stream. // MakeTarSumContext returns a build Context from a tar stream.
@ -78,7 +46,7 @@ func (c *tarSumContext) Stat(path string) (string, FileInfo, error) {
// all those sums then becomes the source of truth for all operations on this Context. // all those sums then becomes the source of truth for all operations on this Context.
// //
// Closing tarStream has to be done by the caller. // Closing tarStream has to be done by the caller.
func MakeTarSumContext(tarStream io.Reader) (ModifiableContext, error) { func MakeTarSumContext(tarStream io.Reader) (builder.Source, error) {
root, err := ioutils.TempDir("", "docker-builder") root, err := ioutils.TempDir("", "docker-builder")
if err != nil { if err != nil {
return nil, err return nil, err
@ -114,46 +82,47 @@ func MakeTarSumContext(tarStream io.Reader) (ModifiableContext, error) {
return tsc, nil return tsc, nil
} }
func (c *tarSumContext) normalize(path string) (cleanpath, fullpath string, err error) { func (c *tarSumContext) Root() string {
cleanpath = filepath.Clean(string(os.PathSeparator) + path)[1:] return c.root
fullpath, err = symlink.FollowSymlinkInScope(filepath.Join(c.root, path), c.root)
if err != nil {
return "", "", fmt.Errorf("Forbidden path outside the build context: %s (%s)", path, fullpath)
}
_, err = os.Lstat(fullpath)
if err != nil {
return "", "", convertPathError(err, path)
}
return
}
func (c *tarSumContext) Walk(root string, walkFn WalkFunc) error {
root = filepath.Join(c.root, filepath.Join(string(filepath.Separator), root))
return filepath.Walk(root, func(fullpath string, info os.FileInfo, err error) error {
rel, err := filepath.Rel(c.root, fullpath)
if err != nil {
return err
}
if rel == "." {
return nil
}
sum := rel
if tsInfo := c.sums.GetFile(filepath.ToSlash(rel)); tsInfo != nil {
sum = tsInfo.Sum()
}
fi := &HashedFileInfo{PathFileInfo{FileInfo: info, FilePath: fullpath}, sum}
if err := walkFn(rel, fi, nil); err != nil {
return err
}
return nil
})
} }
func (c *tarSumContext) Remove(path string) error { func (c *tarSumContext) Remove(path string) error {
_, fullpath, err := c.normalize(path) _, fullpath, err := normalize(path, c.root)
if err != nil { if err != nil {
return err return err
} }
return os.RemoveAll(fullpath) return os.RemoveAll(fullpath)
} }
func (c *tarSumContext) Hash(path string) (string, error) {
cleanpath, fullpath, err := normalize(path, c.root)
if err != nil {
return "", err
}
rel, err := filepath.Rel(c.root, fullpath)
if err != nil {
return "", convertPathError(err, cleanpath)
}
// Use the checksum of the followed path(not the possible symlink) because
// this is the file that is actually copied.
if tsInfo := c.sums.GetFile(filepath.ToSlash(rel)); tsInfo != nil {
return tsInfo.Sum(), nil
}
// We set sum to path by default for the case where GetFile returns nil.
// The usual case is if relative path is empty.
return path, nil // backwards compat TODO: see if really needed
}
func normalize(path, root string) (cleanPath, fullPath string, err error) {
cleanPath = filepath.Clean(string(os.PathSeparator) + path)[1:]
fullPath, err = symlink.FollowSymlinkInScope(filepath.Join(root, path), root)
if err != nil {
return "", "", errors.Wrapf(err, "forbidden path outside the build context: %s (%s)", path, cleanPath)
}
if _, err := os.Lstat(fullPath); err != nil {
return "", "", convertPathError(err, path)
}
return
}

166
builder/remotecontext/tarsum_test.go Normal file
View file

@ -0,0 +1,166 @@
package remotecontext
import (
"io/ioutil"
"os"
"path/filepath"
"runtime"
"testing"
"github.com/docker/docker/builder"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/reexec"
)
const (
filename = "test"
contents = "contents test"
)
func init() {
reexec.Init()
}
func TestCloseRootDirectory(t *testing.T) {
contextDir, err := ioutil.TempDir("", "builder-tarsum-test")
if err != nil {
t.Fatalf("Error with creating temporary directory: %s", err)
}
tarsum := &tarSumContext{root: contextDir}
err = tarsum.Close()
if err != nil {
t.Fatalf("Error while executing Close: %s", err)
}
_, err = os.Stat(contextDir)
if !os.IsNotExist(err) {
t.Fatal("Directory should not exist at this point")
defer os.RemoveAll(contextDir)
}
}
func TestHashFile(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
createTestTempFile(t, contextDir, filename, contents, 0777)
tarSum := makeTestTarsumContext(t, contextDir)
sum, err := tarSum.Hash(filename)
if err != nil {
t.Fatalf("Error when executing Stat: %s", err)
}
if len(sum) == 0 {
t.Fatalf("Hash returned empty sum")
}
expected := "1149ab94af7be6cc1da1335e398f24ee1cf4926b720044d229969dfc248ae7ec"
if runtime.GOOS == "windows" {
expected = "55dfeb344351ab27f59aa60ebb0ed12025a2f2f4677bf77d26ea7a671274a9ca"
}
if actual := sum; expected != actual {
t.Fatalf("invalid checksum. expected %s, got %s", expected, actual)
}
}
func TestHashSubdir(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
contextSubdir := filepath.Join(contextDir, "builder-tarsum-test-subdir")
err := os.Mkdir(contextSubdir, 0700)
if err != nil {
t.Fatalf("Failed to make directory: %s", contextSubdir)
}
testFilename := createTestTempFile(t, contextSubdir, filename, contents, 0777)
tarSum := makeTestTarsumContext(t, contextDir)
relativePath, err := filepath.Rel(contextDir, testFilename)
if err != nil {
t.Fatalf("Error when getting relative path: %s", err)
}
sum, err := tarSum.Hash(relativePath)
if err != nil {
t.Fatalf("Error when executing Stat: %s", err)
}
if len(sum) == 0 {
t.Fatalf("Hash returned empty sum")
}
expected := "d7f8d6353dee4816f9134f4156bf6a9d470fdadfb5d89213721f7e86744a4e69"
if runtime.GOOS == "windows" {
expected = "74a3326b8e766ce63a8e5232f22e9dd895be647fb3ca7d337e5e0a9b3da8ef28"
}
if actual := sum; expected != actual {
t.Fatalf("invalid checksum. expected %s, got %s", expected, actual)
}
}
func TestStatNotExisting(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
tarSum := &tarSumContext{root: contextDir}
_, err := tarSum.Hash("not-existing")
if !os.IsNotExist(err) {
t.Fatalf("This file should not exist: %s", err)
}
}
func TestRemoveDirectory(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
contextSubdir := createTestTempSubdir(t, contextDir, "builder-tarsum-test-subdir")
relativePath, err := filepath.Rel(contextDir, contextSubdir)
if err != nil {
t.Fatalf("Error when getting relative path: %s", err)
}
tarSum := &tarSumContext{root: contextDir}
err = tarSum.Remove(relativePath)
if err != nil {
t.Fatalf("Error when executing Remove: %s", err)
}
_, err = os.Stat(contextSubdir)
if !os.IsNotExist(err) {
t.Fatal("Directory should not exist at this point")
}
}
func makeTestTarsumContext(t *testing.T, dir string) builder.Source {
tarStream, err := archive.Tar(dir, archive.Uncompressed)
if err != nil {
t.Fatalf("error: %s", err)
}
defer tarStream.Close()
tarSum, err := MakeTarSumContext(tarStream)
if err != nil {
t.Fatalf("Error when executing MakeTarSumContext: %s", err)
}
return tarSum
}

8
builder/utils_test.go → builder/remotecontext/utils_test.go
View file

@ -1,4 +1,4 @@
package builder package remotecontext
import ( import (
"io/ioutil" "io/ioutil"
@ -7,12 +7,6 @@ import (
"testing" "testing"
) )
const (
dockerfileContents = "FROM busybox"
dockerignoreFilename = ".dockerignore"
testfileContents = "test"
)
// createTestTempDir creates a temporary directory for testing. // createTestTempDir creates a temporary directory for testing.
// It returns the created path and a cleanup function which is meant to be used as deferred call. // It returns the created path and a cleanup function which is meant to be used as deferred call.
// When an error occurs, it terminates the test. // When an error occurs, it terminates the test.

265
builder/tarsum_test.go
View file

@ -1,265 +0,0 @@
package builder
import (
"bufio"
"bytes"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/reexec"
)
const (
filename = "test"
contents = "contents test"
)
func init() {
reexec.Init()
}
func TestCloseRootDirectory(t *testing.T) {
contextDir, err := ioutil.TempDir("", "builder-tarsum-test")
if err != nil {
t.Fatalf("Error with creating temporary directory: %s", err)
}
tarsum := &tarSumContext{root: contextDir}
err = tarsum.Close()
if err != nil {
t.Fatalf("Error while executing Close: %s", err)
}
_, err = os.Stat(contextDir)
if !os.IsNotExist(err) {
t.Fatal("Directory should not exist at this point")
defer os.RemoveAll(contextDir)
}
}
func TestOpenFile(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
createTestTempFile(t, contextDir, filename, contents, 0777)
tarSum := &tarSumContext{root: contextDir}
file, err := tarSum.Open(filename)
if err != nil {
t.Fatalf("Error when executing Open: %s", err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
buff := bytes.NewBufferString("")
for scanner.Scan() {
buff.WriteString(scanner.Text())
}
if contents != buff.String() {
t.Fatalf("Contents are not equal. Expected: %s, got: %s", contents, buff.String())
}
}
func TestOpenNotExisting(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
tarSum := &tarSumContext{root: contextDir}
file, err := tarSum.Open("not-existing")
if file != nil {
t.Fatal("Opened file should be nil")
}
if !os.IsNotExist(err) {
t.Fatalf("Error when executing Open: %s", err)
}
}
func TestStatFile(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
testFilename := createTestTempFile(t, contextDir, filename, contents, 0777)
tarSum := &tarSumContext{root: contextDir}
relPath, fileInfo, err := tarSum.Stat(filename)
if err != nil {
t.Fatalf("Error when executing Stat: %s", err)
}
if relPath != filename {
t.Fatalf("Relative path should be equal to %s, got %s", filename, relPath)
}
if fileInfo.Path() != testFilename {
t.Fatalf("Full path should be equal to %s, got %s", testFilename, fileInfo.Path())
}
}
func TestStatSubdir(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
contextSubdir := createTestTempSubdir(t, contextDir, "builder-tarsum-test-subdir")
testFilename := createTestTempFile(t, contextSubdir, filename, contents, 0777)
tarSum := &tarSumContext{root: contextDir}
relativePath, err := filepath.Rel(contextDir, testFilename)
if err != nil {
t.Fatalf("Error when getting relative path: %s", err)
}
relPath, fileInfo, err := tarSum.Stat(relativePath)
if err != nil {
t.Fatalf("Error when executing Stat: %s", err)
}
if relPath != relativePath {
t.Fatalf("Relative path should be equal to %s, got %s", relativePath, relPath)
}
if fileInfo.Path() != testFilename {
t.Fatalf("Full path should be equal to %s, got %s", testFilename, fileInfo.Path())
}
}
func TestStatNotExisting(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
tarSum := &tarSumContext{root: contextDir}
relPath, fileInfo, err := tarSum.Stat("not-existing")
if relPath != "" {
t.Fatal("Relative path should be nil")
}
if fileInfo != nil {
t.Fatal("File info should be nil")
}
if !os.IsNotExist(err) {
t.Fatalf("This file should not exist: %s", err)
}
}
func TestRemoveDirectory(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
contextSubdir := createTestTempSubdir(t, contextDir, "builder-tarsum-test-subdir")
relativePath, err := filepath.Rel(contextDir, contextSubdir)
if err != nil {
t.Fatalf("Error when getting relative path: %s", err)
}
tarSum := &tarSumContext{root: contextDir}
err = tarSum.Remove(relativePath)
if err != nil {
t.Fatalf("Error when executing Remove: %s", err)
}
_, err = os.Stat(contextSubdir)
if !os.IsNotExist(err) {
t.Fatal("Directory should not exist at this point")
}
}
func TestMakeTarSumContext(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
createTestTempFile(t, contextDir, filename, contents, 0777)
tarStream, err := archive.Tar(contextDir, archive.Uncompressed)
if err != nil {
t.Fatalf("error: %s", err)
}
defer tarStream.Close()
tarSum, err := MakeTarSumContext(tarStream)
if err != nil {
t.Fatalf("Error when executing MakeTarSumContext: %s", err)
}
if tarSum == nil {
t.Fatal("Tar sum context should not be nil")
}
}
func TestWalkWithoutError(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
contextSubdir := createTestTempSubdir(t, contextDir, "builder-tarsum-test-subdir")
createTestTempFile(t, contextSubdir, filename, contents, 0777)
tarSum := &tarSumContext{root: contextDir}
walkFun := func(path string, fi FileInfo, err error) error {
return nil
}
err := tarSum.Walk(contextSubdir, walkFun)
if err != nil {
t.Fatalf("Error when executing Walk: %s", err)
}
}
type WalkError struct {
}
func (we WalkError) Error() string {
return "Error when executing Walk"
}
func TestWalkWithError(t *testing.T) {
contextDir, cleanup := createTestTempDir(t, "", "builder-tarsum-test")
defer cleanup()
contextSubdir := createTestTempSubdir(t, contextDir, "builder-tarsum-test-subdir")
tarSum := &tarSumContext{root: contextDir}
walkFun := func(path string, fi FileInfo, err error) error {
return WalkError{}
}
err := tarSum.Walk(contextSubdir, walkFun)
if err == nil {
t.Fatal("Error should not be nil")
}
}

33
daemon/archive.go
View file

@ -7,7 +7,6 @@ import (
"strings" "strings"
"github.com/docker/docker/api/types" "github.com/docker/docker/api/types"
"github.com/docker/docker/builder"
"github.com/docker/docker/container" "github.com/docker/docker/container"
"github.com/docker/docker/layer" "github.com/docker/docker/layer"
"github.com/docker/docker/pkg/archive" "github.com/docker/docker/pkg/archive"
@ -15,6 +14,7 @@ import (
"github.com/docker/docker/pkg/idtools" "github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/ioutils" "github.com/docker/docker/pkg/ioutils"
"github.com/docker/docker/pkg/stringid" "github.com/docker/docker/pkg/stringid"
"github.com/docker/docker/pkg/symlink"
"github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/system"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -369,8 +369,12 @@ func (daemon *Daemon) containerCopy(container *container.Container, resource str
// specified by a container object. // specified by a container object.
// TODO: make sure callers don't unnecessarily convert destPath with filepath.FromSlash (Copy does it already). // TODO: make sure callers don't unnecessarily convert destPath with filepath.FromSlash (Copy does it already).
// CopyOnBuild should take in abstract paths (with slashes) and the implementation should convert it to OS-specific paths. // CopyOnBuild should take in abstract paths (with slashes) and the implementation should convert it to OS-specific paths.
func (daemon *Daemon) CopyOnBuild(cID string, destPath string, src builder.FileInfo, decompress bool) error { func (daemon *Daemon) CopyOnBuild(cID, destPath, srcRoot, srcPath string, decompress bool) error {
srcPath := src.Path() fullSrcPath, err := symlink.FollowSymlinkInScope(filepath.Join(srcRoot, srcPath), srcRoot)
if err != nil {
return err
}
destExists := true destExists := true
destDir := false destDir := false
rootUID, rootGID := daemon.GetRemappedUIDGID() rootUID, rootGID := daemon.GetRemappedUIDGID()
@ -418,14 +422,19 @@ func (daemon *Daemon) CopyOnBuild(cID string, destPath string, src builder.FileI
GIDMaps: gidMaps, GIDMaps: gidMaps,
} }
if src.IsDir() { src, err := os.Stat(fullSrcPath)
// copy as directory if err != nil {
if err := archiver.CopyWithTar(srcPath, destPath); err != nil {
return err return err
} }
return fixPermissions(srcPath, destPath, rootUID, rootGID, destExists)
if src.IsDir() {
// copy as directory
if err := archiver.CopyWithTar(fullSrcPath, destPath); err != nil {
return err
} }
if decompress && archive.IsArchivePath(srcPath) { return fixPermissions(fullSrcPath, destPath, rootUID, rootGID, destExists)
}
if decompress && archive.IsArchivePath(fullSrcPath) {
// Only try to untar if it is a file and that we've been told to decompress (when ADD-ing a remote file) // Only try to untar if it is a file and that we've been told to decompress (when ADD-ing a remote file)
// First try to unpack the source as an archive // First try to unpack the source as an archive
@ -438,7 +447,7 @@ func (daemon *Daemon) CopyOnBuild(cID string, destPath string, src builder.FileI
} }
// try to successfully untar the orig // try to successfully untar the orig
err := archiver.UntarPath(srcPath, tarDest) err := archiver.UntarPath(fullSrcPath, tarDest)
/* /*
if err != nil { if err != nil {
logrus.Errorf("Couldn't untar to %s: %v", tarDest, err) logrus.Errorf("Couldn't untar to %s: %v", tarDest, err)
@ -449,17 +458,17 @@ func (daemon *Daemon) CopyOnBuild(cID string, destPath string, src builder.FileI
// only needed for fixPermissions, but might as well put it before CopyFileWithTar // only needed for fixPermissions, but might as well put it before CopyFileWithTar
if destDir || (destExists && destStat.IsDir()) { if destDir || (destExists && destStat.IsDir()) {
destPath = filepath.Join(destPath, src.Name()) destPath = filepath.Join(destPath, filepath.Base(srcPath))
} }
if err := idtools.MkdirAllNewAs(filepath.Dir(destPath), 0755, rootUID, rootGID); err != nil { if err := idtools.MkdirAllNewAs(filepath.Dir(destPath), 0755, rootUID, rootGID); err != nil {
return err return err
} }
if err := archiver.CopyFileWithTar(srcPath, destPath); err != nil { if err := archiver.CopyFileWithTar(fullSrcPath, destPath); err != nil {
return err return err
} }
return fixPermissions(srcPath, destPath, rootUID, rootGID, destExists) return fixPermissions(fullSrcPath, destPath, rootUID, rootGID, destExists)
} }
// MountImage returns mounted path with rootfs of an image. // MountImage returns mounted path with rootfs of an image.

4
integration-cli/docker_api_build_test.go
View file

@ -22,13 +22,11 @@ func (s *DockerSuite) TestBuildAPIDockerFileRemote(c *check.C) {
var testD string var testD string
if testEnv.DaemonPlatform() == "windows" { if testEnv.DaemonPlatform() == "windows" {
testD = `FROM busybox testD = `FROM busybox
COPY * /tmp/
RUN find / -name ba* RUN find / -name ba*
RUN find /tmp/` RUN find /tmp/`
} else { } else {
// -xdev is required because sysfs can cause EPERM // -xdev is required because sysfs can cause EPERM
testD = `FROM busybox testD = `FROM busybox
COPY * /tmp/
RUN find / -xdev -name ba* RUN find / -xdev -name ba*
RUN find /tmp/` RUN find /tmp/`
} }
@ -45,7 +43,7 @@ RUN find /tmp/`
// Make sure Dockerfile exists. // Make sure Dockerfile exists.
// Make sure 'baz' doesn't exist ANYWHERE despite being mentioned in the URL // Make sure 'baz' doesn't exist ANYWHERE despite being mentioned in the URL
out := string(buf) out := string(buf)
c.Assert(out, checker.Contains, "/tmp/Dockerfile") c.Assert(out, checker.Contains, "RUN find /tmp")
c.Assert(out, checker.Not(checker.Contains), "baz") c.Assert(out, checker.Not(checker.Contains), "baz")
} }