1
0
Fork 0
mirror of https://github.com/moby/moby.git synced 2022-11-09 12:21:53 -05:00
moby--moby/integration-cli/docker_api_build_test.go
Yong Tang 8c31aacebf
Merge pull request #36018 from thaJeztah/remove-deprecated-testenv-functions
Remove deprecated integration-cli/environment functions
2018-01-15 13:24:52 -08:00

666 lines
18 KiB
Go

package main
import (
"archive/tar"
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"regexp"
"strings"
"github.com/docker/docker/api/types"
"github.com/docker/docker/integration-cli/checker"
"github.com/docker/docker/integration-cli/cli/build/fakecontext"
"github.com/docker/docker/integration-cli/cli/build/fakegit"
"github.com/docker/docker/integration-cli/cli/build/fakestorage"
"github.com/docker/docker/integration-cli/request"
"github.com/go-check/check"
"github.com/moby/buildkit/session"
"github.com/moby/buildkit/session/filesync"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/context"
"golang.org/x/sync/errgroup"
)
func (s *DockerSuite) TestBuildAPIDockerFileRemote(c *check.C) {
testRequires(c, NotUserNamespace)
var testD string
if testEnv.OSType == "windows" {
testD = `FROM busybox
RUN find / -name ba*
RUN find /tmp/`
} else {
// -xdev is required because sysfs can cause EPERM
testD = `FROM busybox
RUN find / -xdev -name ba*
RUN find /tmp/`
}
server := fakestorage.New(c, "", fakecontext.WithFiles(map[string]string{"testD": testD}))
defer server.Close()
res, body, err := request.Post("/build?dockerfile=baz&remote="+server.URL()+"/testD", request.JSON)
c.Assert(err, checker.IsNil)
c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
buf, err := request.ReadBody(body)
c.Assert(err, checker.IsNil)
// Make sure Dockerfile exists.
// Make sure 'baz' doesn't exist ANYWHERE despite being mentioned in the URL
out := string(buf)
c.Assert(out, checker.Contains, "RUN find /tmp")
c.Assert(out, checker.Not(checker.Contains), "baz")
}
func (s *DockerSuite) TestBuildAPIRemoteTarballContext(c *check.C) {
buffer := new(bytes.Buffer)
tw := tar.NewWriter(buffer)
defer tw.Close()
dockerfile := []byte("FROM busybox")
err := tw.WriteHeader(&tar.Header{
Name: "Dockerfile",
Size: int64(len(dockerfile)),
})
// failed to write tar file header
c.Assert(err, checker.IsNil)
_, err = tw.Write(dockerfile)
// failed to write tar file content
c.Assert(err, checker.IsNil)
// failed to close tar archive
c.Assert(tw.Close(), checker.IsNil)
server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{
"testT.tar": buffer,
}))
defer server.Close()
res, b, err := request.Post("/build?remote="+server.URL()+"/testT.tar", request.ContentType("application/tar"))
c.Assert(err, checker.IsNil)
c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
b.Close()
}
func (s *DockerSuite) TestBuildAPIRemoteTarballContextWithCustomDockerfile(c *check.C) {
buffer := new(bytes.Buffer)
tw := tar.NewWriter(buffer)
defer tw.Close()
dockerfile := []byte(`FROM busybox
RUN echo 'wrong'`)
err := tw.WriteHeader(&tar.Header{
Name: "Dockerfile",
Size: int64(len(dockerfile)),
})
// failed to write tar file header
c.Assert(err, checker.IsNil)
_, err = tw.Write(dockerfile)
// failed to write tar file content
c.Assert(err, checker.IsNil)
custom := []byte(`FROM busybox
RUN echo 'right'
`)
err = tw.WriteHeader(&tar.Header{
Name: "custom",
Size: int64(len(custom)),
})
// failed to write tar file header
c.Assert(err, checker.IsNil)
_, err = tw.Write(custom)
// failed to write tar file content
c.Assert(err, checker.IsNil)
// failed to close tar archive
c.Assert(tw.Close(), checker.IsNil)
server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{
"testT.tar": buffer,
}))
defer server.Close()
url := "/build?dockerfile=custom&remote=" + server.URL() + "/testT.tar"
res, body, err := request.Post(url, request.ContentType("application/tar"))
c.Assert(err, checker.IsNil)
c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
defer body.Close()
content, err := request.ReadBody(body)
c.Assert(err, checker.IsNil)
// Build used the wrong dockerfile.
c.Assert(string(content), checker.Not(checker.Contains), "wrong")
}
func (s *DockerSuite) TestBuildAPILowerDockerfile(c *check.C) {
git := fakegit.New(c, "repo", map[string]string{
"dockerfile": `FROM busybox
RUN echo from dockerfile`,
}, false)
defer git.Close()
res, body, err := request.Post("/build?remote="+git.RepoURL, request.JSON)
c.Assert(err, checker.IsNil)
c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
buf, err := request.ReadBody(body)
c.Assert(err, checker.IsNil)
out := string(buf)
c.Assert(out, checker.Contains, "from dockerfile")
}
func (s *DockerSuite) TestBuildAPIBuildGitWithF(c *check.C) {
git := fakegit.New(c, "repo", map[string]string{
"baz": `FROM busybox
RUN echo from baz`,
"Dockerfile": `FROM busybox
RUN echo from Dockerfile`,
}, false)
defer git.Close()
// Make sure it tries to 'dockerfile' query param value
res, body, err := request.Post("/build?dockerfile=baz&remote="+git.RepoURL, request.JSON)
c.Assert(err, checker.IsNil)
c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
buf, err := request.ReadBody(body)
c.Assert(err, checker.IsNil)
out := string(buf)
c.Assert(out, checker.Contains, "from baz")
}
func (s *DockerSuite) TestBuildAPIDoubleDockerfile(c *check.C) {
testRequires(c, UnixCli) // dockerfile overwrites Dockerfile on Windows
git := fakegit.New(c, "repo", map[string]string{
"Dockerfile": `FROM busybox
RUN echo from Dockerfile`,
"dockerfile": `FROM busybox
RUN echo from dockerfile`,
}, false)
defer git.Close()
// Make sure it tries to 'dockerfile' query param value
res, body, err := request.Post("/build?remote="+git.RepoURL, request.JSON)
c.Assert(err, checker.IsNil)
c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
buf, err := request.ReadBody(body)
c.Assert(err, checker.IsNil)
out := string(buf)
c.Assert(out, checker.Contains, "from Dockerfile")
}
func (s *DockerSuite) TestBuildAPIUnnormalizedTarPaths(c *check.C) {
// Make sure that build context tars with entries of the form
// x/./y don't cause caching false positives.
buildFromTarContext := func(fileContents []byte) string {
buffer := new(bytes.Buffer)
tw := tar.NewWriter(buffer)
defer tw.Close()
dockerfile := []byte(`FROM busybox
COPY dir /dir/`)
err := tw.WriteHeader(&tar.Header{
Name: "Dockerfile",
Size: int64(len(dockerfile)),
})
//failed to write tar file header
c.Assert(err, checker.IsNil)
_, err = tw.Write(dockerfile)
// failed to write Dockerfile in tar file content
c.Assert(err, checker.IsNil)
err = tw.WriteHeader(&tar.Header{
Name: "dir/./file",
Size: int64(len(fileContents)),
})
//failed to write tar file header
c.Assert(err, checker.IsNil)
_, err = tw.Write(fileContents)
// failed to write file contents in tar file content
c.Assert(err, checker.IsNil)
// failed to close tar archive
c.Assert(tw.Close(), checker.IsNil)
res, body, err := request.Post("/build", request.RawContent(ioutil.NopCloser(buffer)), request.ContentType("application/x-tar"))
c.Assert(err, checker.IsNil)
c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
out, err := request.ReadBody(body)
c.Assert(err, checker.IsNil)
lines := strings.Split(string(out), "\n")
c.Assert(len(lines), checker.GreaterThan, 1)
c.Assert(lines[len(lines)-2], checker.Matches, ".*Successfully built [0-9a-f]{12}.*")
re := regexp.MustCompile("Successfully built ([0-9a-f]{12})")
matches := re.FindStringSubmatch(lines[len(lines)-2])
return matches[1]
}
imageA := buildFromTarContext([]byte("abc"))
imageB := buildFromTarContext([]byte("def"))
c.Assert(imageA, checker.Not(checker.Equals), imageB)
}
func (s *DockerSuite) TestBuildOnBuildWithCopy(c *check.C) {
dockerfile := `
FROM ` + minimalBaseImage() + ` as onbuildbase
ONBUILD COPY file /file
FROM onbuildbase
`
ctx := fakecontext.New(c, "",
fakecontext.WithDockerfile(dockerfile),
fakecontext.WithFile("file", "some content"),
)
defer ctx.Close()
res, body, err := request.Post(
"/build",
request.RawContent(ctx.AsTarReader(c)),
request.ContentType("application/x-tar"))
c.Assert(err, checker.IsNil)
c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
out, err := request.ReadBody(body)
c.Assert(err, checker.IsNil)
c.Assert(string(out), checker.Contains, "Successfully built")
}
func (s *DockerSuite) TestBuildOnBuildCache(c *check.C) {
build := func(dockerfile string) []byte {
ctx := fakecontext.New(c, "",
fakecontext.WithDockerfile(dockerfile),
)
defer ctx.Close()
res, body, err := request.Post(
"/build",
request.RawContent(ctx.AsTarReader(c)),
request.ContentType("application/x-tar"))
require.NoError(c, err)
assert.Equal(c, http.StatusOK, res.StatusCode)
out, err := request.ReadBody(body)
require.NoError(c, err)
assert.Contains(c, string(out), "Successfully built")
return out
}
dockerfile := `
FROM ` + minimalBaseImage() + ` as onbuildbase
ENV something=bar
ONBUILD ENV foo=bar
`
build(dockerfile)
dockerfile += "FROM onbuildbase"
out := build(dockerfile)
imageIDs := getImageIDsFromBuild(c, out)
assert.Len(c, imageIDs, 2)
parentID, childID := imageIDs[0], imageIDs[1]
client, err := request.NewClient()
require.NoError(c, err)
// check parentID is correct
image, _, err := client.ImageInspectWithRaw(context.Background(), childID)
require.NoError(c, err)
assert.Equal(c, parentID, image.Parent)
}
func (s *DockerRegistrySuite) TestBuildCopyFromForcePull(c *check.C) {
client, err := request.NewClient()
require.NoError(c, err)
repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL)
// tag the image to upload it to the private registry
err = client.ImageTag(context.TODO(), "busybox", repoName)
assert.Nil(c, err)
// push the image to the registry
rc, err := client.ImagePush(context.TODO(), repoName, types.ImagePushOptions{RegistryAuth: "{}"})
assert.Nil(c, err)
_, err = io.Copy(ioutil.Discard, rc)
assert.Nil(c, err)
dockerfile := fmt.Sprintf(`
FROM %s AS foo
RUN touch abc
FROM %s
COPY --from=foo /abc /
`, repoName, repoName)
ctx := fakecontext.New(c, "",
fakecontext.WithDockerfile(dockerfile),
)
defer ctx.Close()
res, body, err := request.Post(
"/build?pull=1",
request.RawContent(ctx.AsTarReader(c)),
request.ContentType("application/x-tar"))
require.NoError(c, err)
assert.Equal(c, http.StatusOK, res.StatusCode)
out, err := request.ReadBody(body)
require.NoError(c, err)
assert.Contains(c, string(out), "Successfully built")
}
func (s *DockerSuite) TestBuildAddRemoteNoDecompress(c *check.C) {
buffer := new(bytes.Buffer)
tw := tar.NewWriter(buffer)
dt := []byte("contents")
err := tw.WriteHeader(&tar.Header{
Name: "foo",
Size: int64(len(dt)),
Mode: 0600,
Typeflag: tar.TypeReg,
})
require.NoError(c, err)
_, err = tw.Write(dt)
require.NoError(c, err)
err = tw.Close()
require.NoError(c, err)
server := fakestorage.New(c, "", fakecontext.WithBinaryFiles(map[string]*bytes.Buffer{
"test.tar": buffer,
}))
defer server.Close()
dockerfile := fmt.Sprintf(`
FROM busybox
ADD %s/test.tar /
RUN [ -f test.tar ]
`, server.URL())
ctx := fakecontext.New(c, "",
fakecontext.WithDockerfile(dockerfile),
)
defer ctx.Close()
res, body, err := request.Post(
"/build",
request.RawContent(ctx.AsTarReader(c)),
request.ContentType("application/x-tar"))
require.NoError(c, err)
assert.Equal(c, http.StatusOK, res.StatusCode)
out, err := request.ReadBody(body)
require.NoError(c, err)
assert.Contains(c, string(out), "Successfully built")
}
func (s *DockerSuite) TestBuildChownOnCopy(c *check.C) {
testRequires(c, DaemonIsLinux)
dockerfile := `FROM busybox
RUN echo 'test1:x:1001:1001::/bin:/bin/false' >> /etc/passwd
RUN echo 'test1:x:1001:' >> /etc/group
RUN echo 'test2:x:1002:' >> /etc/group
COPY --chown=test1:1002 . /new_dir
RUN ls -l /
RUN [ $(ls -l / | grep new_dir | awk '{print $3":"$4}') = 'test1:test2' ]
RUN [ $(ls -nl / | grep new_dir | awk '{print $3":"$4}') = '1001:1002' ]
`
ctx := fakecontext.New(c, "",
fakecontext.WithDockerfile(dockerfile),
fakecontext.WithFile("test_file1", "some test content"),
)
defer ctx.Close()
res, body, err := request.Post(
"/build",
request.RawContent(ctx.AsTarReader(c)),
request.ContentType("application/x-tar"))
c.Assert(err, checker.IsNil)
c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
out, err := request.ReadBody(body)
require.NoError(c, err)
assert.Contains(c, string(out), "Successfully built")
}
func (s *DockerSuite) TestBuildCopyCacheOnFileChange(c *check.C) {
dockerfile := `FROM busybox
COPY file /file`
ctx1 := fakecontext.New(c, "",
fakecontext.WithDockerfile(dockerfile),
fakecontext.WithFile("file", "foo"))
ctx2 := fakecontext.New(c, "",
fakecontext.WithDockerfile(dockerfile),
fakecontext.WithFile("file", "bar"))
var build = func(ctx *fakecontext.Fake) string {
res, body, err := request.Post("/build",
request.RawContent(ctx.AsTarReader(c)),
request.ContentType("application/x-tar"))
require.NoError(c, err)
assert.Equal(c, http.StatusOK, res.StatusCode)
out, err := request.ReadBody(body)
require.NoError(c, err)
ids := getImageIDsFromBuild(c, out)
return ids[len(ids)-1]
}
id1 := build(ctx1)
id2 := build(ctx1)
id3 := build(ctx2)
if id1 != id2 {
c.Fatal("didn't use the cache")
}
if id1 == id3 {
c.Fatal("COPY With different source file should not share same cache")
}
}
func (s *DockerSuite) TestBuildAddCacheOnFileChange(c *check.C) {
dockerfile := `FROM busybox
ADD file /file`
ctx1 := fakecontext.New(c, "",
fakecontext.WithDockerfile(dockerfile),
fakecontext.WithFile("file", "foo"))
ctx2 := fakecontext.New(c, "",
fakecontext.WithDockerfile(dockerfile),
fakecontext.WithFile("file", "bar"))
var build = func(ctx *fakecontext.Fake) string {
res, body, err := request.Post("/build",
request.RawContent(ctx.AsTarReader(c)),
request.ContentType("application/x-tar"))
require.NoError(c, err)
assert.Equal(c, http.StatusOK, res.StatusCode)
out, err := request.ReadBody(body)
require.NoError(c, err)
ids := getImageIDsFromBuild(c, out)
return ids[len(ids)-1]
}
id1 := build(ctx1)
id2 := build(ctx1)
id3 := build(ctx2)
if id1 != id2 {
c.Fatal("didn't use the cache")
}
if id1 == id3 {
c.Fatal("COPY With different source file should not share same cache")
}
}
func (s *DockerSuite) TestBuildWithSession(c *check.C) {
testRequires(c, ExperimentalDaemon)
dockerfile := `
FROM busybox
COPY file /
RUN cat /file
`
fctx := fakecontext.New(c, "",
fakecontext.WithFile("file", "some content"),
)
defer fctx.Close()
out := testBuildWithSession(c, fctx.Dir, dockerfile)
assert.Contains(c, out, "some content")
fctx.Add("second", "contentcontent")
dockerfile += `
COPY second /
RUN cat /second
`
out = testBuildWithSession(c, fctx.Dir, dockerfile)
assert.Equal(c, strings.Count(out, "Using cache"), 2)
assert.Contains(c, out, "contentcontent")
client, err := request.NewClient()
require.NoError(c, err)
du, err := client.DiskUsage(context.TODO())
assert.Nil(c, err)
assert.True(c, du.BuilderSize > 10)
out = testBuildWithSession(c, fctx.Dir, dockerfile)
assert.Equal(c, strings.Count(out, "Using cache"), 4)
du2, err := client.DiskUsage(context.TODO())
assert.Nil(c, err)
assert.Equal(c, du.BuilderSize, du2.BuilderSize)
// rebuild with regular tar, confirm cache still applies
fctx.Add("Dockerfile", dockerfile)
res, body, err := request.Post(
"/build",
request.RawContent(fctx.AsTarReader(c)),
request.ContentType("application/x-tar"))
require.NoError(c, err)
assert.Equal(c, http.StatusOK, res.StatusCode)
outBytes, err := request.ReadBody(body)
require.NoError(c, err)
assert.Contains(c, string(outBytes), "Successfully built")
assert.Equal(c, strings.Count(string(outBytes), "Using cache"), 4)
_, err = client.BuildCachePrune(context.TODO())
assert.Nil(c, err)
du, err = client.DiskUsage(context.TODO())
assert.Nil(c, err)
assert.Equal(c, du.BuilderSize, int64(0))
}
func testBuildWithSession(c *check.C, dir, dockerfile string) (outStr string) {
client, err := request.NewClient()
require.NoError(c, err)
sess, err := session.NewSession("foo1", "foo")
assert.Nil(c, err)
fsProvider := filesync.NewFSSyncProvider([]filesync.SyncedDir{
{Dir: dir},
})
sess.Allow(fsProvider)
g, ctx := errgroup.WithContext(context.Background())
g.Go(func() error {
return sess.Run(ctx, client.DialSession)
})
g.Go(func() error {
res, body, err := request.Post("/build?remote=client-session&session="+sess.ID(), func(req *http.Request) error {
req.Body = ioutil.NopCloser(strings.NewReader(dockerfile))
return nil
})
if err != nil {
return err
}
assert.Equal(c, res.StatusCode, http.StatusOK)
out, err := request.ReadBody(body)
require.NoError(c, err)
assert.Contains(c, string(out), "Successfully built")
sess.Close()
outStr = string(out)
return nil
})
err = g.Wait()
assert.Nil(c, err)
return
}
func (s *DockerSuite) TestBuildScratchCopy(c *check.C) {
testRequires(c, DaemonIsLinux)
dockerfile := `FROM scratch
ADD Dockerfile /
ENV foo bar`
ctx := fakecontext.New(c, "",
fakecontext.WithDockerfile(dockerfile),
)
defer ctx.Close()
res, body, err := request.Post(
"/build",
request.RawContent(ctx.AsTarReader(c)),
request.ContentType("application/x-tar"))
c.Assert(err, checker.IsNil)
c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
out, err := request.ReadBody(body)
require.NoError(c, err)
assert.Contains(c, string(out), "Successfully built")
}
type buildLine struct {
Stream string
Aux struct {
ID string
}
}
func getImageIDsFromBuild(c *check.C, output []byte) []string {
ids := []string{}
for _, line := range bytes.Split(output, []byte("\n")) {
if len(line) == 0 {
continue
}
entry := buildLine{}
require.NoError(c, json.Unmarshal(line, &entry))
if entry.Aux.ID != "" {
ids = append(ids, entry.Aux.ID)
}
}
return ids
}