From 183076e89df64928bd2e94ad0da9725b482367cd Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Fri, 30 Mar 2018 13:10:02 +0200 Subject: [PATCH] Migrate test-integration-cli experimental build tests to integration All `docker build` tests that require an `ExperimentalDaemon` are migrated to `integration/build` package and start an experimental daemon to test on it. The end goal being to remove the `experimental` builds. Signed-off-by: Vincent Demeester --- integration-cli/docker_api_build_test.go | 103 ------------------ integration-cli/docker_cli_build_test.go | 70 ------------ integration/build/build_session_test.go | 132 +++++++++++++++++++++++ integration/build/build_squash_test.go | 108 +++++++++++++++++++ 4 files changed, 240 insertions(+), 173 deletions(-) create mode 100644 integration/build/build_session_test.go create mode 100644 integration/build/build_squash_test.go diff --git a/integration-cli/docker_api_build_test.go b/integration-cli/docker_api_build_test.go index cae7c1afe7..93469f1d47 100644 --- a/integration-cli/docker_api_build_test.go +++ b/integration-cli/docker_api_build_test.go @@ -20,10 +20,7 @@ import ( "github.com/go-check/check" "github.com/gotestyourself/gotestyourself/assert" is "github.com/gotestyourself/gotestyourself/assert/cmp" - "github.com/moby/buildkit/session" - "github.com/moby/buildkit/session/filesync" "golang.org/x/net/context" - "golang.org/x/sync/errgroup" ) func (s *DockerSuite) TestBuildAPIDockerFileRemote(c *check.C) { @@ -515,106 +512,6 @@ ADD file /file` } } -func (s *DockerSuite) TestBuildWithSession(c *check.C) { - testRequires(c, ExperimentalDaemon) - - dockerfile := ` - FROM busybox - COPY file / - RUN cat /file - ` - - fctx := fakecontext.New(c, "", - fakecontext.WithFile("file", "some content"), - ) - defer fctx.Close() - - out := testBuildWithSession(c, fctx.Dir, dockerfile) - assert.Check(c, is.Contains(out, "some content")) - - fctx.Add("second", "contentcontent") - - dockerfile += ` - COPY second / - RUN cat /second - ` - - out = testBuildWithSession(c, fctx.Dir, dockerfile) - assert.Check(c, is.Equal(strings.Count(out, "Using cache"), 2)) - assert.Check(c, is.Contains(out, "contentcontent")) - - client := testEnv.APIClient() - du, err := client.DiskUsage(context.TODO()) - assert.Check(c, err) - assert.Check(c, du.BuilderSize > 10) - - out = testBuildWithSession(c, fctx.Dir, dockerfile) - assert.Check(c, is.Equal(strings.Count(out, "Using cache"), 4)) - - du2, err := client.DiskUsage(context.TODO()) - assert.Check(c, err) - assert.Check(c, is.Equal(du.BuilderSize, du2.BuilderSize)) - - // rebuild with regular tar, confirm cache still applies - fctx.Add("Dockerfile", dockerfile) - res, body, err := request.Post( - "/build", - request.RawContent(fctx.AsTarReader(c)), - request.ContentType("application/x-tar")) - assert.NilError(c, err) - assert.Check(c, is.DeepEqual(http.StatusOK, res.StatusCode)) - - outBytes, err := request.ReadBody(body) - assert.NilError(c, err) - assert.Check(c, is.Contains(string(outBytes), "Successfully built")) - assert.Check(c, is.Equal(strings.Count(string(outBytes), "Using cache"), 4)) - - _, err = client.BuildCachePrune(context.TODO()) - assert.Check(c, err) - - du, err = client.DiskUsage(context.TODO()) - assert.Check(c, err) - assert.Check(c, is.Equal(du.BuilderSize, int64(0))) -} - -func testBuildWithSession(c *check.C, dir, dockerfile string) (outStr string) { - client := testEnv.APIClient() - sess, err := session.NewSession("foo1", "foo") - assert.Check(c, err) - - fsProvider := filesync.NewFSSyncProvider([]filesync.SyncedDir{ - {Dir: dir}, - }) - sess.Allow(fsProvider) - - g, ctx := errgroup.WithContext(context.Background()) - - g.Go(func() error { - return sess.Run(ctx, client.DialSession) - }) - - g.Go(func() error { - res, body, err := request.Post("/build?remote=client-session&session="+sess.ID(), func(req *http.Request) error { - req.Body = ioutil.NopCloser(strings.NewReader(dockerfile)) - return nil - }) - if err != nil { - return err - } - assert.Check(c, is.DeepEqual(res.StatusCode, http.StatusOK)) - out, err := request.ReadBody(body) - assert.NilError(c, err) - assert.Check(c, is.Contains(string(out), "Successfully built")) - sess.Close() - outStr = string(out) - return nil - }) - - err = g.Wait() - assert.Check(c, err) - return -} - func (s *DockerSuite) TestBuildScratchCopy(c *check.C) { testRequires(c, DaemonIsLinux) dockerfile := `FROM scratch diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index 1e29c53907..030f386c41 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -5598,46 +5598,6 @@ func (s *DockerSuite) TestBuildWithExtraHostInvalidFormat(c *check.C) { } -func (s *DockerSuite) TestBuildSquashParent(c *check.C) { - testRequires(c, ExperimentalDaemon) - dockerFile := ` - FROM busybox - RUN echo hello > /hello - RUN echo world >> /hello - RUN echo hello > /remove_me - ENV HELLO world - RUN rm /remove_me - ` - // build and get the ID that we can use later for history comparison - name := "test" - buildImageSuccessfully(c, name, build.WithDockerfile(dockerFile)) - origID := getIDByName(c, name) - - // build with squash - buildImageSuccessfully(c, name, cli.WithFlags("--squash"), build.WithDockerfile(dockerFile)) - id := getIDByName(c, name) - - out, _ := dockerCmd(c, "run", "--rm", id, "/bin/sh", "-c", "cat /hello") - c.Assert(strings.TrimSpace(out), checker.Equals, "hello\nworld") - - dockerCmd(c, "run", "--rm", id, "/bin/sh", "-c", "[ ! -f /remove_me ]") - dockerCmd(c, "run", "--rm", id, "/bin/sh", "-c", `[ "$(echo $HELLO)" == "world" ]`) - - // make sure the ID produced is the ID of the tag we specified - inspectID := inspectImage(c, "test", ".ID") - c.Assert(inspectID, checker.Equals, id) - - origHistory, _ := dockerCmd(c, "history", origID) - testHistory, _ := dockerCmd(c, "history", "test") - - splitOrigHistory := strings.Split(strings.TrimSpace(origHistory), "\n") - splitTestHistory := strings.Split(strings.TrimSpace(testHistory), "\n") - c.Assert(len(splitTestHistory), checker.Equals, len(splitOrigHistory)+1) - - out = inspectImage(c, id, "len .RootFS.Layers") - c.Assert(strings.TrimSpace(out), checker.Equals, "2") -} - func (s *DockerSuite) TestBuildContChar(c *check.C) { name := "testbuildcontchar" @@ -6237,33 +6197,3 @@ func (s *DockerSuite) TestBuildIidFileCleanupOnFail(c *check.C) { c.Assert(err, check.NotNil) c.Assert(os.IsNotExist(err), check.Equals, true) } - -// FIXME(vdemeester) should migrate to docker/cli tests -func (s *DockerSuite) TestBuildIidFileSquash(c *check.C) { - testRequires(c, ExperimentalDaemon) - tmpDir, err := ioutil.TempDir("", "TestBuildIidFileSquash") - if err != nil { - c.Fatal(err) - } - defer os.RemoveAll(tmpDir) - tmpIidFile := filepath.Join(tmpDir, "iidsquash") - - name := "testbuildiidfilesquash" - // Use a Dockerfile with multiple stages to ensure we get the last one - cli.BuildCmd(c, name, - // This could be minimalBaseImage except - // https://github.com/moby/moby/issues/33823 requires - // `touch` to workaround. - build.WithDockerfile(`FROM busybox -ENV FOO FOO -ENV BAR BAR -RUN touch /foop -`), - cli.WithFlags("--iidfile", tmpIidFile, "--squash")) - - id, err := ioutil.ReadFile(tmpIidFile) - c.Assert(err, check.IsNil) - d, err := digest.Parse(string(id)) - c.Assert(err, check.IsNil) - c.Assert(d.String(), checker.Equals, getIDByName(c, name)) -} diff --git a/integration/build/build_session_test.go b/integration/build/build_session_test.go new file mode 100644 index 0000000000..1fde6a0f9a --- /dev/null +++ b/integration/build/build_session_test.go @@ -0,0 +1,132 @@ +package build + +import ( + "context" + "io/ioutil" + "net/http" + "strings" + "testing" + + dclient "github.com/docker/docker/client" + "github.com/docker/docker/integration-cli/cli/build/fakecontext" + "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/integration-cli/request" + "github.com/gotestyourself/gotestyourself/assert" + is "github.com/gotestyourself/gotestyourself/assert/cmp" + "github.com/moby/buildkit/session" + "github.com/moby/buildkit/session/filesync" + "golang.org/x/sync/errgroup" +) + +func TestBuildWithSession(t *testing.T) { + d := daemon.New(t, "", "dockerd", daemon.Config{ + Experimental: true, + }) + d.StartWithBusybox(t) + defer d.Stop(t) + + client, err := d.NewClient() + assert.NilError(t, err) + + dockerfile := ` + FROM busybox + COPY file / + RUN cat /file + ` + + fctx := fakecontext.New(t, "", + fakecontext.WithFile("file", "some content"), + ) + defer fctx.Close() + + out := testBuildWithSession(t, client, d.Sock(), fctx.Dir, dockerfile) + assert.Check(t, is.Contains(out, "some content")) + + fctx.Add("second", "contentcontent") + + dockerfile += ` + COPY second / + RUN cat /second + ` + + out = testBuildWithSession(t, client, d.Sock(), fctx.Dir, dockerfile) + assert.Check(t, is.Equal(strings.Count(out, "Using cache"), 2)) + assert.Check(t, is.Contains(out, "contentcontent")) + + du, err := client.DiskUsage(context.TODO()) + assert.Check(t, err) + assert.Check(t, du.BuilderSize > 10) + + out = testBuildWithSession(t, client, d.Sock(), fctx.Dir, dockerfile) + assert.Check(t, is.Equal(strings.Count(out, "Using cache"), 4)) + + du2, err := client.DiskUsage(context.TODO()) + assert.Check(t, err) + assert.Check(t, is.Equal(du.BuilderSize, du2.BuilderSize)) + + // rebuild with regular tar, confirm cache still applies + fctx.Add("Dockerfile", dockerfile) + // FIXME(vdemeester) use sock here + res, body, err := request.DoOnHost(d.Sock(), + "/build", + request.Method(http.MethodPost), + request.RawContent(fctx.AsTarReader(t)), + request.ContentType("application/x-tar")) + assert.NilError(t, err) + assert.Check(t, is.DeepEqual(http.StatusOK, res.StatusCode)) + + outBytes, err := request.ReadBody(body) + assert.NilError(t, err) + assert.Check(t, is.Contains(string(outBytes), "Successfully built")) + assert.Check(t, is.Equal(strings.Count(string(outBytes), "Using cache"), 4)) + + _, err = client.BuildCachePrune(context.TODO()) + assert.Check(t, err) + + du, err = client.DiskUsage(context.TODO()) + assert.Check(t, err) + assert.Check(t, is.Equal(du.BuilderSize, int64(0))) +} + +func testBuildWithSession(t *testing.T, client dclient.APIClient, daemonSock string, dir, dockerfile string) (outStr string) { + sess, err := session.NewSession("foo1", "foo") + assert.Check(t, err) + + fsProvider := filesync.NewFSSyncProvider([]filesync.SyncedDir{ + {Dir: dir}, + }) + sess.Allow(fsProvider) + + g, ctx := errgroup.WithContext(context.Background()) + + g.Go(func() error { + return sess.Run(ctx, client.DialSession) + }) + + g.Go(func() error { + // FIXME use sock here + res, body, err := request.DoOnHost( + daemonSock, + "/build?remote=client-session&session="+sess.ID(), + request.Method(http.MethodPost), + func(req *http.Request) error { + req.Body = ioutil.NopCloser(strings.NewReader(dockerfile)) + return nil + }, + ) + if err != nil { + return err + } + assert.Check(t, is.DeepEqual(res.StatusCode, http.StatusOK)) + out, err := request.ReadBody(body) + assert.NilError(t, err) + assert.Check(t, is.Contains(string(out), "Successfully built")) + sess.Close() + outStr = string(out) + return nil + }) + + err = g.Wait() + assert.Check(t, err) + return +} diff --git a/integration/build/build_squash_test.go b/integration/build/build_squash_test.go new file mode 100644 index 0000000000..7f264f1477 --- /dev/null +++ b/integration/build/build_squash_test.go @@ -0,0 +1,108 @@ +package build + +import ( + "bytes" + "context" + "io" + "io/ioutil" + "strings" + "testing" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/integration-cli/cli/build/fakecontext" + "github.com/docker/docker/integration-cli/daemon" + "github.com/docker/docker/integration/internal/container" + "github.com/docker/docker/pkg/stdcopy" + "github.com/gotestyourself/gotestyourself/assert" + is "github.com/gotestyourself/gotestyourself/assert/cmp" +) + +func TestBuildSquashParent(t *testing.T) { + d := daemon.New(t, "", "dockerd", daemon.Config{ + Experimental: true, + }) + d.StartWithBusybox(t) + defer d.Stop(t) + + client, err := d.NewClient() + assert.NilError(t, err) + + dockerfile := ` + FROM busybox + RUN echo hello > /hello + RUN echo world >> /hello + RUN echo hello > /remove_me + ENV HELLO world + RUN rm /remove_me + ` + + // build and get the ID that we can use later for history comparison + ctx := context.Background() + source := fakecontext.New(t, "", fakecontext.WithDockerfile(dockerfile)) + defer source.Close() + + name := "test" + resp, err := client.ImageBuild(ctx, + source.AsTarReader(t), + types.ImageBuildOptions{ + Remove: true, + ForceRemove: true, + Tags: []string{name}, + }) + assert.NilError(t, err) + _, err = io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + assert.NilError(t, err) + + inspect, _, err := client.ImageInspectWithRaw(ctx, name) + assert.NilError(t, err) + origID := inspect.ID + + // build with squash + resp, err = client.ImageBuild(ctx, + source.AsTarReader(t), + types.ImageBuildOptions{ + Remove: true, + ForceRemove: true, + Squash: true, + Tags: []string{name}, + }) + assert.NilError(t, err) + _, err = io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + assert.NilError(t, err) + + cid := container.Run(t, ctx, client, + container.WithImage(name), + container.WithCmd("/bin/sh", "-c", "cat /hello"), + ) + reader, err := client.ContainerLogs(ctx, cid, types.ContainerLogsOptions{ + ShowStdout: true, + }) + assert.NilError(t, err) + + actualStdout := new(bytes.Buffer) + actualStderr := ioutil.Discard + _, err = stdcopy.StdCopy(actualStdout, actualStderr, reader) + assert.NilError(t, err) + assert.Check(t, is.Equal(strings.TrimSpace(actualStdout.String()), "hello\nworld")) + + container.Run(t, ctx, client, + container.WithImage(name), + container.WithCmd("/bin/sh", "-c", "[ ! -f /remove_me ]"), + ) + container.Run(t, ctx, client, + container.WithImage(name), + container.WithCmd("/bin/sh", "-c", `[ "$(echo $HELLO)" == "world" ]`), + ) + + origHistory, err := client.ImageHistory(ctx, origID) + assert.NilError(t, err) + testHistory, err := client.ImageHistory(ctx, name) + assert.NilError(t, err) + + inspect, _, err = client.ImageInspectWithRaw(ctx, name) + assert.NilError(t, err) + assert.Check(t, is.Len(testHistory, len(origHistory)+1)) + assert.Check(t, is.Len(inspect.RootFS.Layers, 2)) +}