From 6b025a8b665f7b976bb2560488ba07f55dbf0fb6 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Tue, 23 May 2017 13:32:34 -0700 Subject: [PATCH] Introduce `test-integration` target (and deprecate/freeze `test-integration-cli`) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a new package `integration` where `engine` integration tests should live. Those integration tests should not depends on any `cli` components (except from the `dockerd` daemon for now — to actually start a daemon). Signed-off-by: Vincent Demeester --- CONTRIBUTING.md | 4 + Makefile | 7 +- hack/make.sh | 1 + hack/make/.integration-test-helpers | 19 +++ hack/make/test-integration | 28 +++++ integration-cli/docker_api_create_test.go | 76 ------------ integration/container/create_test.go | 141 ++++++++++++++++++++++ integration/doc.go | 3 + 8 files changed, 201 insertions(+), 78 deletions(-) create mode 100755 hack/make/test-integration create mode 100644 integration/container/create_test.go create mode 100644 integration/doc.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 917214cd1b..75d14cda41 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -160,6 +160,10 @@ it! Take a look at existing tests for inspiration. [Run the full test suite](https://docs.docker.com/opensource/project/test-and-docs/) on your branch before submitting a pull request. +If your changes need integration tests, write them against the API. The `cli` +integration tests are slowly either migrated to API tests or moved away as unit +tests in `docker/cli` and end-to-end tests for docker. + Update the documentation when creating or modifying features. Test your documentation changes for clarity, concision, and correctness, as well as a clean documentation build. See our contributors guide for [our style diff --git a/Makefile b/Makefile index 0d99606cc2..4bbf5e2800 100644 --- a/Makefile +++ b/Makefile @@ -154,8 +154,11 @@ test: build ## run the unit, integration and docker-py tests test-docker-py: build ## run the docker-py tests $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-docker-py -test-integration-cli: build ## run the integration tests - $(DOCKER_RUN_DOCKER) hack/make.sh build-integration-test-binary dynbinary test-integration-cli +test-integration-cli: build ## (DEPRECATED) use test-integration + $(DOCKER_RUN_DOCKER) hack/make.sh build-integration-test-cli-binary dynbinary test-integration + +test-integration: build ## run the integration tests + $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration test-unit: build ## run the unit tests $(DOCKER_RUN_DOCKER) hack/make.sh test-unit diff --git a/hack/make.sh b/hack/make.sh index e625b863de..f5bf87fc35 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -60,6 +60,7 @@ DEFAULT_BUNDLES=( dynbinary test-unit + test-integration test-integration-cli test-docker-py diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers index 4ff9677c79..d557cba897 100644 --- a/hack/make/.integration-test-helpers +++ b/hack/make/.integration-test-helpers @@ -7,6 +7,25 @@ bundle_test_integration_cli() { go_test_dir integration-cli $DOCKER_INTEGRATION_TESTS_VERIFIED } +bundle_test_integration() { + TESTFLAGS="$TESTFLAGS -v -test.timeout=60m" + ( + set -e + cd integration + INCBUILD="-i" + count=0 + for flag in "${BUILDFLAGS[@]}"; do + if [ "${flag}" == ${INCBUILD} ]; then + unset BUILDFLAGS[${count}] + break + fi + count=$[ ${count} + 1 ] + done + echo go test -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS ./... + go test -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" $TESTFLAGS ./... + ) +} + # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. # You can use this to select certain tests to run, e.g. # diff --git a/hack/make/test-integration b/hack/make/test-integration new file mode 100755 index 0000000000..f5cd1fd63e --- /dev/null +++ b/hack/make/test-integration @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -e + +source hack/make/.integration-test-helpers + +# subshell so that we can export PATH without breaking other things +( + bundle .integration-daemon-start + + bundle .integration-daemon-setup + + bundle_test_integration + + bundle .integration-daemon-stop + + if [ "$(go env GOOS)" != 'windows' ] + then + leftovers=$(ps -ax -o pid,cmd | awk '$2 == "docker-containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration-cli/ { print $1 }') + if [ -n "$leftovers" ] + then + ps aux + kill -9 $leftovers 2> /dev/null + echo "!!!! WARNING you have left over shim(s), Cleanup your test !!!!" + exit 1 + fi + fi + +) 2>&1 | tee -a "$DEST/test.log" diff --git a/integration-cli/docker_api_create_test.go b/integration-cli/docker_api_create_test.go index e404b6cf58..c2152d32a4 100644 --- a/integration-cli/docker_api_create_test.go +++ b/integration-cli/docker_api_create_test.go @@ -11,82 +11,6 @@ import ( "github.com/go-check/check" ) -func (s *DockerSuite) TestAPICreateWithNotExistImage(c *check.C) { - name := "test" - config := map[string]interface{}{ - "Image": "test456:v1", - "Volumes": map[string]struct{}{"/tmp": {}}, - } - - status, body, err := request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusNotFound) - expected := "No such image: test456:v1" - c.Assert(getErrorMessage(c, body), checker.Contains, expected) - - config2 := map[string]interface{}{ - "Image": "test456", - "Volumes": map[string]struct{}{"/tmp": {}}, - } - - status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config2, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusNotFound) - expected = "No such image: test456:latest" - c.Assert(getErrorMessage(c, body), checker.Equals, expected) - - config3 := map[string]interface{}{ - "Image": "sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa", - } - - status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config3, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusNotFound) - expected = "No such image: sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa" - c.Assert(getErrorMessage(c, body), checker.Equals, expected) - -} - -// Test for #25099 -func (s *DockerSuite) TestAPICreateEmptyEnv(c *check.C) { - name := "test1" - config := map[string]interface{}{ - "Image": "busybox", - "Env": []string{"", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, - "Cmd": []string{"true"}, - } - - status, body, err := request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusInternalServerError) - expected := "invalid environment variable:" - c.Assert(getErrorMessage(c, body), checker.Contains, expected) - - name = "test2" - config = map[string]interface{}{ - "Image": "busybox", - "Env": []string{"=", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, - "Cmd": []string{"true"}, - } - status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusInternalServerError) - expected = "invalid environment variable: =" - c.Assert(getErrorMessage(c, body), checker.Contains, expected) - - name = "test3" - config = map[string]interface{}{ - "Image": "busybox", - "Env": []string{"=foo", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, - "Cmd": []string{"true"}, - } - status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusInternalServerError) - expected = "invalid environment variable: =foo" - c.Assert(getErrorMessage(c, body), checker.Contains, expected) -} - func (s *DockerSuite) TestAPICreateWithInvalidHealthcheckParams(c *check.C) { // test invalid Interval in Healthcheck: less than 0s name := "test1" diff --git a/integration/container/create_test.go b/integration/container/create_test.go new file mode 100644 index 0000000000..05cd5ccef4 --- /dev/null +++ b/integration/container/create_test.go @@ -0,0 +1,141 @@ +package container + +import ( + "context" + "fmt" + "os" + "testing" + + "strconv" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/client" + "github.com/docker/docker/integration-cli/environment" + "github.com/docker/docker/integration-cli/fixtures/load" + "github.com/stretchr/testify/require" +) + +var ( + testEnv *environment.Execution +) + +func TestMain(m *testing.M) { + var err error + testEnv, err = environment.New() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + if testEnv.LocalDaemon() { + fmt.Println("INFO: Testing against a local daemon") + } else { + fmt.Println("INFO: Testing against a remote daemon") + } + + // TODO: ensure and protect images + res := m.Run() + os.Exit(res) +} + +func TestAPICreateWithNotExistImage(t *testing.T) { + defer setupTest(t)() + clt := createClient(t) + + testCases := []struct { + image string + expectedError string + }{ + { + image: "test456:v1", + expectedError: "No such image: test456:v1", + }, + { + image: "test456", + expectedError: "No such image: test456", + }, + { + image: "sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa", + expectedError: "No such image: sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa", + }, + } + + for index, tc := range testCases { + tc := tc + t.Run(strconv.Itoa(index), func(t *testing.T) { + t.Parallel() + _, err := clt.ContainerCreate(context.Background(), + &container.Config{ + Image: tc.image, + }, + &container.HostConfig{}, + &network.NetworkingConfig{}, + "foo", + ) + require.Error(t, err) + require.Contains(t, err.Error(), tc.expectedError) + }) + } +} + +func TestAPICreateEmptyEnv(t *testing.T) { + defer setupTest(t)() + clt := createClient(t) + + testCases := []struct { + env string + expectedError string + }{ + { + env: "", + expectedError: "invalid environment variable:", + }, + { + env: "=", + expectedError: "invalid environment variable: =", + }, + { + env: "=foo", + expectedError: "invalid environment variable: =foo", + }, + } + + for index, tc := range testCases { + tc := tc + t.Run(strconv.Itoa(index), func(t *testing.T) { + t.Parallel() + _, err := clt.ContainerCreate(context.Background(), + &container.Config{ + Image: "busybox", + Env: []string{tc.env}, + }, + &container.HostConfig{}, + &network.NetworkingConfig{}, + "foo", + ) + require.Error(t, err) + require.Contains(t, err.Error(), tc.expectedError) + }) + } +} + +func createClient(t *testing.T) client.APIClient { + clt, err := client.NewEnvClient() + require.NoError(t, err) + return clt +} + +func setupTest(t *testing.T) func() { + if testEnv.DaemonPlatform() == "linux" { + images := []string{"busybox:latest", "hello-world:frozen", "debian:jessie"} + err := load.FrozenImagesLinux(testEnv.DockerBinary(), images...) + if err != nil { + t.Fatalf("%+v", err) + } + defer testEnv.ProtectImage(t, images...) + } + return func() { + testEnv.Clean(t, testEnv.DockerBinary()) + } +} diff --git a/integration/doc.go b/integration/doc.go new file mode 100644 index 0000000000..2fdf62eef7 --- /dev/null +++ b/integration/doc.go @@ -0,0 +1,3 @@ +// Package integration provides integrations tests for Moby (API). +// These tests require a daemon (dockerd for now) to run. +package integration