From 1887d85e2157d909c53774c7cdfbc745c1298629 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 16 Jun 2022 22:47:36 +0200 Subject: [PATCH] ci(integration-cli): dynamically split tests in matrix Signed-off-by: CrazyMax --- .github/workflows/windows.yml | 42 +++++++++++++++++++++++++++-- Dockerfile.e2e | 2 +- docs/contributing/test.md | 6 ++--- hack/make/.integration-test-helpers | 2 +- 4 files changed, 45 insertions(+), 7 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index f6592b5277..8fc1a324c7 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -16,6 +16,7 @@ on: env: GO_VERSION: 1.18.3 + GOTESTLIST_VERSION: v0.2.0 WINDOWS_BASE_IMAGE: mcr.microsoft.com/windows/servercore WINDOWS_BASE_TAG_2019: ltsc2019 WINDOWS_BASE_TAG_2022: ltsc2022 @@ -188,10 +189,43 @@ jobs: name: ${{ matrix.os }}-unit-reports path: ${{ env.GOPATH }}\src\github.com\docker\docker\bundles\* + integration-test-prepare: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.tests.outputs.matrix }} + steps: + - + name: Checkout + uses: actions/checkout@v3 + - + name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + - + name: Install gotestlist + run: + go install github.com/crazy-max/gotestlist/cmd/gotestlist@${{ env.GOTESTLIST_VERSION }} + - + name: Create matrix + id: tests + working-directory: ./integration-cli + run: | + # Distribute integration-cli tests for the matrix in integration-test job. + # Also prepend ./... to the matrix. This is a special case to run "Test integration" step exclusively. + matrix="$(gotestlist -d 4 ./...)" + matrix="$(echo "$matrix" | jq -c '. |= ["./..."] + .')" + echo "::set-output name=matrix::$matrix" + - + name: Show matrix + run: | + echo ${{ steps.tests.outputs.matrix }} + integration-test: runs-on: ${{ matrix.os }} needs: - build + - integration-test-prepare strategy: fail-fast: false matrix: @@ -201,6 +235,7 @@ jobs: runtime: - builtin - containerd + test: ${{ fromJson(needs.integration-test-prepare.outputs.matrix) }} env: GOPATH: ${{ github.workspace }}\go GOBIN: ${{ github.workspace }}\go\bin @@ -343,7 +378,8 @@ jobs: with: go-version: ${{ env.GO_VERSION }} - - name: Test API + name: Test integration + if: matrix.test == './...' run: | .\hack\make.ps1 -TestIntegration env: @@ -351,7 +387,8 @@ jobs: GO111MODULE: "off" TEST_CLIENT_BINARY: ${{ env.BIN_OUT }}\docker - - name: Test CLI + name: Test integration-cli + if: matrix.test != './...' run: | & gotestsum ` --format=standard-verbose ` @@ -362,6 +399,7 @@ jobs: "-coverprofile" "./bundles/coverage-report-int-cli-tests.txt" ` "-covermode" "atomic" ` "-tags" "autogen" ` + "-test.run" "(${{ matrix.test }})/" ` "-test.timeout" "200m" env: DOCKER_HOST: npipe:////./pipe/docker_engine diff --git a/Dockerfile.e2e b/Dockerfile.e2e index 8db21210bb..898196d547 100644 --- a/Dockerfile.e2e +++ b/Dockerfile.e2e @@ -31,7 +31,7 @@ COPY hack/dockerfile/install/install.sh ./install.sh COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./ RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME -# Build DockerSuite.TestBuild* dependency +# TestDockerCLIBuildSuite dependency FROM base AS contrib COPY contrib/syscall-test /build/syscall-test COPY contrib/httpserver/Dockerfile /build/httpserver/Dockerfile diff --git a/docs/contributing/test.md b/docs/contributing/test.md index 3bbec68071..099b92c7d8 100644 --- a/docs/contributing/test.md +++ b/docs/contributing/test.md @@ -171,16 +171,16 @@ $ TESTDIRS='github.com/docker/docker/opts' TESTFLAGS='-test.run ^TestValidateIPA We use [gocheck](https://labix.org/gocheck) for our integration-cli tests. You can use the `TESTFLAGS` environment variable to run a single test. The flag's value is passed as arguments to the `go test` command. For example, from -your local host you can run the `TestBuild` test with this command: +your local host you can run the `TestDockerCLIBuildSuite` test with this command: ```bash -$ TESTFLAGS='-test.run TestDockerSuite/TestBuild*' make test-integration +$ TESTFLAGS='-test.run TestDockerCLIBuildSuite' make test-integration ``` To run the same test inside your Docker development container, you do this: ```bash -# TESTFLAGS='-test.run TestDockerSuite/TestBuild*' hack/make.sh binary test-integration +# TESTFLAGS='-test.run TestDockerCLIBuildSuite' hack/make.sh binary test-integration ``` ## Test the Windows binary against a Linux daemon diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers index adc9c0c0a7..0641598298 100644 --- a/hack/make/.integration-test-helpers +++ b/hack/make/.integration-test-helpers @@ -3,7 +3,7 @@ # For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want # to run certain tests on your local host, you should run with command: # -# TESTFLAGS='-test.run TestDockerSuite/TestBuild*' ./hack/make.sh binary test-integration +# TESTFLAGS='-test.run TestDockerCLIBuildSuite' ./hack/make.sh binary test-integration # if [ -z "${MAKEDIR}" ]; then