From 1887d85e2157d909c53774c7cdfbc745c1298629 Mon Sep 17 00:00:00 2001
From: CrazyMax <crazy-max@users.noreply.github.com>
Date: Thu, 16 Jun 2022 22:47:36 +0200
Subject: [PATCH] ci(integration-cli): dynamically split tests in matrix

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
---
 .github/workflows/windows.yml       | 42 +++++++++++++++++++++++++++--
 Dockerfile.e2e                      |  2 +-
 docs/contributing/test.md           |  6 ++---
 hack/make/.integration-test-helpers |  2 +-
 4 files changed, 45 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
index f6592b5277..8fc1a324c7 100644
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@@ -16,6 +16,7 @@ on:
 
 env:
   GO_VERSION: 1.18.3
+  GOTESTLIST_VERSION: v0.2.0
   WINDOWS_BASE_IMAGE: mcr.microsoft.com/windows/servercore
   WINDOWS_BASE_TAG_2019: ltsc2019
   WINDOWS_BASE_TAG_2022: ltsc2022
@@ -188,10 +189,43 @@ jobs:
           name: ${{ matrix.os }}-unit-reports
           path: ${{ env.GOPATH }}\src\github.com\docker\docker\bundles\*
 
+  integration-test-prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.tests.outputs.matrix }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      -
+        name: Install gotestlist
+        run:
+          go install github.com/crazy-max/gotestlist/cmd/gotestlist@${{ env.GOTESTLIST_VERSION }}
+      -
+        name: Create matrix
+        id: tests
+        working-directory: ./integration-cli
+        run: |
+          # Distribute integration-cli tests for the matrix in integration-test job.
+          # Also prepend ./... to the matrix. This is a special case to run "Test integration" step exclusively.
+          matrix="$(gotestlist -d 4 ./...)"
+          matrix="$(echo "$matrix" | jq -c '. |= ["./..."] + .')"
+          echo "::set-output name=matrix::$matrix"
+      -
+        name: Show matrix
+        run: |
+          echo ${{ steps.tests.outputs.matrix }}
+
   integration-test:
     runs-on: ${{ matrix.os }}
     needs:
       - build
+      - integration-test-prepare
     strategy:
       fail-fast: false
       matrix:
@@ -201,6 +235,7 @@ jobs:
         runtime:
           - builtin
           - containerd
+        test: ${{ fromJson(needs.integration-test-prepare.outputs.matrix) }}
     env:
       GOPATH: ${{ github.workspace }}\go
       GOBIN: ${{ github.workspace }}\go\bin
@@ -343,7 +378,8 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
       -
-        name: Test API
+        name: Test integration
+        if: matrix.test == './...'
         run: |
           .\hack\make.ps1 -TestIntegration
         env:
@@ -351,7 +387,8 @@ jobs:
           GO111MODULE: "off"
           TEST_CLIENT_BINARY: ${{ env.BIN_OUT }}\docker
       -
-        name: Test CLI
+        name: Test integration-cli
+        if: matrix.test != './...'
         run: |
           & gotestsum `
             --format=standard-verbose `
@@ -362,6 +399,7 @@ jobs:
               "-coverprofile" "./bundles/coverage-report-int-cli-tests.txt" `
               "-covermode" "atomic" `
               "-tags" "autogen" `
+              "-test.run" "(${{ matrix.test }})/" `
               "-test.timeout" "200m"
         env:
           DOCKER_HOST: npipe:////./pipe/docker_engine
diff --git a/Dockerfile.e2e b/Dockerfile.e2e
index 8db21210bb..898196d547 100644
--- a/Dockerfile.e2e
+++ b/Dockerfile.e2e
@@ -31,7 +31,7 @@ COPY hack/dockerfile/install/install.sh ./install.sh
 COPY hack/dockerfile/install/$INSTALL_BINARY_NAME.installer ./
 RUN PREFIX=/build ./install.sh $INSTALL_BINARY_NAME
 
-# Build DockerSuite.TestBuild* dependency
+# TestDockerCLIBuildSuite dependency
 FROM base AS contrib
 COPY contrib/syscall-test           /build/syscall-test
 COPY contrib/httpserver/Dockerfile  /build/httpserver/Dockerfile
diff --git a/docs/contributing/test.md b/docs/contributing/test.md
index 3bbec68071..099b92c7d8 100644
--- a/docs/contributing/test.md
+++ b/docs/contributing/test.md
@@ -171,16 +171,16 @@ $ TESTDIRS='github.com/docker/docker/opts' TESTFLAGS='-test.run ^TestValidateIPA
 We use [gocheck](https://labix.org/gocheck) for our integration-cli tests.
 You can use the `TESTFLAGS` environment variable to run a single test. The
 flag's value is passed as arguments to the `go test` command. For example, from
-your local host you can run the `TestBuild` test with this command:
+your local host you can run the `TestDockerCLIBuildSuite` test with this command:
 
 ```bash
-$ TESTFLAGS='-test.run TestDockerSuite/TestBuild*' make test-integration
+$ TESTFLAGS='-test.run TestDockerCLIBuildSuite' make test-integration
 ```
 
 To run the same test inside your Docker development container, you do this:
 
 ```bash
-# TESTFLAGS='-test.run TestDockerSuite/TestBuild*' hack/make.sh binary test-integration
+# TESTFLAGS='-test.run TestDockerCLIBuildSuite' hack/make.sh binary test-integration
 ```
 
 ## Test the Windows binary against a Linux daemon
diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers
index adc9c0c0a7..0641598298 100644
--- a/hack/make/.integration-test-helpers
+++ b/hack/make/.integration-test-helpers
@@ -3,7 +3,7 @@
 # For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want
 # to run certain tests on your local host, you should run with command:
 #
-#     TESTFLAGS='-test.run TestDockerSuite/TestBuild*' ./hack/make.sh binary test-integration
+#     TESTFLAGS='-test.run TestDockerCLIBuildSuite' ./hack/make.sh binary test-integration
 #
 
 if [ -z "${MAKEDIR}" ]; then