Merge pull request #39911 from tiborvass/gotestsum-integration-cli

hack: have integration-cli use gotestsum codepath

Jenkinsfile

@@ -307,10 +307,10 @@ pipeline {
                                 TEST_INTEGRATION_DEST=1 CONTAINER_NAME=${CONTAINER_NAME}-1 TEST_SKIP_INTEGRATION_CLI=1 run_tests test-integration-flaky &
 
                                 # integration-cli first set
-                                TEST_INTEGRATION_DEST=2 CONTAINER_NAME=${CONTAINER_NAME}-2 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-test.run /(DockerSuite|DockerNetworkSuite|DockerHubPullSuite|DockerRegistrySuite|DockerSchema1RegistrySuite|DockerRegistryAuthTokenSuite|DockerRegistryAuthHtpasswdSuite)/" run_tests &
+                                TEST_INTEGRATION_DEST=2 CONTAINER_NAME=${CONTAINER_NAME}-2 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-test.run Test(DockerSuite|DockerNetworkSuite|DockerHubPullSuite|DockerRegistrySuite|DockerSchema1RegistrySuite|DockerRegistryAuthTokenSuite|DockerRegistryAuthHtpasswdSuite)/" run_tests &
 
                                 # integration-cli second set
-                                TEST_INTEGRATION_DEST=3 CONTAINER_NAME=${CONTAINER_NAME}-3 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-test.run /(DockerSwarmSuite|DockerDaemonSuite|DockerExternalVolumeSuite)/" run_tests &
+                                TEST_INTEGRATION_DEST=3 CONTAINER_NAME=${CONTAINER_NAME}-3 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-test.run Test(DockerSwarmSuite|DockerDaemonSuite|DockerExternalVolumeSuite)/" run_tests &
 
                                 set +x
                                 c=0

docs/contributing/test.md

@@ -174,13 +174,13 @@ flag's value is passed as arguments to the `go test` command. For example, from
 your local host you can run the `TestBuild` test with this command:
 
 ```bash
-$ TESTFLAGS='-test.run /DockerSuite/TestBuild*' make test-integration
+$ TESTFLAGS='-test.run TestDockerSuite/TestBuild*' make test-integration
 ```
 
 To run the same test inside your Docker development container, you do this:
 
 ```bash
-# TESTFLAGS='-test.run /DockerSuite/TestBuild*' hack/make.sh binary test-integration
+# TESTFLAGS='-test.run TestDockerSuite/TestBuild*' hack/make.sh binary test-integration
 ```
 
 ## Test the Windows binary against a Linux daemon
@@ -228,11 +228,11 @@ run a Bash terminal on Windows.
     ```
 
     Should you wish to run a single test such as one with the name
-    'TestExample', you can pass in `TESTFLAGS='-test.run //TestExample'`. For
+    'TestExample', you can pass in `TESTFLAGS='-test.run /TestExample'`. For
     example
 
     ```bash
-    $ TESTFLAGS='-test.run //TestExample' hack/make.sh binary test-integration
+    $ TESTFLAGS='-test.run /TestExample' hack/make.sh binary test-integration
     ```
 
 You can now choose to make changes to the Moby source or the tests. If you

hack/make/.integration-test-helpers

@@ -3,7 +3,7 @@
 # For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want
 # to run certain tests on your local host, you should run with command:
 #
-#     TESTFLAGS='-test.run /DockerSuite/TestBuild*' ./hack/make.sh binary test-integration
+#     TESTFLAGS='-test.run TestDockerSuite/TestBuild*' ./hack/make.sh binary test-integration
 #
 
 if [ -z "${MAKEDIR}" ]; then
@@ -47,16 +47,17 @@ integration_api_dirs="${TEST_INTEGRATION_DIR:-$(go list  -test -f '{{- if ne .Fo
 run_test_integration() {
 	set_platform_timeout
 	if [ -z "${TEST_SKIP_INTEGRATION}" ]; then
-		run_test_integration_suites
+		run_test_integration_suites "${integration_api_dirs}"
 	fi
 	if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
-		run_test_integration_legacy_suites
+		TIMEOUT=360m run_test_integration_suites integration-cli
 	fi
 }
 
 run_test_integration_suites() {
 	local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS"
-	for dir in ${integration_api_dirs}; do
+	local dirs="$1"
+	for dir in ${dirs}; do
 		if ! (
 			cd "$dir"
 			# Create a useful package name based on the tests's $dir. We need to take
@@ -84,16 +85,6 @@ run_test_integration_suites() {
 	done
 }
 
-run_test_integration_legacy_suites() {
-	(
-		flags="-test.v -test.timeout=360m $TESTFLAGS"
-		cd integration-cli
-		echo "Running $PWD flags=${flags}"
-		# shellcheck disable=SC2086
-		test_env ./test.main $flags
-	)
-}
-
 build_test_suite_binaries() {
 	if [ -n "${DOCKER_INTEGRATION_TESTS_VERIFIED}" ]; then
 		echo "Skipping building test binaries; as DOCKER_INTEGRATION_TESTS_VERIFIED is set"

integration-cli/check_test.go

@@ -9,7 +9,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"sync"
 	"syscall"
@@ -45,6 +44,8 @@ var (
 
 	// the docker client binary to use
 	dockerBinary = ""
+
+	testEnvOnce sync.Once
 )
 
 func init() {
@@ -74,25 +75,72 @@ func TestMain(m *testing.M) {
 	os.Exit(m.Run())
 }
 
-func Test(t *testing.T) {
-	cli.SetTestEnvironment(testEnv)
-	fakestorage.SetTestEnvironment(&testEnv.Execution)
-	ienv.ProtectAll(t, &testEnv.Execution)
+func ensureTestEnvSetup(t *testing.T) {
+	testEnvOnce.Do(func() {
+		cli.SetTestEnvironment(testEnv)
+		fakestorage.SetTestEnvironment(&testEnv.Execution)
+		ienv.ProtectAll(t, &testEnv.Execution)
+	})
+}
+
+func TestDockerSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerSuite{})
+}
+
+func TestDockerRegistrySuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerRegistrySuite{ds: &DockerSuite{}})
+}
+
+func TestDockerSchema1RegistrySuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerSchema1RegistrySuite{ds: &DockerSuite{}})
+}
+
+func TestDockerRegistryAuthHtpasswdSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerRegistryAuthHtpasswdSuite{ds: &DockerSuite{}})
+}
+
+func TestDockerRegistryAuthTokenSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerRegistryAuthTokenSuite{ds: &DockerSuite{}})
+}
+
+func TestDockerDaemonSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerDaemonSuite{ds: &DockerSuite{}})
+}
+
+func TestDockerSwarmSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerSwarmSuite{ds: &DockerSuite{}})
+}
+
+func TestDockerPluginSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
 	suite.Run(t, &DockerPluginSuite{ds: &DockerSuite{}})
-	if runtime.GOOS != "windows" {
-		suite.Run(t, &DockerExternalVolumeSuite{ds: &DockerSuite{}})
-		suite.Run(t, &DockerNetworkSuite{ds: &DockerSuite{}})
-		// FIXME. Temporarily turning this off for Windows as GH16039 was breaking
-		// Windows to Linux CI @icecrime
-		suite.Run(t, newDockerHubPullSuite())
-	}
+}
+
+func TestDockerExternalVolumeSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
+	testRequires(t, DaemonIsLinux)
+	suite.Run(t, &DockerExternalVolumeSuite{ds: &DockerSuite{}})
+}
+
+func TestDockerNetworkSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
+	testRequires(t, DaemonIsLinux)
+	suite.Run(t, &DockerExternalVolumeSuite{ds: &DockerSuite{}})
+}
+
+func TestDockerHubPullSuite(t *testing.T) {
+	ensureTestEnvSetup(t)
+	// FIXME. Temporarily turning this off for Windows as GH16039 was breaking
+	// Windows to Linux CI @icecrime
+	testRequires(t, DaemonIsLinux)
+	suite.Run(t, newDockerHubPullSuite())
 }
 
 type DockerSuite struct {

internal/test/suite/suite.go

@@ -30,13 +30,12 @@ func Run(t *testing.T, suite interface{}) {
 	}()
 
 	methodFinder := reflect.TypeOf(suite)
-	suiteName := methodFinder.Elem().Name()
 	for index := 0; index < methodFinder.NumMethod(); index++ {
 		method := methodFinder.Method(index)
 		if !methodFilter(method.Name, method.Type) {
 			continue
 		}
-		t.Run(suiteName+"/"+method.Name, func(t *testing.T) {
+		t.Run(method.Name, func(t *testing.T) {
 			defer failOnPanic(t)
 
 			if !suiteSetupDone {