From da9289fb54df2c8486a5b0e3bb25fd1a209f3b2c Mon Sep 17 00:00:00 2001
From: Brian Goff <cpuguy83@gmail.com>
Date: Mon, 29 Jul 2019 15:00:55 -0700
Subject: [PATCH] Improvements to the test runners

1. Use `go list` to get list of integration dirs to build. This means we
   do not need to have a valid `.go` in every subdirectory and also
   filters out other dirs like "bundles" which may have been created.
2. Add option to specify custom flags for integration and
   integration-cli. This is needed so both suites can be run AND set
   custom flags... since the cli suite does not support standard go
   flags.
3. Add options to skip an entire integration suite.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>
(cherry picked from commit abece9b56221bd682f723d5049f7cc047cd522ba)
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
---
 Makefile                                      |  9 +++++
 hack/make.ps1                                 | 13 +++---
 hack/make/.integration-test-helpers           | 40 +++++++++++++------
 hack/make/test-integration                    |  6 +++
 .../logging/cmd/close_on_start/main_test.go   |  1 -
 integration/plugin/logging/cmd/cmd_test.go    |  1 -
 .../plugin/logging/cmd/dummy/main_test.go     |  1 -
 integration/plugin/volumes/cmd/cmd_test.go    |  1 -
 .../plugin/volumes/cmd/dummy/main_test.go     |  1 -
 9 files changed, 47 insertions(+), 26 deletions(-)
 delete mode 100644 integration/plugin/logging/cmd/close_on_start/main_test.go
 delete mode 100644 integration/plugin/logging/cmd/cmd_test.go
 delete mode 100644 integration/plugin/logging/cmd/dummy/main_test.go
 delete mode 100644 integration/plugin/volumes/cmd/cmd_test.go
 delete mode 100644 integration/plugin/volumes/cmd/dummy/main_test.go

diff --git a/Makefile b/Makefile
index 4c70301866..8a68874f1d 100644
--- a/Makefile
+++ b/Makefile
@@ -54,9 +54,13 @@ DOCKER_ENVS := \
 	-e DOCKER_USERLANDPROXY \
 	-e DOCKERD_ARGS \
 	-e TEST_INTEGRATION_DIR \
+	-e TEST_SKIP_INTEGRATION \
+	-e TEST_SKIP_INTEGRATION_CLI \
 	-e TESTDEBUG \
 	-e TESTDIRS \
 	-e TESTFLAGS \
+	-e TESTFLAGS_INTEGRATION \
+	-e TESTFLAGS_INTEGRATION_CLI \
 	-e TIMEOUT \
 	-e VALIDATE_REPO \
 	-e VALIDATE_BRANCH \
@@ -181,8 +185,13 @@ test-docker-py: build ## run the docker-py tests
 
 test-integration-cli: test-integration ## (DEPRECATED) use test-integration
 
+ifneq ($(and $(TEST_SKIP_INTEGRATION),$(TEST_SKIP_INTEGRATION_CLI)),)
+test-integration:
+	@echo Both integrations suites skipped per environment variables
+else
 test-integration: build ## run the integration tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration
+endif
 
 test-integration-flaky: build ## run the stress test for all new integration tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration-flaky
diff --git a/hack/make.ps1 b/hack/make.ps1
index bde0679c60..63fdbb32ad 100644
--- a/hack/make.ps1
+++ b/hack/make.ps1
@@ -327,19 +327,16 @@ Function Run-UnitTests() {
 # Run the integration tests
 Function Run-IntegrationTests() {
     $env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp"
-    $dirs =  Get-ChildItem -Path integration -Directory -Recurse
+    $dirs = go list -test -f '{{- if ne .ForTest `"`" -}}{{- .Dir -}}{{- end -}}' .\integration\...
     $integration_api_dirs = @()
     ForEach($dir in $dirs) {
-        $RelativePath = "." + $dir.FullName -replace "$($PWD.Path -replace "\\","\\")",""
-        If ($RelativePath -notmatch '(^.\\integration($|\\internal)|\\testdata)') {
-            $integration_api_dirs += $dir
-            Write-Host "Building test suite binary $RelativePath"
-            go test -c -o "$RelativePath\test.exe" $RelativePath
-        }
+        $integration_api_dirs += $dir
+        Write-Host "Building test suite binary $dir"
+        go test -c -o "$dir\test.exe" $dir
     }
 
     ForEach($dir in $integration_api_dirs) {
-        Set-Location $dir.FullName
+        Set-Location $dir
         Write-Host "Running $($PWD.Path)"
         $pinfo = New-Object System.Diagnostics.ProcessStartInfo
         $pinfo.FileName = "$($PWD.Path)\test.exe"
diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers
index 85394483cb..abe69474ae 100644
--- a/hack/make/.integration-test-helpers
+++ b/hack/make/.integration-test-helpers
@@ -5,6 +5,18 @@
 #
 #     TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration
 #
+
+if [[ "${TESTFLAGS}" = *-check.f* ]]; then
+	echo Skipping integration tests since TESTFLAGS includes integration-cli only flags
+	TEST_SKIP_INTEGRATION=1
+fi
+
+if [[ "${TESTFLAGS}" = *-test.run* ]]; then
+	echo Skipping integration-cli tests since TESTFLAGS includes integration only flags
+	TEST_SKIP_INTEGRATION_CLI=1
+fi
+
+
 if [ -z ${MAKEDIR} ]; then
 	export MAKEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 fi
@@ -15,26 +27,24 @@ source "$MAKEDIR/.go-autogen"
 : ${TESTFLAGS:=}
 : ${TESTDEBUG:=}
 
-integration_api_dirs=${TEST_INTEGRATION_DIR:-"$(
-	find ./integration -type d |
-	grep -vE '(^./integration($|/internal)|/testdata)')"}
+integration_api_dirs=${TEST_INTEGRATION_DIR:-"$(go list  -test -f '{{- if ne .ForTest "" -}}{{- .Dir -}}{{- end -}}'  ./integration/...)"}
 
 run_test_integration() {
 	set_platform_timeout
-	if [[ "$TESTFLAGS" != *-check.f* ]]; then
+	if [ -z "${TEST_SKIP_INTEGRATION}" ]; then
 		run_test_integration_suites
 	fi
-	if [[ "$TESTFLAGS" != *-test.run* ]]; then
+	if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
 		run_test_integration_legacy_suites
 	fi
 }
 
 run_test_integration_suites() {
-	local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS"
+	local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS ${TESTFLAGS_INTEGRATION}"
 	for dir in ${integration_api_dirs}; do
 		if ! (
 			cd "$dir"
-			echo "Running $PWD"
+			echo "Running $PWD flags=${flags}"
 			test_env ./test.main ${flags}
 		); then exit 1; fi
 	done
@@ -42,9 +52,9 @@ run_test_integration_suites() {
 
 run_test_integration_legacy_suites() {
 	(
-		flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS"
+		flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS ${TESTFLAGS_INTEGRATION_CLI}"
 		cd integration-cli
-		echo "Running $PWD"
+		echo "Running $PWD flags=${flags}"
 		test_env ./test.main $flags
 	)
 }
@@ -54,10 +64,14 @@ build_test_suite_binaries() {
 		echo "Skipping building test binaries; as DOCKER_INTEGRATION_TESTS_VERIFIED is set"
 		return
 	fi
-	build_test_suite_binary ./integration-cli "test.main"
-	for dir in ${integration_api_dirs}; do
-		build_test_suite_binary "$dir" "test.main"
-	done
+	if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
+		build_test_suite_binary ./integration-cli "test.main"
+	fi
+	if [ -z "${TEST_SKIP_INTEGRATION}" ]; then
+		for dir in ${integration_api_dirs}; do
+			build_test_suite_binary "$dir" "test.main"
+		done
+	fi
 }
 
 # Build a binary for a test suite package
diff --git a/hack/make/test-integration b/hack/make/test-integration
index 6908935d1f..6d68dd5dff 100755
--- a/hack/make/test-integration
+++ b/hack/make/test-integration
@@ -3,6 +3,12 @@ set -e -o pipefail
 
 source hack/make/.integration-test-helpers
 
+if [ ! -z "${TEST_SKIP_INTEGRATION}" ] && [ ! -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
+	echo integration and integraiton-cli skipped according to env vars
+	exit 0
+fi
+
+
 (
 	build_test_suite_binaries
 	bundle .integration-daemon-start
diff --git a/integration/plugin/logging/cmd/close_on_start/main_test.go b/integration/plugin/logging/cmd/close_on_start/main_test.go
deleted file mode 100644
index 06ab7d0f9a..0000000000
--- a/integration/plugin/logging/cmd/close_on_start/main_test.go
+++ /dev/null
@@ -1 +0,0 @@
-package main
diff --git a/integration/plugin/logging/cmd/cmd_test.go b/integration/plugin/logging/cmd/cmd_test.go
deleted file mode 100644
index 1d619dd05e..0000000000
--- a/integration/plugin/logging/cmd/cmd_test.go
+++ /dev/null
@@ -1 +0,0 @@
-package cmd
diff --git a/integration/plugin/logging/cmd/dummy/main_test.go b/integration/plugin/logging/cmd/dummy/main_test.go
deleted file mode 100644
index 06ab7d0f9a..0000000000
--- a/integration/plugin/logging/cmd/dummy/main_test.go
+++ /dev/null
@@ -1 +0,0 @@
-package main
diff --git a/integration/plugin/volumes/cmd/cmd_test.go b/integration/plugin/volumes/cmd/cmd_test.go
deleted file mode 100644
index 1d619dd05e..0000000000
--- a/integration/plugin/volumes/cmd/cmd_test.go
+++ /dev/null
@@ -1 +0,0 @@
-package cmd
diff --git a/integration/plugin/volumes/cmd/dummy/main_test.go b/integration/plugin/volumes/cmd/dummy/main_test.go
deleted file mode 100644
index 06ab7d0f9a..0000000000
--- a/integration/plugin/volumes/cmd/dummy/main_test.go
+++ /dev/null
@@ -1 +0,0 @@
-package main