diff --git a/Makefile b/Makefile index 4c70301866..8a68874f1d 100644 --- a/Makefile +++ b/Makefile @@ -54,9 +54,13 @@ DOCKER_ENVS := \ -e DOCKER_USERLANDPROXY \ -e DOCKERD_ARGS \ -e TEST_INTEGRATION_DIR \ + -e TEST_SKIP_INTEGRATION \ + -e TEST_SKIP_INTEGRATION_CLI \ -e TESTDEBUG \ -e TESTDIRS \ -e TESTFLAGS \ + -e TESTFLAGS_INTEGRATION \ + -e TESTFLAGS_INTEGRATION_CLI \ -e TIMEOUT \ -e VALIDATE_REPO \ -e VALIDATE_BRANCH \ @@ -181,8 +185,13 @@ test-docker-py: build ## run the docker-py tests test-integration-cli: test-integration ## (DEPRECATED) use test-integration +ifneq ($(and $(TEST_SKIP_INTEGRATION),$(TEST_SKIP_INTEGRATION_CLI)),) +test-integration: + @echo Both integrations suites skipped per environment variables +else test-integration: build ## run the integration tests $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration +endif test-integration-flaky: build ## run the stress test for all new integration tests $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration-flaky diff --git a/hack/make.ps1 b/hack/make.ps1 index bde0679c60..63fdbb32ad 100644 --- a/hack/make.ps1 +++ b/hack/make.ps1 @@ -327,19 +327,16 @@ Function Run-UnitTests() { # Run the integration tests Function Run-IntegrationTests() { $env:DOCKER_INTEGRATION_DAEMON_DEST = $root + "\bundles\tmp" - $dirs = Get-ChildItem -Path integration -Directory -Recurse + $dirs = go list -test -f '{{- if ne .ForTest `"`" -}}{{- .Dir -}}{{- end -}}' .\integration\... $integration_api_dirs = @() ForEach($dir in $dirs) { - $RelativePath = "." + $dir.FullName -replace "$($PWD.Path -replace "\\","\\")","" - If ($RelativePath -notmatch '(^.\\integration($|\\internal)|\\testdata)') { - $integration_api_dirs += $dir - Write-Host "Building test suite binary $RelativePath" - go test -c -o "$RelativePath\test.exe" $RelativePath - } + $integration_api_dirs += $dir + Write-Host "Building test suite binary $dir" + go test -c -o "$dir\test.exe" $dir } ForEach($dir in $integration_api_dirs) { - Set-Location $dir.FullName + Set-Location $dir Write-Host "Running $($PWD.Path)" $pinfo = New-Object System.Diagnostics.ProcessStartInfo $pinfo.FileName = "$($PWD.Path)\test.exe" diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers index 85394483cb..abe69474ae 100644 --- a/hack/make/.integration-test-helpers +++ b/hack/make/.integration-test-helpers @@ -5,6 +5,18 @@ # # TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration # + +if [[ "${TESTFLAGS}" = *-check.f* ]]; then + echo Skipping integration tests since TESTFLAGS includes integration-cli only flags + TEST_SKIP_INTEGRATION=1 +fi + +if [[ "${TESTFLAGS}" = *-test.run* ]]; then + echo Skipping integration-cli tests since TESTFLAGS includes integration only flags + TEST_SKIP_INTEGRATION_CLI=1 +fi + + if [ -z ${MAKEDIR} ]; then export MAKEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" fi @@ -15,26 +27,24 @@ source "$MAKEDIR/.go-autogen" : ${TESTFLAGS:=} : ${TESTDEBUG:=} -integration_api_dirs=${TEST_INTEGRATION_DIR:-"$( - find ./integration -type d | - grep -vE '(^./integration($|/internal)|/testdata)')"} +integration_api_dirs=${TEST_INTEGRATION_DIR:-"$(go list -test -f '{{- if ne .ForTest "" -}}{{- .Dir -}}{{- end -}}' ./integration/...)"} run_test_integration() { set_platform_timeout - if [[ "$TESTFLAGS" != *-check.f* ]]; then + if [ -z "${TEST_SKIP_INTEGRATION}" ]; then run_test_integration_suites fi - if [[ "$TESTFLAGS" != *-test.run* ]]; then + if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then run_test_integration_legacy_suites fi } run_test_integration_suites() { - local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS" + local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS ${TESTFLAGS_INTEGRATION}" for dir in ${integration_api_dirs}; do if ! ( cd "$dir" - echo "Running $PWD" + echo "Running $PWD flags=${flags}" test_env ./test.main ${flags} ); then exit 1; fi done @@ -42,9 +52,9 @@ run_test_integration_suites() { run_test_integration_legacy_suites() { ( - flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS" + flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS ${TESTFLAGS_INTEGRATION_CLI}" cd integration-cli - echo "Running $PWD" + echo "Running $PWD flags=${flags}" test_env ./test.main $flags ) } @@ -54,10 +64,14 @@ build_test_suite_binaries() { echo "Skipping building test binaries; as DOCKER_INTEGRATION_TESTS_VERIFIED is set" return fi - build_test_suite_binary ./integration-cli "test.main" - for dir in ${integration_api_dirs}; do - build_test_suite_binary "$dir" "test.main" - done + if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then + build_test_suite_binary ./integration-cli "test.main" + fi + if [ -z "${TEST_SKIP_INTEGRATION}" ]; then + for dir in ${integration_api_dirs}; do + build_test_suite_binary "$dir" "test.main" + done + fi } # Build a binary for a test suite package diff --git a/hack/make/test-integration b/hack/make/test-integration index 6908935d1f..6d68dd5dff 100755 --- a/hack/make/test-integration +++ b/hack/make/test-integration @@ -3,6 +3,12 @@ set -e -o pipefail source hack/make/.integration-test-helpers +if [ ! -z "${TEST_SKIP_INTEGRATION}" ] && [ ! -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then + echo integration and integraiton-cli skipped according to env vars + exit 0 +fi + + ( build_test_suite_binaries bundle .integration-daemon-start diff --git a/integration/plugin/logging/cmd/close_on_start/main_test.go b/integration/plugin/logging/cmd/close_on_start/main_test.go deleted file mode 100644 index 06ab7d0f9a..0000000000 --- a/integration/plugin/logging/cmd/close_on_start/main_test.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/integration/plugin/logging/cmd/cmd_test.go b/integration/plugin/logging/cmd/cmd_test.go deleted file mode 100644 index 1d619dd05e..0000000000 --- a/integration/plugin/logging/cmd/cmd_test.go +++ /dev/null @@ -1 +0,0 @@ -package cmd diff --git a/integration/plugin/logging/cmd/dummy/main_test.go b/integration/plugin/logging/cmd/dummy/main_test.go deleted file mode 100644 index 06ab7d0f9a..0000000000 --- a/integration/plugin/logging/cmd/dummy/main_test.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/integration/plugin/volumes/cmd/cmd_test.go b/integration/plugin/volumes/cmd/cmd_test.go deleted file mode 100644 index 1d619dd05e..0000000000 --- a/integration/plugin/volumes/cmd/cmd_test.go +++ /dev/null @@ -1 +0,0 @@ -package cmd diff --git a/integration/plugin/volumes/cmd/dummy/main_test.go b/integration/plugin/volumes/cmd/dummy/main_test.go deleted file mode 100644 index 06ab7d0f9a..0000000000 --- a/integration/plugin/volumes/cmd/dummy/main_test.go +++ /dev/null @@ -1 +0,0 @@ -package main