2017-02-13 14:01:54 -05:00
|
|
|
#!/usr/bin/env bash
|
2017-07-06 17:20:41 -04:00
|
|
|
#
|
|
|
|
# For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want
|
|
|
|
# to run certain tests on your local host, you should run with command:
|
|
|
|
#
|
|
|
|
# TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration
|
|
|
|
#
|
2019-07-29 18:00:55 -04:00
|
|
|
|
|
|
|
if [[ "${TESTFLAGS}" = *-check.f* ]]; then
|
|
|
|
echo Skipping integration tests since TESTFLAGS includes integration-cli only flags
|
|
|
|
TEST_SKIP_INTEGRATION=1
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [[ "${TESTFLAGS}" = *-test.run* ]]; then
|
|
|
|
echo Skipping integration-cli tests since TESTFLAGS includes integration only flags
|
|
|
|
TEST_SKIP_INTEGRATION_CLI=1
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
2019-01-09 20:50:47 -05:00
|
|
|
if [ -z ${MAKEDIR} ]; then
|
2017-11-01 06:12:27 -04:00
|
|
|
export MAKEDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
|
|
|
fi
|
|
|
|
source "$MAKEDIR/.go-autogen"
|
2016-07-18 17:26:35 -04:00
|
|
|
|
2017-08-30 12:01:01 -04:00
|
|
|
# Set defaults
|
2017-07-06 17:20:41 -04:00
|
|
|
: ${TEST_REPEAT:=1}
|
2017-08-30 12:01:01 -04:00
|
|
|
: ${TESTFLAGS:=}
|
|
|
|
: ${TESTDEBUG:=}
|
2017-07-06 17:20:41 -04:00
|
|
|
|
2019-08-02 16:32:36 -04:00
|
|
|
setup_integration_test_filter() {
|
|
|
|
if [ -z "${TEST_FILTER}" ]; then
|
|
|
|
return
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ -z "${TEST_SKIP_INTEGRATION}" ]; then
|
|
|
|
: ${TEST_INTEGRATION_DIR:=$(grep -rl "func\ .*${TEST_FILTER}.*\(t\ \*testing\.T\)" ./integration | grep '_test\.go' | xargs -I file dirname file | uniq)}
|
|
|
|
if [ -z "${TEST_INTEGRATION_DIR}" ]; then
|
|
|
|
echo Skipping integration tests since the supplied filter \"${TEST_FILTER}\" omits all integration tests
|
|
|
|
TEST_SKIP_INTEGRATION=1
|
|
|
|
else
|
|
|
|
TESTFLAGS_INTEGRATION+="-test.run ${TEST_FILTER}"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
|
|
|
|
# ease up on the filtering here since CLI suites are namespaced by an object
|
|
|
|
if grep -r "${TEST_FILTER}.*\(c\ \*check\.C\)" ./integration-cli | grep -q '_test\.go$'; then
|
|
|
|
TEST_SKIP_INTEGRATION_CLI=1
|
|
|
|
echo Skipping integration-cli tests since the supplied filter \"${TEST_FILTER}\" omits all integration-cli tests
|
|
|
|
else
|
|
|
|
TESTFLAGS_INTEGRATION_CLI+="-check.f ${TEST_FILTER}"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
setup_integration_test_filter
|
|
|
|
integration_api_dirs=${TEST_INTEGRATION_DIR:-$(go list -test -f '{{- if ne .ForTest "" -}}{{- .Dir -}}{{- end -}}' ./integration/...)}
|
2017-07-06 17:20:41 -04:00
|
|
|
|
|
|
|
run_test_integration() {
|
2019-07-17 01:05:35 -04:00
|
|
|
set_platform_timeout
|
2019-07-29 18:00:55 -04:00
|
|
|
if [ -z "${TEST_SKIP_INTEGRATION}" ]; then
|
2019-07-16 16:18:32 -04:00
|
|
|
run_test_integration_suites
|
|
|
|
fi
|
2019-07-29 18:00:55 -04:00
|
|
|
if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
|
2019-07-16 16:18:32 -04:00
|
|
|
run_test_integration_legacy_suites
|
|
|
|
fi
|
2017-08-30 12:01:01 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
run_test_integration_suites() {
|
2019-07-29 18:00:55 -04:00
|
|
|
local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS ${TESTFLAGS_INTEGRATION}"
|
2019-01-09 20:50:47 -05:00
|
|
|
for dir in ${integration_api_dirs}; do
|
2017-08-30 12:01:01 -04:00
|
|
|
if ! (
|
2019-01-09 20:50:47 -05:00
|
|
|
cd "$dir"
|
2019-07-29 18:00:55 -04:00
|
|
|
echo "Running $PWD flags=${flags}"
|
2019-01-09 20:50:47 -05:00
|
|
|
test_env ./test.main ${flags}
|
2017-08-30 12:01:01 -04:00
|
|
|
); then exit 1; fi
|
2017-07-06 17:20:41 -04:00
|
|
|
done
|
2017-08-30 12:01:01 -04:00
|
|
|
}
|
2016-07-18 17:26:35 -04:00
|
|
|
|
2017-08-30 12:01:01 -04:00
|
|
|
run_test_integration_legacy_suites() {
|
2017-06-16 20:48:05 -04:00
|
|
|
(
|
2019-07-29 18:00:55 -04:00
|
|
|
flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS ${TESTFLAGS_INTEGRATION_CLI}"
|
2017-07-12 19:13:37 -04:00
|
|
|
cd integration-cli
|
2019-07-29 18:00:55 -04:00
|
|
|
echo "Running $PWD flags=${flags}"
|
2017-07-06 17:20:41 -04:00
|
|
|
test_env ./test.main $flags
|
2017-06-16 20:48:05 -04:00
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
build_test_suite_binaries() {
|
2019-01-09 20:23:38 -05:00
|
|
|
if [ ${DOCKER_INTEGRATION_TESTS_VERIFIED-} ]; then
|
2017-08-12 02:19:12 -04:00
|
|
|
echo "Skipping building test binaries; as DOCKER_INTEGRATION_TESTS_VERIFIED is set"
|
|
|
|
return
|
|
|
|
fi
|
2019-07-29 18:00:55 -04:00
|
|
|
if [ -z "${TEST_SKIP_INTEGRATION_CLI}" ]; then
|
|
|
|
build_test_suite_binary ./integration-cli "test.main"
|
|
|
|
fi
|
|
|
|
if [ -z "${TEST_SKIP_INTEGRATION}" ]; then
|
|
|
|
for dir in ${integration_api_dirs}; do
|
|
|
|
build_test_suite_binary "$dir" "test.main"
|
|
|
|
done
|
|
|
|
fi
|
2017-06-16 20:48:05 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
# Build a binary for a test suite package
|
|
|
|
build_test_suite_binary() {
|
|
|
|
local dir="$1"
|
|
|
|
local out="$2"
|
|
|
|
echo Building test suite binary "$dir/$out"
|
2017-07-06 17:20:41 -04:00
|
|
|
go test -c -o "$dir/$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" "$dir"
|
2017-05-23 16:32:34 -04:00
|
|
|
}
|
|
|
|
|
2017-07-06 17:20:41 -04:00
|
|
|
cleanup_test_suite_binaries() {
|
2019-01-09 20:23:38 -05:00
|
|
|
[ -n "$TESTDEBUG" ] && return
|
2017-07-06 17:20:41 -04:00
|
|
|
echo "Removing test suite binaries"
|
|
|
|
find integration* -name test.main | xargs -r rm
|
2016-07-18 17:26:35 -04:00
|
|
|
}
|
|
|
|
|
2017-07-06 17:20:41 -04:00
|
|
|
repeat() {
|
2019-01-09 20:50:47 -05:00
|
|
|
for i in $(seq 1 ${TEST_REPEAT}); do
|
2017-07-06 17:20:41 -04:00
|
|
|
echo "Running integration-test (iteration $i)"
|
|
|
|
$@
|
|
|
|
done
|
|
|
|
}
|
|
|
|
|
|
|
|
# use "env -i" to tightly control the environment variables that bleed into the tests
|
2016-07-18 17:26:35 -04:00
|
|
|
test_env() {
|
2016-09-02 08:58:13 -04:00
|
|
|
(
|
2017-07-06 17:20:41 -04:00
|
|
|
set -e
|
2019-01-09 20:23:38 -05:00
|
|
|
[ -n "$TESTDEBUG" ] && set -x
|
2016-09-02 08:58:13 -04:00
|
|
|
env -i \
|
2017-07-06 17:20:41 -04:00
|
|
|
DEST="$ABS_DEST" \
|
2017-04-17 19:18:46 -04:00
|
|
|
DOCKER_API_VERSION="$DOCKER_API_VERSION" \
|
2018-05-15 20:09:20 -04:00
|
|
|
DOCKER_BUILDKIT="$DOCKER_BUILDKIT" \
|
[EXPERIMENTAL] Integration Test on Swarm
This commit adds contrib/integration-cli-on-swarm/integration-cli-on-swarm.sh,
which enables IT to be running in parallel, using Swarm-mode and Funker.
Please refer to contrib/integration-cli-on-swarm/README.md
The test takes almost 5 to 6 minutes, with 10 n1-standard-4 GCE instances.
$ ./contrib/integration-cli-on-swarm/integration-cli-on-swarm.sh --push-worker-image example.gcr.io/foo/bar --replicas 30 --shuffle
2016/12/29 08:32:15 Loaded 1618 tests (30 chunks)
2016/12/29 08:32:15 Executing 30 chunks in parallel, against "integration-cli-worker"
2016/12/29 08:32:15 Executing chunk 0 (contains 54 test filters)
..
2016/12/29 08:34:34 Finished chunk 28 [1/30] with 54 test filters in 2m19.098068003s, code=0.
2016/12/29 08:34:38 Finished chunk 12 [2/30] with 54 test filters in 2m23.088569511s, code=0.
2016/12/29 08:34:48 Finished chunk 10 [3/30] with 54 test filters in 2m33.880679079s, code=0.
2016/12/29 08:34:54 Finished chunk 20 [4/30] with 54 test filters in 2m39.973747028s, code=0.
2016/12/29 08:35:11 Finished chunk 18 [5/30] with 54 test filters in 2m56.28384361s, code=0.
2016/12/29 08:35:11 Finished chunk 29 [6/30] with 52 test filters in 2m56.54047088s, code=0.
2016/12/29 08:35:15 Finished chunk 1 [7/30] with 54 test filters in 3m0.285044426s, code=0.
2016/12/29 08:35:22 Finished chunk 6 [8/30] with 54 test filters in 3m7.211775338s, code=0.
2016/12/29 08:35:24 Finished chunk 25 [9/30] with 54 test filters in 3m9.938413009s, code=0.
2016/12/29 08:35:30 Finished chunk 27 [10/30] with 54 test filters in 3m15.219834368s, code=0.
2016/12/29 08:35:36 Finished chunk 9 [11/30] with 54 test filters in 3m21.615434162s, code=0.
2016/12/29 08:35:41 Finished chunk 13 [12/30] with 54 test filters in 3m26.576907401s, code=0.
2016/12/29 08:35:45 Finished chunk 17 [13/30] with 54 test filters in 3m30.290752537s, code=0.
2016/12/29 08:35:53 Finished chunk 2 [14/30] with 54 test filters in 3m38.148423321s, code=0.
2016/12/29 08:35:55 Finished chunk 24 [15/30] with 54 test filters in 3m40.09669137s, code=0.
2016/12/29 08:35:57 Finished chunk 8 [16/30] with 54 test filters in 3m42.299945108s, code=0.
2016/12/29 08:35:57 Finished chunk 22 [17/30] with 54 test filters in 3m42.946558809s, code=0.
2016/12/29 08:35:59 Finished chunk 23 [18/30] with 54 test filters in 3m44.232557165s, code=0.
2016/12/29 08:36:02 Finished chunk 3 [19/30] with 54 test filters in 3m47.112051358s, code=0.
2016/12/29 08:36:11 Finished chunk 15 [20/30] with 54 test filters in 3m56.340656645s, code=0.
2016/12/29 08:36:11 Finished chunk 11 [21/30] with 54 test filters in 3m56.882401231s, code=0.
2016/12/29 08:36:22 Finished chunk 19 [22/30] with 54 test filters in 4m7.551093516s, code=0.
2016/12/29 08:36:23 Finished chunk 21 [23/30] with 54 test filters in 4m8.221093446s, code=0.
2016/12/29 08:36:25 Finished chunk 16 [24/30] with 54 test filters in 4m10.450451705s, code=0.
2016/12/29 08:36:27 Finished chunk 5 [25/30] with 54 test filters in 4m12.162272692s, code=0.
2016/12/29 08:36:28 Finished chunk 14 [26/30] with 54 test filters in 4m13.977801031s, code=0.
2016/12/29 08:36:29 Finished chunk 0 [27/30] with 54 test filters in 4m14.34086812s, code=0.
2016/12/29 08:36:49 Finished chunk 26 [28/30] with 54 test filters in 4m34.437085539s, code=0.
2016/12/29 08:37:14 Finished chunk 7 [29/30] with 54 test filters in 4m59.22902721s, code=0.
2016/12/29 08:37:20 Finished chunk 4 [30/30] with 54 test filters in 5m5.103469214s, code=0.
2016/12/29 08:37:20 Executed 30 chunks in 5m5.104379119s. PASS: 30, FAIL: 0.
Signed-off-by: Akihiro Suda <suda.akihiro@lab.ntt.co.jp>
2016-12-07 02:16:48 -05:00
|
|
|
DOCKER_INTEGRATION_DAEMON_DEST="$DOCKER_INTEGRATION_DAEMON_DEST" \
|
2016-09-02 08:58:13 -04:00
|
|
|
DOCKER_TLS_VERIFY="$DOCKER_TEST_TLS_VERIFY" \
|
|
|
|
DOCKER_CERT_PATH="$DOCKER_TEST_CERT_PATH" \
|
|
|
|
DOCKER_ENGINE_GOARCH="$DOCKER_ENGINE_GOARCH" \
|
|
|
|
DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" \
|
|
|
|
DOCKER_USERLANDPROXY="$DOCKER_USERLANDPROXY" \
|
|
|
|
DOCKER_HOST="$DOCKER_HOST" \
|
|
|
|
DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \
|
|
|
|
DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \
|
2016-08-31 15:31:06 -04:00
|
|
|
DOCKERFILE="$DOCKERFILE" \
|
2016-09-02 08:58:13 -04:00
|
|
|
GOPATH="$GOPATH" \
|
|
|
|
GOTRACEBACK=all \
|
|
|
|
HOME="$ABS_DEST/fake-HOME" \
|
|
|
|
PATH="$PATH" \
|
|
|
|
TEMP="$TEMP" \
|
2017-04-17 19:18:46 -04:00
|
|
|
TEST_CLIENT_BINARY="$TEST_CLIENT_BINARY" \
|
2016-09-02 08:58:13 -04:00
|
|
|
"$@"
|
|
|
|
)
|
2016-07-18 17:26:35 -04:00
|
|
|
}
|
2017-07-06 17:20:41 -04:00
|
|
|
|
|
|
|
error_on_leaked_containerd_shims() {
|
2019-07-17 01:05:35 -04:00
|
|
|
if [ "$(go env GOOS)" = 'windows' ]; then
|
2017-07-06 17:20:41 -04:00
|
|
|
return
|
|
|
|
fi
|
|
|
|
|
|
|
|
leftovers=$(ps -ax -o pid,cmd |
|
2018-09-21 18:58:34 -04:00
|
|
|
awk '$2 == "containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration/ { print $1 }')
|
2019-01-09 20:23:38 -05:00
|
|
|
if [ -n "$leftovers" ]; then
|
2017-07-06 17:20:41 -04:00
|
|
|
ps aux
|
2019-01-09 20:50:47 -05:00
|
|
|
kill -9 ${leftovers} 2> /dev/null
|
2017-07-06 17:20:41 -04:00
|
|
|
echo "!!!! WARNING you have left over shim(s), Cleanup your test !!!!"
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
}
|
2019-07-17 01:05:35 -04:00
|
|
|
|
|
|
|
set_platform_timeout() {
|
|
|
|
# Test timeout.
|
|
|
|
if [ "${DOCKER_ENGINE_GOARCH}" = "arm64" ] || [ "${DOCKER_ENGINE_GOARCH}" = "arm" ]; then
|
|
|
|
: ${TIMEOUT:=10m}
|
|
|
|
elif [ "${DOCKER_ENGINE_GOARCH}" = "windows" ]; then
|
|
|
|
: ${TIMEOUT:=8m}
|
|
|
|
else
|
|
|
|
: ${TIMEOUT:=5m}
|
|
|
|
fi
|
|
|
|
|
|
|
|
if [ "${TEST_REPEAT}" -gt 1 ]; then
|
|
|
|
# TIMEOUT needs to take TEST_REPEAT into account, or a premature time out may happen.
|
|
|
|
# The following ugliness will:
|
|
|
|
# - remove last character (usually 'm' from '10m')
|
|
|
|
# - multiply by testcount
|
|
|
|
# - add last character back
|
|
|
|
TIMEOUT=$((${TIMEOUT::-1} * ${TEST_REPEAT}))${TIMEOUT:$((${#TIMEOUT}-1)):1}
|
|
|
|
fi
|
|
|
|
}
|