Verify integration tests before building bundles

Adds a new bundle `verify-integration-tests` which pre-compiles a test
binary for the integration tests.

This makes sure that the integration tests will actually compile before
doing other tasks which take much longer, such as building dockerd and
loading test fixtures.
When it comes time to actually run the tests, the pre-compiled binary
will be used so it doesn't have to compile the tests a 2nd time.

Signed-off-by: Brian Goff <cpuguy83@gmail.com>

Makefile

@@ -125,7 +125,7 @@ test-docker-py: build ## run the docker-py tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-docker-py
 
 test-integration-cli: build ## run the integration tests
-	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration-cli
+	$(DOCKER_RUN_DOCKER) hack/make.sh build-integration-test-binary dynbinary test-integration-cli
 
 test-unit: build ## run the unit tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh test-unit

hack/make.sh

@@ -285,7 +285,6 @@ install_binary() {
 	fi
 }
 
-
 main() {
 	# We want this to fail if the bundles already exist and cannot be removed.
 	# This is to avoid mixing bundles from different versions of the code.

hack/make/.integration-test-helpers

@@ -4,7 +4,7 @@
 
 bundle_test_integration_cli() {
 	TESTFLAGS="$TESTFLAGS -check.v -check.timeout=${TIMEOUT} -test.timeout=360m"
-	go_test_dir ./integration-cli
+	go_test_dir integration-cli $DOCKER_INTEGRATION_TESTS_VERIFIED
 }
 
 # If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'.
@@ -19,16 +19,17 @@ bundle_test_integration_cli() {
 #
 go_test_dir() {
 	dir=$1
-	coverpkg=$2
+	precompiled=$2
+	testbinary="$DEST/test.main"
 	testcover=()
 	testcoverprofile=()
-	testbinary="$DEST/test.main"
 	(
 		mkdir -p "$DEST/coverprofiles"
-		echo '+ go test' $TESTFLAGS "${DOCKER_PKG}${dir#.}"
-		cd "$dir"
 		export DEST="$ABS_DEST" # in a subshell this is safe -- our integration-cli tests need DEST, and "cd" screws it up
-		go test -c -o "$testbinary" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}"
+		if [ -z $precompiled ]; then
+			ensure_test_dir $1 $testbinary
+		fi
+		cd "$dir"
 		i=0
 		while ((++i)); do
 			test_env "$testbinary" $TESTFLAGS
@@ -40,22 +41,37 @@ go_test_dir() {
 	)
 }
 
-test_env() {
-	# use "env -i" to tightly control the environment variables that bleed into the tests
-	env -i \
-		DEST="$DEST" \
-		DOCKER_TLS_VERIFY="$DOCKER_TEST_TLS_VERIFY" \
-		DOCKER_CERT_PATH="$DOCKER_TEST_CERT_PATH" \
-		DOCKER_ENGINE_GOARCH="$DOCKER_ENGINE_GOARCH" \
-		DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" \
-		DOCKER_USERLANDPROXY="$DOCKER_USERLANDPROXY" \
-		DOCKER_HOST="$DOCKER_HOST" \
-		DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \
-		DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \
-		GOPATH="$GOPATH" \
-		GOTRACEBACK=all \
-		HOME="$ABS_DEST/fake-HOME" \
-		PATH="$PATH" \
-		TEMP="$TEMP" \
-		"$@"
+ensure_test_dir() {
+	(
+		# make sure a test dir will compile
+		dir="$1"
+		out="$2"
+		echo Building test dir: "$dir"
+		set -xe
+		cd "$dir"
+		go test -c -o "$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}"
+	)
+}
+
+test_env() {
+	(
+		set -xe
+		# use "env -i" to tightly control the environment variables that bleed into the tests
+		env -i \
+			DEST="$DEST" \
+			DOCKER_TLS_VERIFY="$DOCKER_TEST_TLS_VERIFY" \
+			DOCKER_CERT_PATH="$DOCKER_TEST_CERT_PATH" \
+			DOCKER_ENGINE_GOARCH="$DOCKER_ENGINE_GOARCH" \
+			DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" \
+			DOCKER_USERLANDPROXY="$DOCKER_USERLANDPROXY" \
+			DOCKER_HOST="$DOCKER_HOST" \
+			DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \
+			DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \
+			GOPATH="$GOPATH" \
+			GOTRACEBACK=all \
+			HOME="$ABS_DEST/fake-HOME" \
+			PATH="$PATH" \
+			TEMP="$TEMP" \
+			"$@"
+	)
 }

hack/make/build-integration-test-binary

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+rm -rf "$DEST"
+DEST="$DEST/../test-integration-cli"
+
+if [ -z $DOCKER_INTEGRATION_TESTS_VERIFIED ]; then
+	source ${MAKEDIR}/.integration-test-helpers
+	ensure_test_dir integration-cli "$DEST/test.main"
+	export DOCKER_INTEGRATION_TESTS_VERIFIED=1
+fi