From 8a8fd37f6fb53716cb4b3a7e93e1e3cf385927e2 Mon Sep 17 00:00:00 2001
From: Olli Janatuinen <olli.janatuinen@gmail.com>
Date: Mon, 7 Jan 2019 19:05:54 +0200
Subject: [PATCH] CI: Introduce flaky test finder

comparing PR commit(s) to HEAD of moby/moby master branch and if founds
new (or renamed) integration tests will run stress tests for them.

Signed-off-by: Olli Janatuinen <olli.janatuinen@gmail.com>
---
 Makefile                         |  3 +++
 hack/ci/janky                    |  1 +
 hack/make/test-integration-flaky | 26 ++++++++++++++++++++++++++
 3 files changed, 30 insertions(+)
 create mode 100644 hack/make/test-integration-flaky

diff --git a/Makefile b/Makefile
index 945678e1a7..e8431fae73 100644
--- a/Makefile
+++ b/Makefile
@@ -165,6 +165,9 @@ test-integration-cli: test-integration ## (DEPRECATED) use test-integration
 test-integration: build ## run the integration tests
 	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration
 
+test-integration-flaky: build ## run the stress test for all new integration tests
+	$(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration-flaky
+
 test-unit: build ## run the unit tests
 	$(DOCKER_RUN_DOCKER) hack/test/unit
 
diff --git a/hack/ci/janky b/hack/ci/janky
index f2bdfbf326..88cb9d9c61 100755
--- a/hack/ci/janky
+++ b/hack/ci/janky
@@ -13,5 +13,6 @@ hack/make.sh \
 	binary-daemon \
 	dynbinary \
 	test-docker-py \
+	test-integration-flaky \
 	test-integration \
 	cross
diff --git a/hack/make/test-integration-flaky b/hack/make/test-integration-flaky
new file mode 100644
index 0000000000..00723a75bb
--- /dev/null
+++ b/hack/make/test-integration-flaky
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -e -o pipefail
+
+source hack/validate/.validate
+new_tests=$(
+    validate_diff --diff-filter=ACMR --unified=0 -- 'integration/*_test.go' |
+    grep -E '^(\+func )(.*)(\*testing)' || true
+)
+
+if [ -z "$new_tests" ]; then
+    echo 'No new tests added to integration.'
+    return
+fi
+
+echo
+echo "Found new integrations tests:"
+echo "$new_tests"
+echo "Running stress test for them."
+
+(
+    TESTARRAY=$(echo "$new_tests" | sed 's/+func //' | awk -F'\\(' '{print $1}' | tr '\n' '|')
+    export TESTFLAGS="-test.count 5 -test.run ${TESTARRAY%?}"
+    export TEST_REPEAT=5
+    echo "Using test flags: $TESTFLAGS"
+    source hack/make/test-integration
+)