Add etcd integrations tests

- Added etcd integration test for overlay
    - Added etcd integration test for multinode
      with mock test driver suitable for circleci
    - Added multinode tests for zookeeper
    - Made the script smart enough to only start
      data stores necessary for the requested suites

Signed-off-by: Jana Radhakrishnan <mrjana@docker.com>

libnetwork/test/integration/dnet/helpers.bash

@@ -1,3 +1,7 @@
+function get_docker_bridge_ip() {
+    echo $(docker run --rm -it busybox ip route show | grep default | cut -d" " -f3)
+}
+
 function inst_id2port() {
     echo $((41000+${1}-1))
 }
@@ -108,6 +112,7 @@ function start_dnet() {
     hport=$((41000+${inst}-1))
     cport=2385
     hopt=""
+    store=${suffix}
 
     while [ -n "$1" ]
     do
@@ -122,7 +127,7 @@ function start_dnet() {
 	shift
     done
 
-    bridge_ip=$(docker inspect --format '{{.NetworkSettings.Gateway}}' pr_consul)
+    bridge_ip=$(get_docker_bridge_ip)
 
     echo "start_dnet parsed values: " ${inst} ${suffix} ${name} ${hport} ${cport} ${hopt} ${store} ${labels}
 
@@ -131,6 +136,8 @@ function start_dnet() {
 
     if [ "$store" = "zookeeper" ]; then
 	read discovery provider address < <(parse_discovery_str zk://${bridge_ip}:2182)
+    elif [ "$store" = "etcd" ]; then
+	read discovery provider address < <(parse_discovery_str etcd://${bridge_ip}:42000)
     else
 	read discovery provider address < <(parse_discovery_str consul://${bridge_ip}:8500)
     fi
@@ -149,7 +156,6 @@ title = "LibNetwork Configuration file for ${name}"
       provider = "${provider}"
       address = "${address}"
 EOF
-    echo $tomlfile}
     cat ${tomlfile}
     docker run \
 	   -d \
@@ -207,6 +213,27 @@ function runc() {
     dnet_exec ${dnet} "umount /var/run/netns/c && rm /var/run/netns/c"
 }
 
+function start_etcd() {
+    local bridge_ip
+    stop_etcd
+
+    bridge_ip=$(get_docker_bridge_ip)
+    docker run -d \
+	   --net=host \
+	   --name=dn_etcd \
+	   mrjana/etcd --listen-client-urls http://0.0.0.0:42000 \
+	   --advertise-client-urls http://${bridge_ip}:42000
+    sleep 2
+}
+
+function stop_etcd() {
+    docker stop dn_etcd || true
+    # You cannot destroy a container in Circle CI. So do not attempt destroy in circleci
+    if [ -z "$CIRCLECI" ]; then
+	docker rm -f dn_etcd || true
+    fi
+}
+
 function start_zookeeper() {
     stop_zookeeper
     docker run -d \
@@ -261,4 +288,6 @@ function test_overlay() {
 	net_disconnect ${i} container_${i} multihost
 	dnet_cmd $(inst_id2port $i) container rm container_${i}
     done
+
+    dnet_cmd $(inst_id2port 2) network rm multihost
 }

libnetwork/test/integration/dnet/overlay-consul.bats

@@ -5,7 +5,5 @@ load helpers
 
 @test "Test overlay network with consul" {
     skip_for_circleci
-    run test_overlay consul
-    [ "$status" -eq 0 ]
-    run dnet_cmd $(inst_id2port 2) network rm multihost
+    test_overlay consul
 }

libnetwork/test/integration/dnet/overlay-etcd.bats

@@ -0,0 +1,9 @@
+# -*- mode: sh -*-
+#!/usr/bin/env bats
+
+load helpers
+
+@test "Test overlay network with etcd" {
+    skip_for_circleci
+    test_overlay etcd
+}

libnetwork/test/integration/dnet/overlay-zookeeper.bats

@@ -5,8 +5,5 @@ load helpers
 
 @test "Test overlay network with zookeeper" {
     skip_for_circleci
-    run test_overlay zookeeper
-    [ "$status" -eq 0 ]
-    run dnet_cmd $(inst_id2port 2) network rm multihost
+    test_overlay zookeeper
 }
-

libnetwork/test/integration/dnet/run-integration-tests.sh

@@ -12,9 +12,9 @@ trap "cleanup_containers" EXIT SIGINT
 function cleanup_containers() {
     for c in "${!cmap[@]}";
     do
-	docker stop $c || true
+	docker stop $c 1>>${INTEGRATION_ROOT}/test.log 2>&1 || true
 	if [ -z "$CIRCLECI" ]; then
-	    docker rm -f $c || true
+	    docker rm -f $c 1>>${INTEGRATION_ROOT}/test.log 2>&1 || true
 	fi
     done
 
@@ -58,9 +58,6 @@ function run_overlay_consul_tests() {
 
 function run_overlay_zk_tests() {
     ## Test overlay network with zookeeper
-    start_zookeeper 1>>${INTEGRATION_ROOT}/test.log 2>&1
-    cmap[zookeeper_server]=zookeeper_server
-
     start_dnet 1 zookeeper 1>>${INTEGRATION_ROOT}/test.log 2>&1
     cmap[dnet-1-zookeeper]=dnet-1-zookeeper
     start_dnet 2 zookeeper 1>>${INTEGRATION_ROOT}/test.log 2>&1
@@ -76,9 +73,25 @@ function run_overlay_zk_tests() {
     unset cmap[dnet-2-zookeeper]
     stop_dnet 3 zookeeper 1>>${INTEGRATION_ROOT}/test.log 2>&1
     unset cmap[dnet-3-zookeeper]
+}
 
-    stop_zookeeper 1>>${INTEGRATION_ROOT}/test.log 2>&1
-    unset cmap[zookeeper_server]
+function run_overlay_etcd_tests() {
+    ## Test overlay network with etcd
+    start_dnet 1 etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dnet-1-etcd]=dnet-1-etcd
+    start_dnet 2 etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dnet-2-etcd]=dnet-2-etcd
+    start_dnet 3 etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dnet-3-etcd]=dnet-3-etcd
+
+    ./integration-tmp/bin/bats ./test/integration/dnet/overlay-etcd.bats
+
+    stop_dnet 1 etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    unset cmap[dnet-1-etcd]
+    stop_dnet 2 etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    unset cmap[dnet-2-etcd]
+    stop_dnet 3 etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    unset cmap[dnet-3-etcd]
 }
 
 function run_dnet_tests() {
@@ -100,27 +113,73 @@ function run_simple_tests() {
     unset cmap[dnet-1-simple]
 }
 
-function run_multi_tests() {
-    # Test multi node configuration with a global scope test driver
+function run_multi_consul_tests() {
+    # Test multi node configuration with a global scope test driver backed by consul
 
     ## Setup
-    start_dnet 1 multi 1>>${INTEGRATION_ROOT}/test.log 2>&1
-    cmap[dnet-1-multi]=dnet-1-multi
-    start_dnet 2 multi 1>>${INTEGRATION_ROOT}/test.log 2>&1
-    cmap[dnet-2-multi]=dnet-2-multi
-    start_dnet 3 multi 1>>${INTEGRATION_ROOT}/test.log 2>&1
-    cmap[dnet-3-multi]=dnet-3-multi
+    start_dnet 1 multi_consul consul 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dnet-1-multi_consul]=dnet-1-multi_consul
+    start_dnet 2 multi_consul consul 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dnet-2-multi_consul]=dnet-2-multi_consul
+    start_dnet 3 multi_consul consul 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dnet-3-multi_consul]=dnet-3-multi_consul
 
     ## Run the test cases
     ./integration-tmp/bin/bats ./test/integration/dnet/multi.bats
 
     ## Teardown
-    stop_dnet 1 multi 1>>${INTEGRATION_ROOT}/test.log 2>&1
-    unset cmap[dnet-1-multi]
-    stop_dnet 2 multi 1>>${INTEGRATION_ROOT}/test.log 2>&1
-    unset cmap[dnet-2-multi]
-    stop_dnet 3 multi 1>>${INTEGRATION_ROOT}/test.log 2>&1
-    unset cmap[dnet-3-multi]
+    stop_dnet 1 multi_consul 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    unset cmap[dnet-1-multi_consul]
+    stop_dnet 2 multi_consul 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    unset cmap[dnet-2-multi_consul]
+    stop_dnet 3 multi_consul 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    unset cmap[dnet-3-multi_consul]
+}
+
+function run_multi_zk_tests() {
+    # Test multi node configuration with a global scope test driver backed by zookeeper
+
+    ## Setup
+    start_dnet 1 multi_zk zookeeper 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dnet-1-multi_zk]=dnet-1-multi_zk
+    start_dnet 2 multi_zk zookeeper 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dnet-2-multi_zk]=dnet-2-multi_zk
+    start_dnet 3 multi_zk zookeeper 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dnet-3-multi_zk]=dnet-3-multi_zk
+
+    ## Run the test cases
+    ./integration-tmp/bin/bats ./test/integration/dnet/multi.bats
+
+    ## Teardown
+    stop_dnet 1 multi_zk 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    unset cmap[dnet-1-multi_zk]
+    stop_dnet 2 multi_zk 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    unset cmap[dnet-2-multi_zk]
+    stop_dnet 3 multi_zk 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    unset cmap[dnet-3-multi_zk]
+}
+
+function run_multi_etcd_tests() {
+    # Test multi node configuration with a global scope test driver backed by etcd
+
+    ## Setup
+    start_dnet 1 multi_etcd etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dnet-1-multi_etcd]=dnet-1-multi_etcd
+    start_dnet 2 multi_etcd etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dnet-2-multi_etcd]=dnet-2-multi_etcd
+    start_dnet 3 multi_etcd etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dnet-3-multi_etcd]=dnet-3-multi_etcd
+
+    ## Run the test cases
+    ./integration-tmp/bin/bats ./test/integration/dnet/multi.bats
+
+    ## Teardown
+    stop_dnet 1 multi_etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    unset cmap[dnet-1-multi_etcd]
+    stop_dnet 2 multi_etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    unset cmap[dnet-2-multi_etcd]
+    stop_dnet 3 multi_etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    unset cmap[dnet-3-multi_etcd]
 }
 
 source ./test/integration/dnet/helpers.bash
@@ -140,28 +199,44 @@ if [ ! -d ${TMPC_ROOT} ]; then
 fi
 
 # Suite setup
-start_consul 1>>${INTEGRATION_ROOT}/test.log 2>&1
-cmap[pr_consul]=pr_consul
 
 if [ -z "$SUITES" ]; then
     if [ -n "$CIRCLECI" ]
     then
 	# We can only run a limited list of suites in circleci because of the
 	# old kernel and limited docker environment.
-	suites="dnet simple multi"
+	suites="dnet simple multi_consul multi_zk multi_etcd"
     else
-	suites="dnet simple multi bridge overlay_consul overlay_zk"
+	suites="dnet simple multi_consul multi_zk multi_etcd  bridge overlay_consul overlay_zk overlay_etcd"
     fi
 else
     suites="$SUITES"
 fi
 
+if [[ "$suites" =~ .*consul.* ]]; then
+    echo "Starting consul ..."
+    start_consul 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[pr_consul]=pr_consul
+fi
+
+if [[ "$suites" =~ .*zk.* ]]; then
+    echo "Starting zookeeper ..."
+    start_zookeeper 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[zookeeper_server]=zookeeper_server
+fi
+
+if [[ "$suites" =~ .*etcd.* ]]; then
+    echo "Starting etcd ..."
+    start_etcd 1>>${INTEGRATION_ROOT}/test.log 2>&1
+    cmap[dn_etcd]=dn_etcd
+fi
+
+echo ""
+
 for suite in ${suites};
 do
     suite_func=run_${suite}_tests
+    echo "Running ${suite}_tests ..."
     declare -F $suite_func >/dev/null && $suite_func
+    echo ""
 done
-
-# Suite teardowm
-stop_consul 1>>${INTEGRATION_ROOT}/test.log 2>&1
-unset cmap[pr_consul]