Redesign integration testing to copy images to nested containers

Previously, the strategy for avoiding lots of rebuilding and repulling
for each Docker version being tested was to use a mountpoint to persist
/var/lib/docker. This was pretty broken, and may not be a reliable
strategy. This commit changes the scripts to instead build/pull images
outside the innermost container, and copy them to the final test
environment with docker save/docker load.

This requires a fair amount of changes, since run.sh must now
communicate with the Docker engine that was formerly started by
test_runner.sh. The code that starts this engine has been broken out to
run_engine.sh so that starting the engine and running the tests under it
can be done separately (with the images loaded in between these steps).

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

contrib/docker-integration/helpers.bash

@@ -0,0 +1,21 @@
+# Start docker daemon
+function start_daemon() {
+	# Drivers to use for Docker engines the tests are going to create.
+	STORAGE_DRIVER=${STORAGE_DRIVER:-overlay}
+	EXEC_DRIVER=${EXEC_DRIVER:-native}
+
+	docker --daemon --log-level=panic \
+		--storage-driver="$STORAGE_DRIVER" --exec-driver="$EXEC_DRIVER" &
+	DOCKER_PID=$!
+
+	# Wait for it to become reachable.
+	tries=10
+	until docker version &> /dev/null; do
+		(( tries-- ))
+		if [ $tries -le 0 ]; then
+			echo >&2 "error: daemon failed to start"
+			exit 1
+		fi
+		sleep 1
+	done
+}

contrib/docker-integration/install_certs.sh

@@ -1,6 +1,10 @@
 #!/bin/sh
 set -e
 
+# Set IP address in /etc/hosts for localregistry
+IP=$(ifconfig eth0|grep "inet addr:"| cut -d: -f2 | awk '{ print $1}')
+echo "$IP localregistry" >> /etc/hosts
+
 hostname=$1
 if [ "$hostname" = "" ]; then
 	hostname="localhost"

contrib/docker-integration/run.sh

@@ -1,11 +1,20 @@
 #!/usr/bin/env bash
 set -e
+set -x
+
+source helpers.bash
 
 cd "$(dirname "$(readlink -f "$BASH_SOURCE")")"
 
+# Port used by engine under test
+ENGINE_PORT=5216
+
 # Root directory of Distribution
 DISTRIBUTION_ROOT=$(cd ../..; pwd -P)
 
+DOCKER_GRAPHDRIVER=${DOCKER_GRAPHDRIVER:-overlay}
+EXEC_DRIVER=${EXEC_DRIVER:-native}
+
 volumeMount=""
 if [ "$DOCKER_VOLUME" != "" ]; then
 	volumeMount="-v ${DOCKER_VOLUME}:/var/lib/docker"
@@ -14,18 +23,58 @@ fi
 dockerMount=""
 if [ "$DOCKER_BINARY" != "" ]; then
 	dockerMount="-v ${DOCKER_BINARY}:/usr/local/bin/docker"
+else
+	DOCKER_BINARY=docker
 fi
 
 # Image containing the integration tests environment.
 INTEGRATION_IMAGE=${INTEGRATION_IMAGE:-distribution/docker-integration}
 
+if [ "$1" == "-d" ]; then
+	start_daemon
+	shift
+fi
+
+TESTS=${@:-.}
+
 # Make sure we upgrade the integration environment.
 docker pull $INTEGRATION_IMAGE
 
-# Start the integration tests in a Docker container.
-docker run --rm -t --privileged $volumeMount $dockerMount \
+# Start a Docker engine inside a docker container
+ID=$(docker run -d -it -p $ENGINE_PORT:$ENGINE_PORT --privileged $volumeMount $dockerMount \
 	-v ${DISTRIBUTION_ROOT}:/go/src/github.com/docker/distribution \
-	-e "STORAGE_DRIVER=$DOCKER_GRAPHDRIVER" \
+	-e "ENGINE_PORT=$ENGINE_PORT" \
+	-e "DOCKER_GRAPHDRIVER=$DOCKER_GRAPHDRIVER" \
 	-e "EXEC_DRIVER=$EXEC_DRIVER" \
 	${INTEGRATION_IMAGE} \
-	./test_runner.sh "$@"
+	./run_engine.sh)
+
+# Wait for it to become reachable.
+tries=10
+until "$DOCKER_BINARY" -H "127.0.0.1:$ENGINE_PORT" version &> /dev/null; do
+	(( tries-- ))
+	if [ $tries -le 0 ]; then
+		echo >&2 "error: daemon failed to start"
+		exit 1
+	fi
+	sleep 1
+done
+
+# Make sure we have images outside the container, to transfer to the container.
+# Not much will happen here if the images are already present.
+docker-compose pull
+docker-compose build
+
+# Transfer images to the inner container.
+for image in "$INTEGRATION_IMAGE" registry:0.9.1 dockerintegration_nginx dockerintegration_registryv2; do
+	docker save "$image" | "$DOCKER_BINARY" -H "127.0.0.1:$ENGINE_PORT" load
+done
+
+#DOCKER_HOST="tcp://127.0.0.1:$ENGINE_PORT" docker-compose pull
+#DOCKER_HOST="tcp://127.0.0.1:$ENGINE_PORT" docker-compose build
+
+# Run the tests.
+docker exec -it "$ID" sh -c "DOCKER_HOST=tcp://127.0.0.1:$ENGINE_PORT ./test_runner.sh $TESTS"
+
+# Stop container
+docker rm -f -v "$ID"

contrib/docker-integration/run_engine.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -e
+set -x
+
+# Set IP address in /etc/hosts for localregistry
+IP=$(ifconfig eth0|grep "inet addr:"| cut -d: -f2 | awk '{ print $1}')
+echo "$IP localregistry" >> /etc/hosts
+
+sh install_certs.sh localregistry
+
+docker --daemon -H "0.0.0.0:$ENGINE_PORT" --log-level=panic \
+	--storage-driver="$DOCKER_GRAPHDRIVER" --exec-driver="$EXEC_DRIVER"

contrib/docker-integration/run_multiversion.sh

@@ -5,44 +5,20 @@
 set -e
 set -x
 
-# Don't use /tmp because this isn't available in boot2docker
-tmpdir_template="`pwd`/docker-versions.XXXXX"
+source helpers.bash
+
+if [ `uname` = "Linux" ]; then
+	tmpdir_template="$TMPDIR/docker-versions.XXXXX"
+else
+	# /tmp isn't available for mounting in boot2docker
+	tmpdir_template="`pwd`/../../../docker-versions.XXXXX"
+fi
+
 tmpdir=`mktemp -d "$tmpdir_template"`
 trap "rm -rf $tmpdir" EXIT
 
 if [ "$1" == "-d" ]; then
-	# Start docker daemon
-
-	# Drivers to use for Docker engines the tests are going to create.
-	STORAGE_DRIVER=${STORAGE_DRIVER:-overlay}
-	EXEC_DRIVER=${EXEC_DRIVER:-native}
-
-	docker --daemon --log-level=panic \
-		--storage-driver="$STORAGE_DRIVER" --exec-driver="$EXEC_DRIVER" &
-	DOCKER_PID=$!
-
-	# Wait for it to become reachable.
-	tries=10
-	until docker version &> /dev/null; do
-		(( tries-- ))
-		if [ $tries -le 0 ]; then
-			echo >&2 "error: daemon failed to start"
-			exit 1
-		fi
-		sleep 1
-	done
-fi
-
-# If DOCKER_VOLUME is unset, create a temporary directory to cache containers
-# between runs
-# Only do this on Linux, because using /var/lib/docker from a host volume seems
-# problematic with boot2docker.
-if [ "$DOCKER_VOLUME" = "" -a `uname` = "Linux" ]; then
-	volumes_template="`pwd`/docker-versions.XXXXX"
-	volume=`mktemp -d "$volumes_template"`
-	trap "rm -rf $tmpdir $volume" EXIT
-else
-	volume="$DOCKER_VOLUME"
+	start_daemon
 fi
 
 # Released versions
@@ -56,7 +32,7 @@ for v in $versions; do
 	docker cp "$ID:/usr/local/bin/docker" "$tmpdir/docker-$v"
 
 	echo "Running tests with Docker $v"
-	DOCKER_BINARY="$binpath" DOCKER_VOLUME="$volume" ./run.sh
+	DOCKER_BINARY="$binpath" DOCKER_VOLUME="$DOCKER_VOLUME" DOCKER_GRAPHDRIVER="$DOCKER_GRAPHDRIVER" ./run.sh
 
 	# Cleanup.
 	docker rm -f "$ID"
@@ -71,7 +47,7 @@ ID=$(docker create dockerswarm/dind-master)
 docker cp "$ID:/usr/local/bin/docker" "$tmpdir/docker-master"
 
 echo "Running tests with Docker master"
-DOCKER_BINARY="$binpath" DOCKER_VOLUME="$volume" ./run.sh
+DOCKER_BINARY="$binpath" DOCKER_VOLUME="$DOCKER_VOLUME" ./run.sh
 
 # Cleanup.
 docker rm -f "$ID"

contrib/docker-integration/test_runner.sh

@@ -3,48 +3,16 @@ set -e
 
 cd "$(dirname "$(readlink -f "$BASH_SOURCE")")"
 
-# Load the helpers.
-#. helpers.bash
-
 TESTS=${@:-.}
 
-# Drivers to use for Docker engines the tests are going to create.
-STORAGE_DRIVER=${STORAGE_DRIVER:-overlay}
-EXEC_DRIVER=${EXEC_DRIVER:-native}
-
-
 function execute() {
 	>&2 echo "++ $@"
 	eval "$@"
 }
 
-# Set IP address in /etc/hosts for localregistry
-IP=$(ifconfig eth0|grep "inet addr:"| cut -d: -f2 | awk '{ print $1}')
-execute echo "$IP localregistry" >> /etc/hosts
-
-# Setup certificates
-execute sh install_certs.sh localregistry
-
-# Start the docker engine.
-execute docker --daemon --log-level=panic \
-	--storage-driver="$STORAGE_DRIVER" --exec-driver="$EXEC_DRIVER" &
-DOCKER_PID=$!
-
-# Wait for it to become reachable.
-tries=10
-until docker version &> /dev/null; do
-	(( tries-- ))
-	if [ $tries -le 0 ]; then
-		echo >&2 "error: daemon failed to start"
-		exit 1
-	fi
-	sleep 1
-done
-
 execute time docker-compose build
 
 execute docker-compose up -d
 
 # Run the tests.
 execute time bats -p $TESTS
-