integration tests: add CI scripts

Add some scripts that can be run from Jenkins to build/push test images
and to run the tests.
Updated README to expand on running tests.

test-requirements.txt

@@ -12,4 +12,3 @@ coverage==4.3.4
 mock==2.0.0
 pytest==3.0.7
 pytest-cov==2.4.0
-docker-py==1.10.6

test/integration/README.md

@@ -1,12 +1,39 @@
 # Integration tests
 
-Integration tests exercise the OpenShift Ansible playbooks by performing
-simulated installations in Docker containers.
+Integration tests exercise the OpenShift Ansible playbooks by running them
+against an inventory with Docker containers as hosts.
+
+## Requirements
+
+The tests assume that:
+
+* docker is running on localhost and the present user has access to use it.
+* golang is installed and the go binary is in PATH.
+* python and tox are installed.
+
+## Building images
+
+The tests rely on images built in the local docker index. You can build them
+from the repository root with:
+
+```
+./test/integration/build-images.sh
+```
+
+Use the `--help` option to view available options.
 
 ## Running the tests
 
-From the repository root, run with:
+From the repository root, run the integration tests with:
+
+```
+./test/integration/run-tests.sh
+```
+
+Use the `--help` option to view available options.
+
+You can also run tests more directly, for example to run a specific check:
 
 ```
-tox -e integration
+go test ./test/integration/... -run TestPackageUpdateDepMissing
 ```

test/integration/build-images.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# This is intended to run either locally (in which case a push is not
+# necessary) or in a CI job (where the results should be pushed to a
+# registry for use in later CI test jobs). Images are tagged locally with
+# both the base name (e.g. "test-target-base") and with the prefix given;
+# then only the prefixed name is pushed if --push is specified, assuming
+# any necessary credentials are available for the push. The same prefix
+# can then be used for the testing script. By default a local (non-registry)
+# prefix is used and no push can occur. To push to e.g. dockerhub:
+#
+# ./build-images.sh --push --prefix=docker.io/openshift/ansible-integration-
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+STARTTIME=$(date +%s)
+source_root=$(dirname "${0}")
+
+prefix="${PREFIX:-openshift-ansible-integration-}"
+push=false
+verbose=false
+build_options="${DOCKER_BUILD_OPTIONS:-}"
+help=false
+
+for args in "$@"
+do
+  case $args in
+      --prefix=*)
+        prefix="${args#*=}"
+        ;;
+      --push)
+        push=true
+        ;;
+      --no-cache)
+        build_options="${build_options} --no-cache"
+        ;;
+      --verbose)
+        verbose=true
+        ;;
+      --help)
+        help=true
+        ;;
+  esac
+done
+
+if [ "$help" = true ]; then
+  echo "Builds the docker images for openshift-ansible integration tests"
+  echo "and pushes them to a central registry."
+  echo
+  echo "Options: "
+  echo "  --prefix=PREFIX"
+  echo "  The prefix to use for the image names."
+  echo "  default: openshift-ansible-integration-"
+  echo
+  echo "  --push"
+  echo "  If set will push the tagged image"
+  echo 
+  echo "  --no-cache"
+  echo "  If set will perform the build without a cache."
+  echo
+  echo "  --verbose"
+  echo "  Enables printing of the commands as they run."
+  echo
+  echo "  --help"
+  echo "  Prints this help message"
+  echo
+  exit 0
+fi
+
+if [ "$verbose" = true ]; then
+  set -x
+fi
+
+
+declare -a build_order                       ; declare -A images
+build_order+=( test-target-base )            ; images[test-target-base]=openshift_health_checker/builds/test-target-base
+build_order+=( preflight-aos-package-checks ); images[preflight-aos-package-checks]=openshift_health_checker/builds/aos-package-checks
+for image in "${build_order[@]}"; do
+  BUILD_STARTTIME=$(date +%s)
+  docker_tag=${prefix}${image}
+  echo
+  echo "--- Building component '$image' with docker tag '$docker_tag' ---"
+  docker build ${build_options} -t $image -t $docker_tag "$source_root/${images[$image]}"
+  echo
+  BUILD_ENDTIME=$(date +%s); echo "--- build $docker_tag took $(($BUILD_ENDTIME - $BUILD_STARTTIME)) seconds ---"
+  if [ "$push" = true ]; then
+    docker push $docker_tag
+    PUSH_ENDTIME=$(date +%s); echo "--- push $docker_tag took $(($PUSH_ENDTIME - $BUILD_ENDTIME)) seconds ---"
+  fi
+done
+
+echo
+echo
+echo "++ Active images"
+docker images | grep ${prefix} | sort
+echo
+
+
+ret=$?; ENDTIME=$(date +%s); echo "$0 took $(($ENDTIME - $STARTTIME)) seconds"; exit "$ret"

test/integration/openshift_health_checker/common.go

@@ -25,7 +25,7 @@ func (p PlaybookTest) Run(t *testing.T) {
 	// A PlaybookTest is intended to be run in parallel with other tests.
 	t.Parallel()
 
-	cmd := exec.Command("ansible-playbook", p.Path)
+	cmd := exec.Command("ansible-playbook", "-i", "/dev/null", p.Path)
 	cmd.Env = append(os.Environ(), "ANSIBLE_FORCE_COLOR=1")
 	b, err := cmd.CombinedOutput()
 

test/integration/openshift_health_checker/setup_container.yml

@@ -23,7 +23,7 @@
     - name: start container
       docker_container:
         name: "{{ container_name }}"
-        image: "{{ image | default('test-target-base') }}"
+        image: "{{ lookup('env', 'IMAGE_PREFIX') | default('openshift-ansible-integration-', true) }}{{ image | default('test-target-base') }}"
         command: "{{ command | default('sleep 1800') }}"
         recreate: yes
         # NOTE: When/if we need to run containers that are docker hosts as well:

test/integration/run-tests.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# This script runs the golang integration tests in the directories underneath.
+# It should be run from the same directory it is in, or in a directory above.
+# Specify the same image prefix used (if any) with build-images.sh
+#
+# Example:
+# ./run-tests.sh --prefix=docker.io/openshift/ansible-integration- --parallel=16
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+source_root=$(dirname "${0}")
+
+prefix="${PREFIX:-openshift-ansible-integration-}"
+gotest_options="${GOTEST_OPTIONS:--v}"
+push=false
+verbose=false
+help=false
+
+for args in "$@"
+do
+  case $args in
+      --prefix=*)
+        prefix="${args#*=}"
+        ;;
+      --parallel=*)
+        gotest_options="${gotest_options} -parallel ${args#*=}"
+        ;;
+      --verbose)
+        verbose=true
+        ;;
+      --help)
+        help=true
+        ;;
+  esac
+done
+
+if [ "$help" = true ]; then
+  echo "Runs the openshift-ansible integration tests."
+  echo
+  echo "Options: "
+  echo "  --prefix=PREFIX"
+  echo "  The prefix to use for the image names."
+  echo "  default: openshift-ansible-integration-"
+  echo
+  echo "  --parallel=NUMBER"
+  echo "  Number of tests to run in parallel."
+  echo "  default: GOMAXPROCS (typically, number of processors)"
+  echo
+  echo "  --verbose"
+  echo "  Enables printing of the commands as they run."
+  echo
+  echo "  --help"
+  echo "  Prints this help message"
+  echo
+  exit 0
+fi
+
+
+
+if ! [ -d $source_root/../../.tox/integration ]; then
+  # have tox create a consistent virtualenv
+  pushd $source_root/../..; tox -e integration; popd
+fi
+# use the virtualenv from tox
+set +o nounset; source $source_root/../../.tox/integration/bin/activate; set -o nounset
+
+if [ "$verbose" = true ]; then
+  set -x
+fi
+
+# Run the tests. NOTE: "go test" requires a relative path for this purpose.
+# The PWD trick below will only work if cwd is in/above where this script lives.
+retval=0
+IMAGE_PREFIX="${prefix}" env -u GOPATH \
+  go test ./${source_root#$PWD}/... ${gotest_options}
+
+

tox.ini

@@ -13,8 +13,7 @@ deps =
     -rrequirements.txt
     -rtest-requirements.txt
     py35-flake8: flake8-bugbear==17.3.0
-
-whitelist_externals = env
+    integration: docker-py==1.10.6
 
 commands =
     unit: pip install -e utils
@@ -25,8 +24,9 @@ commands =
     generate_validation: python setup.py generate_validation
     # TODO(rhcarvalho): check syntax of other important entrypoint playbooks
     ansible_syntax: python setup.py ansible_syntax
-
-    # Unset GOPATH because tests use relative imports. This should be removed if
-    # we require openshift-ansible to live in a Go work space and use absolute
-    # imports in tests (desirable).
-    integration: env -u GOPATH go test -v ./test/integration/...
+    # ansible 2.2.2+ unfortunately breaks the integration test runner
+    # because it can no longer set facts on the test docker hosts.
+    # So for now, install separate ansible version for integration.
+    # PR that fixes it: https://github.com/ansible/ansible/pull/23599
+    # Once that PR is available, drop this and use same ansible.
+    integration: pip install ansible==2.2.1.0