testrunner: implement cluster provisioning support

We break out the provisioning part into its own script so that we can
more easily provision multiple hosts simultaneously.

We also make sure that SSH is all set up so that users can just directly
start SSH'ing between nodes.

Dockerfile

@@ -50,10 +50,6 @@ LABEL RUN="/usr/bin/docker run --rm --privileged \
              \${OPT1} \
              \${IMAGE}"
 
-# When run in e.g. Jenkins, it's really annoying to not see
-# any output of e.g. the provisioner until it's all done.
-ENV PYTHONUNBUFFERED 1
-
 COPY . /redhat-ci
 
 RUN pip3 install -r /redhat-ci/requirements.txt

main

@@ -38,6 +38,8 @@ main() {
         exit 0
     fi
 
+    export PYTHONUNBUFFERED=1
+
     exec $THIS_DIR/spawner.py
 }
 

provisioner

@@ -0,0 +1,133 @@
+#!/bin/bash
+set -Exeuo pipefail
+
+# This script provisions a node on OpenStack. It may be
+# called multiple times in parallel.
+
+THIS_DIR=$(dirname $0)
+
+source $THIS_DIR/utils/common.sh
+
+main() {
+
+    # NB: see the various NBs in the main() of main.
+
+    state=$1; shift
+    parsedhost=$1; shift
+    outdir=$1; shift
+
+    [ -d $parsedhost ]
+    mkdir $outdir
+
+    provision_host
+}
+
+provision_host() {
+
+    # XXX: We hardcode m1.small for now, but these really
+    # should be specified indirectly from the .redhat-ci
+    # YAML file through e.g. min-* vars.
+    env \
+        os_image="$(cat $parsedhost/distro)" \
+        os_flavor=m1.small \
+        os_name_prefix=github-ci-testnode \
+        os_user_data="$THIS_DIR/utils/user-data" \
+        "$THIS_DIR/utils/os_provision.py" $outdir
+
+    ssh_wait $(cat $outdir/node_addr) $state/node_key
+
+    if [ -f $parsedhost/ostree_revision ]; then
+        if ! on_atomic_host; then
+            update_github error "Cannot specify 'ostree' on non-AH."
+            touch $state/exit # signal testrunner to exit nicely
+            exit 0
+        fi
+        deploy_ostree
+    fi
+}
+
+deploy_ostree() {
+    local remote=$(cat $parsedhost/ostree_remote)
+    local branch=$(cat $parsedhost/ostree_branch)
+    local revision=$(cat $parsedhost/ostree_revision)
+
+    local rc=0
+    local skip_reboot=0
+    if [ -z "$remote" ] && [ -z "$branch" ]; then
+
+        if [ -z "$revision" ]; then
+            vmssh rpm-ostree upgrade --upgrade-unchanged-exit-77 || rc=$?
+        else
+            vmssh rpm-ostree deploy "$revision" || rc=$?
+        fi
+
+        if [ $rc == 77 ]; then
+            skip_reboot=1
+        elif [ $rc != 0 ]; then
+            update_github error "Failed to upgrade or deploy."
+            touch $state/exit # signal testrunner to exit nicely
+            exit 0
+        fi
+    else
+        local refspec
+
+        if [ -n "$remote" ]; then
+            vmssh ostree remote add --no-gpg-verify rhci "$remote"
+            refspec=rhci:
+        fi
+
+        if [ -n "$branch" ]; then
+            refspec="${refspec}$branch"
+        fi
+
+        if vmssh rpm-ostree rebase "$refspec"; then
+            update_github error "Failed to rebase onto refspec."
+            touch $state/exit # signal testrunner to exit nicely
+            exit 0
+        fi
+
+        if [ -n "$revision" ]; then
+            # we should really be able to do this in a single step
+            # https://github.com/projectatomic/rpm-ostree/issues/212
+            vmreboot
+            vmssh rpm-ostree deploy "$revision" || rc=$?
+
+            if [ $rc == 77 ]; then
+                skip_reboot=1
+            elif [ $rc != 0 ]; then
+                update_github error "Failed to upgrade or deploy."
+                touch $state/exit # signal testrunner to exit nicely
+                exit 0
+            fi
+        fi
+    fi
+
+    if [ $skip_reboot != 1 ]; then
+        vmreboot
+    fi
+}
+
+update_github() {
+    local context=$(cat $state/parsed/context)
+    common_update_github "$context" "$@"
+}
+
+vmssh() {
+    ssh -q -n -i $state/node_key \
+        -o StrictHostKeyChecking=no \
+        -o PasswordAuthentication=no \
+        -o UserKnownHostsFile=/dev/null \
+        root@$(cat $outdir/node_addr) "$@"
+}
+
+vmreboot() {
+    vmssh systemctl reboot || :
+    sleep 3 # give time for port to go down
+    ssh_wait $(cat $outdir/node_addr) $state/node_key
+}
+
+on_atomic_host() {
+    vmssh test -f /run/ostree-booted
+}
+
+main "$@"

spawner.py

@@ -100,6 +100,7 @@ def read_pipe(idx, fd):
     while s != b'':
         if not s.endswith(b'\n'):
             s += b'\n'
+            # pylint: disable=no-member
         sys.stdout.buffer.write((b'[%d] ' % idx) + s)
         s = fd.readline()