Support autoscaling and test in CI

.github/workflows/arcus.yml

@@ -0,0 +1,146 @@
+
+name: Test on Arcus OpenStack in rcp-cloud-portal-demo
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+concurrency: rcp-cloud-portal_demo # openstack project
+jobs:
+  arcus:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup ssh
+        run: |
+          set -x
+          mkdir ~/.ssh
+          echo "$SSH_KEY" > ~/.ssh/id_rsa
+          chmod 0600 ~/.ssh/id_rsa
+        env:
+          SSH_KEY: ${{ secrets.ARCUS_SSH_KEY }}
+
+      - name: Add bastion's ssh key to known_hosts
+        run: cat environments/arcus/bastion_fingerprint >> ~/.ssh/known_hosts
+        shell: bash
+
+      - name: Install ansible etc
+        run: dev/setup-env.sh
+
+      - name: Install terraform
+        uses: hashicorp/setup-terraform@v1
+
+      - name: Initialise terraform
+        run: terraform init
+        working-directory: ${{ github.workspace }}/environments/arcus/terraform
+
+      - name: Write clouds.yaml
+        run: |
+          mkdir -p ~/.config/openstack/
+          echo "$CLOUDS_YAML" > ~/.config/openstack/clouds.yaml
+        shell: bash
+        env:
+          CLOUDS_YAML: ${{ secrets.ARCUS_CLOUDS_YAML }}
+
+      - name: Provision infrastructure
+        id: provision
+        run: |
+          . venv/bin/activate
+          . environments/arcus/activate
+          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
+          terraform apply -auto-approve
+        env:
+          OS_CLOUD: openstack
+          TF_VAR_cluster_name: ci${{ github.run_id }}
+
+      - name: Get server provisioning failure messages
+        id: provision_failure
+        run: |
+          . venv/bin/activate
+          . environments/arcus/activate
+          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
+          echo "::set-output name=messages::$(./getfaults.py)"
+        env:
+          OS_CLOUD: openstack
+          TF_VAR_cluster_name: ci${{ github.run_id }}
+        if: always() && steps.provision.outcome == 'failure'
+
+      - name: Delete infrastructure if failed due to lack of hosts
+        run: |
+          . venv/bin/activate
+          . environments/arcus/activate
+          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
+          terraform destroy -auto-approve
+        env:
+          OS_CLOUD: openstack
+          TF_VAR_cluster_name: ci${{ github.run_id }}
+        if: ${{ always() && steps.provision.outcome == 'failure' && contains('not enough hosts available', steps.provision_failure.messages) }}
+
+      - name: Directly configure cluster and build compute + login images
+        # see pre-hook for the image build
+        run: |
+          . venv/bin/activate
+          . environments/arcus/activate
+          ansible all -m wait_for_connection
+          ansible-playbook ansible/adhoc/generate-passwords.yml
+          ansible-playbook -vv ansible/site.yml
+        env:
+          OS_CLOUD: openstack
+          ANSIBLE_FORCE_COLOR: True
+
+      - name: Test reimage of login and compute nodes
+        run: |
+          . venv/bin/activate
+          . environments/arcus/activate
+          ansible all -m wait_for_connection
+          ansible-playbook -vv ansible/ci/test_reimage.yml
+        env:
+          OS_CLOUD: openstack
+          ANSIBLE_FORCE_COLOR: True
+
+      - name: Update cloud image and reconfigure Slurm
+        run: |
+          . venv/bin/activate
+          . environments/arcus/activate
+          ansible-playbook -vv ansible/ci/update_cloudnode_image.yml
+          ansible-playbook -vv ansible/slurm.yml --tags openhpc --skip-tags install
+        env:
+          ANSIBLE_FORCE_COLOR: True
+          OS_CLOUD: openstack
+
+      - name: Run MPI-based tests (triggers autoscaling)
+        run: |
+          . venv/bin/activate
+          . environments/arcus/activate
+          ansible-playbook -vv ansible/adhoc/hpctests.yml
+        env:
+          ANSIBLE_FORCE_COLOR: True
+
+      - name: Wait for CLOUD nodes to be destroyed
+        run: |
+          . venv/bin/activate
+          . environments/arcus/activate
+          ansible-playbook -vv ansible/ci/wait_for_scaledown.yml
+        env:
+          OS_CLOUD: openstack
+          ANSIBLE_FORCE_COLOR: True
+
+      - name: Delete infrastructure
+        run: |
+          . venv/bin/activate
+          . environments/arcus/activate
+          cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
+          terraform destroy -auto-approve
+        env:
+          TF_VAR_cluster_name: ci${{ github.run_id }}
+        if: ${{ success() || cancelled() }}
+
+      - name: Delete images
+        run: |
+          . venv/bin/activate
+          . environments/arcus/activate
+          ansible-playbook -vv ansible/ci/delete_images.yml
+        env:
+          OS_CLOUD: openstack
+          ANSIBLE_FORCE_COLOR: True

.github/workflows/smslabs.yml

@@ -1,13 +1,13 @@
 
-name: Test on OpenStack via smslabs
+name: Test on SMS-Labs OpenStack in stackhpc-ci
 on:
   push:
     branches:
       - main
   pull_request:
 concurrency: stackhpc-ci # openstack project
 jobs:
-  openstack-example:
+  smslabs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
@@ -22,7 +22,7 @@ jobs:
           SSH_KEY: ${{ secrets.SSH_KEY }}
 
       - name: Add bastion's ssh key to known_hosts
-        run: cat environments/smslabs-example/bastion_fingerprint >> ~/.ssh/known_hosts
+        run: cat environments/smslabs/bastion_fingerprint >> ~/.ssh/known_hosts
         shell: bash
 
       - name: Install ansible etc
@@ -33,7 +33,7 @@ jobs:
 
       - name: Initialise terraform
         run: terraform init
-        working-directory: ${{ github.workspace }}/environments/smslabs-example/terraform
+        working-directory: ${{ github.workspace }}/environments/smslabs/terraform
 
       - name: Write clouds.yaml
         run: |
@@ -47,7 +47,7 @@ jobs:
         id: provision
         run: |
           . venv/bin/activate
-          . environments/smslabs-example/activate
+          . environments/smslabs/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform apply -auto-approve
         env:
@@ -58,7 +58,7 @@ jobs:
         id: provision_failure
         run: |
           . venv/bin/activate
-          . environments/smslabs-example/activate
+          . environments/smslabs/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           echo "::set-output name=messages::$(./getfaults.py)"
         env:
@@ -69,71 +69,81 @@ jobs:
       - name: Delete infrastructure if failed due to lack of hosts
         run: |
           . venv/bin/activate
-          . environments/smslabs-example/activate
+          . environments/smslabs/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform destroy -auto-approve
         env:
           OS_CLOUD: openstack
           TF_VAR_cluster_name: ci${{ github.run_id }}
         if: ${{ always() && steps.provision.outcome == 'failure' && contains('not enough hosts available', steps.provision_failure.messages) }}
 
-      - name: Configure infrastructure
+      - name: Directly configure cluster and build compute + login images
+        # see pre-hook for the image build
         run: |
           . venv/bin/activate
-          . environments/smslabs-example/activate
+          . environments/smslabs/activate
           ansible all -m wait_for_connection
           ansible-playbook ansible/adhoc/generate-passwords.yml
+          echo test_user_password: "$TEST_USER_PASSWORD" > $APPLIANCES_ENVIRONMENT_ROOT/inventory/group_vars/basic_users/defaults.yml
           ansible-playbook -vv ansible/site.yml
         env:
+          OS_CLOUD: openstack
           ANSIBLE_FORCE_COLOR: True
           TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
-      - name: Run MPI-based tests
+      - name: Test reimage of login and compute nodes
         run: |
           . venv/bin/activate
-          . environments/smslabs-example/activate
-          ansible-playbook -vv ansible/adhoc/hpctests.yml
+          . environments/smslabs/activate
+          ansible all -m wait_for_connection
+          ansible-playbook -vv ansible/ci/test_reimage.yml
         env:
+          OS_CLOUD: openstack
           ANSIBLE_FORCE_COLOR: True
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
-      - name: Build control and compute images
+      - name: Update cloud image and reconfigure Slurm
         run: |
           . venv/bin/activate
-          . environments/smslabs-example/activate
-          cd packer
-          PACKER_LOG=1 PACKER_LOG_PATH=build.log packer build -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
+          . environments/smslabs/activate
+          ansible-playbook -vv ansible/ci/update_cloudnode_image.yml
+          ansible-playbook -vv ansible/slurm.yml --tags openhpc --skip-tags install
         env:
+          ANSIBLE_FORCE_COLOR: True
           OS_CLOUD: openstack
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
-      - name: Reimage compute nodes via slurm and check cluster still up
+      - name: Run MPI-based tests (triggers autoscaling)
         run: |
           . venv/bin/activate
-          . environments/smslabs-example/activate
-          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/ci/reimage-compute.yml
-          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/hooks/post.yml
+          . environments/smslabs/activate
+          ansible-playbook -vv ansible/adhoc/hpctests.yml
         env:
+          ANSIBLE_FORCE_COLOR: True
           OS_CLOUD: openstack
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
-      - name: Reimage login nodes via openstack and check cluster still up
+      - name: Wait for CLOUD nodes to be destroyed
         run: |
           . venv/bin/activate
-          . environments/smslabs-example/activate
-          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/ci/reimage-login.yml
-          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/hooks/post.yml
+          . environments/smslabs/activate
+          ansible-playbook -vv ansible/ci/wait_for_scaledown.yml
         env:
           OS_CLOUD: openstack
-          TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+          ANSIBLE_FORCE_COLOR: True
 
       - name: Delete infrastructure
         run: |
           . venv/bin/activate
-          . environments/smslabs-example/activate
+          . environments/smslabs/activate
           cd $APPLIANCES_ENVIRONMENT_ROOT/terraform
           terraform destroy -auto-approve
         env:
-          OS_CLOUD: openstack
           TF_VAR_cluster_name: ci${{ github.run_id }}
         if: ${{ success() || cancelled() }}
+
+      - name: Delete images
+        run: |
+          . venv/bin/activate
+          . environments/smslabs/activate
+          ansible-playbook -vv ansible/ci/delete_images.yml
+        env:
+          OS_CLOUD: openstack
+          ANSIBLE_FORCE_COLOR: True

ansible/adhoc/restart-slurm.yml

@@ -20,7 +20,7 @@
         name: slurmctld
         state: restarted
 
-- hosts: compute,login
+- hosts: compute,login # FIXME: doesn't work if using `login` as combined slurmctld
   become: yes
   gather_facts: no
   tasks:

environments/smslabs-example/ci/reimage-login.yml → ansible/ci/delete_images.yml

@@ -1,23 +1,23 @@
-# Reimage login nodes via OpenStack
-
-- hosts: login
+- hosts: login:!builder
   become: no
+  gather_facts: no
   tasks:
     - name: Read packer build manifest
       set_fact:
         manifest: "{{ lookup('file', manifest_path) | from_json }}"
       vars:
         manifest_path: "{{ lookup('env', 'APPLIANCES_REPO_ROOT') }}/packer/packer-manifest.json"
       delegate_to: localhost
-      
-    - name: Get latest login image build
+
+    - name: Get latest image builds
       set_fact:
         login_build: "{{ manifest['builds'] | selectattr('custom_data', 'eq', {'source': 'login'}) | last }}"
+        compute_build: "{{ manifest['builds'] | selectattr('custom_data', 'eq', {'source': 'compute'}) | last }}"
 
-    - name: Reimage node via openstack
+    - name: Delete images
       shell:
-        cmd: "openstack server rebuild {{ instance_id | default(inventory_hostname) }} --image {{ login_build.artifact_id }}"
+        cmd: |
+          openstack image delete {{ login_build.artifact_id }}
+          openstack image delete {{ compute_build.artifact_id }}
       delegate_to: localhost
-
-    - name: Wait for connection
-      wait_for_connection:
+