Enable autoscaling

.github/workflows/smslabs.yml

@@ -7,7 +7,7 @@ on:
   pull_request:
 concurrency: stackhpc-ci # openstack project
 jobs:
-  openstack-example:
+  smslabs:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2
@@ -77,50 +77,44 @@ jobs:
           TF_VAR_cluster_name: ci${{ github.run_id }}
         if: ${{ always() && steps.provision.outcome == 'failure' && contains('not enough hosts available', steps.provision_failure.messages) }}
 
-      - name: Configure infrastructure
+      - name: Directly configure cluster and build compute + login images
         run: |
           . venv/bin/activate
           . environments/smslabs-example/activate
           ansible all -m wait_for_connection
           ansible-playbook ansible/adhoc/generate-passwords.yml
           ansible-playbook -vv ansible/site.yml
         env:
+          OS_CLOUD: openstack # required so openhpc_slurm_partitions filter used by stackhpc.slurm_openstack_tools.autoscale can use clouds.yaml file to run openstack cli to get node config
           ANSIBLE_FORCE_COLOR: True
-
-      - name: Run MPI-based tests
+          
+      - name: Test reimage of login and compute nodes
         run: |
           . venv/bin/activate
           . environments/smslabs-example/activate
-          ansible-playbook -vv ansible/adhoc/hpctests.yml
-        env:
-          ANSIBLE_FORCE_COLOR: True
-
-      - name: Build control and compute images
-        run: |
-          . venv/bin/activate
-          . environments/smslabs-example/activate
-          cd packer
-          PACKER_LOG=1 PACKER_LOG_PATH=build.log packer build -var-file=$PKR_VAR_environment_root/builder.pkrvars.hcl openstack.pkr.hcl
+          ansible all -m wait_for_connection
+          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/ci/test_reimage.yml
         env:
           OS_CLOUD: openstack
-
-      - name: Reimage compute nodes via slurm and check cluster still up
+          ANSIBLE_FORCE_COLOR: True
+
+      - name: Update cloud image and reconfigure Slurm
         run: |
           . venv/bin/activate
           . environments/smslabs-example/activate
-          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/ci/reimage-compute.yml
-          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/hooks/post.yml
+          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/ci/update_cloudnode_image.yml
+          ansible-playbook -vv ansible/slurm.yml --tags openhpc --skip-tags install
         env:
+          ANSIBLE_FORCE_COLOR: True
           OS_CLOUD: openstack
 
-      - name: Reimage login nodes via openstack and check cluster still up
+      - name: Run MPI-based tests (triggers autoscaling)
         run: |
           . venv/bin/activate
           . environments/smslabs-example/activate
-          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/ci/reimage-login.yml
-          ansible-playbook -vv $APPLIANCES_ENVIRONMENT_ROOT/hooks/post.yml
+          ansible-playbook -vv ansible/adhoc/hpctests.yml
         env:
-          OS_CLOUD: openstack
+          ANSIBLE_FORCE_COLOR: True
 
       - name: Delete infrastructure
         run: |
@@ -132,3 +126,5 @@ jobs:
           OS_CLOUD: openstack
           TF_VAR_cluster_name: ci${{ github.run_id }}
         if: ${{ success() || cancelled() }}
+
+# TODO: delete images!

README.md

@@ -98,9 +98,17 @@ NB: This section describes generic instructions - check for any environment-spec
 
         source environments/<environment>/activate
 
-2. Deploy instances - see environment-specific instructions.
+1. Activate your OpenStack credentials (required if Slurm-controlled rebuild or Slurm autoscaling is enabled):
 
-3. Generate passwords:
+        # either source an openrc.sh file
+        source path_to/openrc.sh
+
+        # or if using a clouds.yaml file in ~/.config/openstack/clouds.yaml:
+        export OS_CLOUD=openstack
+
+1. Deploy instances - see environment-specific instructions.
+
+1. Generate passwords:
 
         ansible-playbook ansible/adhoc/generate-passwords.yml
 
@@ -110,7 +118,7 @@ NB: This section describes generic instructions - check for any environment-spec
 
     See the [Ansible vault documentation](https://docs.ansible.com/ansible/latest/user_guide/vault.html) for more details.
 
-4. Deploy the appliance:
+1. Deploy the appliance:
 
         ansible-playbook ansible/site.yml
 
@@ -120,7 +128,7 @@ NB: This section describes generic instructions - check for any environment-spec
 
     Tags as defined in the various sub-playbooks defined in `ansible/` may be used to only run part of the `site` tasks.
 
-5. "Utility" playbooks for managing a running appliance are contained in `ansible/adhoc` - run these by activating the environment and using:
+1. "Utility" playbooks for managing a running appliance are contained in `ansible/adhoc` - run these by activating the environment and using:
 
         ansible-playbook ansible/adhoc/<playbook name>
 

ansible/adhoc/restart-slurm.yml

@@ -20,7 +20,7 @@
         name: slurmctld
         state: restarted
 
-- hosts: compute,login
+- hosts: compute,login # FIXME: doesn't work if using `login` as combined slurmctld
   become: yes
   gather_facts: no
   tasks:

ansible/slurm.yml

@@ -9,24 +9,54 @@
     - include_role:
         name:  geerlingguy.mysql
 
-- name: Setup slurm
+- name: Setup Slurm-driven reimage on OpenStack
+  hosts: rebuild
+  become: yes
+  tags:
+    - rebuild
+    - openhpc
+  tasks:
+    - import_role:
+        name: stackhpc.slurm_openstack_tools.rebuild
+
+- name: Preinstall Slurm packages to create slurm user
+  # This is an optimisation for speed as it avoids having to do this once for `control` then again for `openhpc` nodes.
   hosts: openhpc
   become: yes
   tags:
+    - openstack_autoscale
     - openhpc
+    - install
   tasks:
     - import_role:
         name: stackhpc.openhpc
+        tasks_from: install.yml
+      when: groups.get('openstack_autoscale', []) | length > 0
 
-- name: Setup slurm-driven reimage
-  hosts: rebuild
+- name: Setup autoscaling on OpenStack
+  hosts: openstack_autoscale
   become: yes
   tags:
-    - rebuild
+    - openstack_autoscale
     - openhpc
   tasks:
     - import_role:
-        name: stackhpc.slurm_openstack_tools.rebuild
+        name: stackhpc.slurm_openstack_tools.autoscale
+
+- name: Setup slurm
+  hosts: openhpc
+  become: yes
+  tags:
+    - openhpc
+  tasks:
+    - assert:
+        that: "'enable_configless' in openhpc_config.SlurmctldParameters | default([])"
+        fail_msg: |
+          'enable_configless' not found in openhpc_config.SlurmctldParameters - is variable openhpc_config overridden?
+          Additional slurm.conf parameters should be provided using variable openhpc_config_extra.
+        success_msg: Checked Slurm will be configured for configless operation
+    - import_role:
+        name: stackhpc.openhpc
 
 - name: Set locked memory limits on user-facing nodes
   hosts:

environments/common/inventory/group_vars/all/autoscale.yml

@@ -0,0 +1,4 @@
+autoscale_rebuild_clouds: ~/.config/openstack/clouds.yaml
+autoscale_suspend_exc_nodes_default: "{{ (groups.get('compute', []) + groups.get('login', [])) }}" # i.e. all non-CLOUD nodes, and prevent login-only slurmd nodes getting powered down
+autoscale_suspend_exc_nodes_extra: []
+autoscale_suspend_exc_nodes: "{{ autoscale_suspend_exc_nodes_default + autoscale_suspend_exc_nodes_extra }}"

environments/common/inventory/group_vars/all/openhpc.yml

@@ -22,5 +22,10 @@ openhpc_packages_default:
 openhpc_packages_extra: []
 openhpc_packages: "{{ openhpc_packages_default + openhpc_packages_extra }}"
 openhpc_munge_key: "{{ vault_openhpc_mungekey | b64decode }}"
-openhpc_slurm_configless: true
-openhpc_login_only_nodes: login
+openhpc_login_only_nodes: login
+openhpc_config_default:
+  SlurmctldParameters:
+    - enable_configless
+openhpc_config_extra: {}
+openhpc_config: "{{ openhpc_config_default | combine(openhpc_config_extra, list_merge='append') }}"
+openhpc_ram_multiplier: 0.90 # TODO: DOCS: needs to be available to stackhpc.slurm_openstack_tools.autoscale role, plus lowered a bit to cope with autoscale problems

environments/common/inventory/group_vars/all/pytools.yml

@@ -0,0 +1,2 @@
+# pytools_editable: false
+pytools_gitref: feature/ports

environments/common/inventory/group_vars/all/rebuild.yml

@@ -0,0 +1 @@
+openhpc_rebuild_clouds: ~/.config/openstack/clouds.yaml

environments/common/inventory/groups

@@ -75,6 +75,10 @@ cluster
 [update]
 # All hosts to (optionally) run yum update on.
 
+[autoscale]
+# Add control to enable autoscaling on OpenStack.
+# See ansible/collections/ansible_collections/stackhpc/slurm_openstack_tools/roles/autoscale/README.md
+
 [block_devices]
 # Superset of hosts to configure filesystems on - see ansible/roles/block_devices/README.md
 

environments/common/layouts/everything

@@ -41,3 +41,6 @@ cluster
 
 [basic_users]
 # Add `openhpc` group to add Slurm users via creation of users on each node.
+
+[openstack_autoscale]
+# Add `control` group to configure autoscaling on OpenStack clouds.

environments/smslabs-example/ci/reimage-compute.yml

@@ -14,6 +14,13 @@
       set_fact:
         compute_build: "{{ manifest['builds'] | selectattr('custom_data', 'eq', {'source': 'compute'}) | last }}"
 
+    - name: Add compute image ID to autoscale definition
+      copy:
+        dest: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/group_vars/openhpc/autoscale.yml"
+        content: |
+          openhpc_autoscale_image: {{ compute_build.artifact_id }}
+      delegate_to: localhost
+
     - name: Request compute node rebuild via Slurm
       shell:
         cmd: scontrol reboot ASAP nextstate=RESUME reason='rebuild image:{{ compute_build.artifact_id }}' {{ openhpc_cluster_name }}-compute-[0-1]

environments/smslabs-example/ci/test_reimage.yml

@@ -0,0 +1,64 @@
+- hosts: login:!builder
+  become: no
+  tasks:
+    - name: Read packer build manifest
+      set_fact:
+        manifest: "{{ lookup('file', manifest_path) | from_json }}"
+      vars:
+        manifest_path: "{{ lookup('env', 'APPLIANCES_REPO_ROOT') }}/packer/packer-manifest.json"
+      delegate_to: localhost
+
+    - name: Get latest image builds
+      set_fact:
+        login_build: "{{ manifest['builds'] | selectattr('custom_data', 'eq', {'source': 'login'}) | last }}"
+        compute_build: "{{ manifest['builds'] | selectattr('custom_data', 'eq', {'source': 'compute'}) | last }}"
+
+    - name: Reimage login node via openstack
+      shell:
+        cmd: "openstack server rebuild {{ instance_id | default(inventory_hostname) }} --image {{ login_build.artifact_id }}"
+      delegate_to: localhost
+
+    - name: Check login node rebuild completed
+      shell:
+        cmd: openstack server show {{ inventory_hostname }} --format value -c image
+      register: openstack_login
+      delegate_to: localhost
+      retries: 5
+      delay: 30
+      until: login_build.artifact_id in openstack_login.stdout
+      changed_when: false
+
+    - name: Wait for login connection
+      wait_for_connection:
+        timeout: 800
+
+    - name: Check slurm up after reimaging login node
+      import_tasks: ../hooks/check_slurm.yml
+
+    - name: Request compute node rebuild via Slurm
+      shell:
+        cmd: scontrol reboot ASAP nextstate=RESUME reason='rebuild image:{{ compute_build.artifact_id }}' {{ openhpc_cluster_name }}-compute-[0-1]
+      become: yes
+
+    - name: Check compute node rebuild completed
+      shell:
+        cmd: openstack server show {{ item }} --format value -c image
+      register: openstack_compute
+      delegate_to: localhost
+      loop: "{{ groups['compute'] }}"
+      retries: 5
+      delay: 30
+      until: compute_build.artifact_id in openstack_compute.stdout
+      changed_when: false
+
+- hosts: compute:!builder
+  become: no
+  gather_facts: no
+  tasks:
+    - name: Wait for compute connection
+      wait_for_connection:
+        timeout: 800
+
+    - name: Check slurm up after reimaging login node
+      import_tasks: ../hooks/check_slurm.yml
+      run_once: true

environments/smslabs-example/ci/update_cloudnode_image.yml

@@ -0,0 +1,22 @@
+- hosts: localhost
+  become: no
+  tasks:
+    - name: Read packer build manifest
+      set_fact:
+        manifest: "{{ lookup('file', manifest_path) | from_json }}"
+      vars:
+        manifest_path: "{{ lookup('env', 'APPLIANCES_REPO_ROOT') }}/packer/packer-manifest.json"
+      delegate_to: localhost
+
+    - name: Get latest image builds
+      set_fact:
+        login_build: "{{ manifest['builds'] | selectattr('custom_data', 'eq', {'source': 'login'}) | last }}"
+        compute_build: "{{ manifest['builds'] | selectattr('custom_data', 'eq', {'source': 'compute'}) | last }}"
+
+    - name: Add compute image ID to autoscale definition (for later autoscaling tests)
+      copy:
+        dest: "{{ lookup('env', 'APPLIANCES_ENVIRONMENT_ROOT') }}/inventory/group_vars/openhpc/autoscale.yml"
+        content: |
+          openhpc_autoscale_image: {{ compute_build.artifact_id }}
+      delegate_to: localhost
+      run_once: true

environments/smslabs-example/hooks/check_slurm.yml

@@ -0,0 +1,21 @@
+- name: Run sinfo
+  shell: 'sinfo --noheader --format="%N %P %a %l %D %t" | sort' # using --format ensures we control whitespace: Partition,partition_state,max_jobtime,num_nodes,node_state,node_name
+  register: sinfo
+  changed_when: false
+  until: "'boot' not in sinfo.stdout_lines"
+  retries: 5
+  delay: 10
+- name: Check nodes have expected slurm state
+  assert:
+    that: sinfo.stdout_lines == expected_sinfo
+    fail_msg: |
+      sinfo output not as expected:
+      actual:
+      {{ sinfo.stdout_lines }}
+      expected:
+      {{ expected_sinfo }}
+      <end>
+  vars:
+    expected_sinfo:
+      - "{{ openhpc_cluster_name }}-compute-[0-1] {{ openhpc_slurm_partitions[0].name }}* up 60-00:00:00 2 idle"
+      - "{{ openhpc_cluster_name }}-compute-[2-3] {{ openhpc_slurm_partitions[0].name }}* up 60-00:00:00 2 idle~"

environments/smslabs-example/hooks/post.yml

@@ -4,11 +4,22 @@
   tasks:
     - block:
       - name: Run sinfo
-        shell: 'sinfo --noheader --format="%N %P %a %l %D %t"' # using --format ensures we control whitespace: Partition,partition_state,max_jobtime,num_nodes,node_state,node_name
+        shell: 'sinfo --noheader --format="%N %P %a %l %D %t" | sort' # using --format ensures we control whitespace: Partition,partition_state,max_jobtime,num_nodes,node_state,node_name
         register: sinfo
         changed_when: false
       - name: Check nodes have expected slurm state
         assert:
-          that: "(sinfo.stdout_lines[0] | split)[1:] == ['small*', 'up', '60-00:00:00', '2', 'idle']" # don't know what instance names are as have CI run ID in them
-          fail_msg: "sinfo output not as expected: {{ sinfo.stdout }}"
+          that: sinfo.stdout_lines == expected_sinfo
+          fail_msg: |
+            sinfo output not as expected:
+            actual:
+            {{ sinfo.stdout_lines }}
+            expected:
+            {{ expected_sinfo }}
+            <end>
+        vars:
+          expected_sinfo:
+            - "{{ openhpc_cluster_name }}-compute-[0-1] small* up 60-00:00:00 2 idle"
+            - "{{ openhpc_cluster_name }}-compute-[2-3] small* up 60-00:00:00 2 idle~"
+
       when: "'builder' not in group_names" # won't have a slurm control daemon when in build