diff --git a/.github/workflows/extra.yml b/.github/workflows/extra.yml index 05911f28c..57f7ba40e 100644 --- a/.github/workflows/extra.yml +++ b/.github/workflows/extra.yml @@ -8,6 +8,23 @@ name: Test extra build on: workflow_call: workflow_dispatch: + # checkov:skip=CKV_GHA_7: "The build output cannot be affected by user parameters other than the build entry point and the top-level source location. GitHub Actions workflow_dispatch inputs MUST be empty. " + inputs: + ci_cloud: + description: 'Select the CI_CLOUD' + required: true + type: choice + options: + - default + - LEAFCLOUD + - SMS + - ARCUS + default: default # Use repo CI_CLOUD setting or PR label + cleanup_on_failure: + description: Cleanup Packer resources on failure + type: boolean + required: true + default: true permissions: contents: read @@ -34,9 +51,10 @@ jobs: env: ANSIBLE_FORCE_COLOR: True OS_CLOUD: openstack - CI_CLOUD: ${{ vars.CI_CLOUD }} # default from repo settings + CI_CLOUD: ${{ github.event.inputs.ci_cloud == 'default' && vars.CI_CLOUD || github.event.inputs.ci_cloud || vars.CI_CLOUD }} ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }} PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PACKER_ON_ERROR: ${{ github.event.inputs.cleanup_on_failure == 'false' && 'abort' || vars.PACKER_ON_ERROR }} steps: - uses: actions/checkout@v4 @@ -50,9 +68,24 @@ jobs: echo EOF } >> "$GITHUB_ENV" + - name: Override CI_CLOUD if PR label is present + if: ${{ github.event_name == 'pull_request' }} + run: | + # Iterate over the labels + labels=$(echo '${{ toJSON(github.event.pull_request.labels) }}' | jq -r '.[].name') + echo "$labels" + for label in $labels; do + if [[ $label == CI_CLOUD=* ]]; then + # Extract the value after 'CI_CLOUD=' + CI_CLOUD_OVERRIDE=${label#CI_CLOUD=} + echo "CI_CLOUD=${CI_CLOUD_OVERRIDE}" >> "$GITHUB_ENV" + fi + done + - name: Record settings run: | echo CI_CLOUD: ${{ env.CI_CLOUD }} + echo PACKER_ON_ERROR: ${{ env.PACKER_ON_ERROR}} echo "FAT_IMAGES: ${FAT_IMAGES}" - name: Setup ssh @@ -61,10 +94,14 @@ jobs: mkdir ~/.ssh echo "${{ secrets[format('{0}_SSH_KEY', env.CI_CLOUD)] }}" > ~/.ssh/id_rsa chmod 0600 ~/.ssh/id_rsa + ssh-keygen -f ~/.ssh/id_rsa -y # tests key format is correct shell: bash - - name: Add bastion's ssh key to known_hosts - run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts + - name: Add bastion ssh fingerprints to known_hosts + run: | + cat >> ~/.ssh/known_hosts << 'EOF' + ${{ vars.BASTION_FINGERPRINTS }} + EOF shell: bash - name: Install ansible etc @@ -91,7 +128,7 @@ jobs: packer init . PACKER_LOG=1 packer build \ - -on-error=${{ vars.PACKER_ON_ERROR }} \ + -on-error=${{ env.PACKER_ON_ERROR }} \ -var-file="$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl" \ -var "source_image_name=${{ fromJSON(env.FAT_IMAGES)['cluster_image'][matrix.build.source_image_name_key] }}" \ -var "image_name=${{ matrix.build.image_name }}" \ diff --git a/.github/workflows/fatimage.yml b/.github/workflows/fatimage.yml index 361f9955a..3407857e4 100644 --- a/.github/workflows/fatimage.yml +++ b/.github/workflows/fatimage.yml @@ -64,10 +64,14 @@ jobs: mkdir ~/.ssh echo "${{ secrets[format('{0}_SSH_KEY', env.CI_CLOUD)] }}" > ~/.ssh/id_rsa chmod 0600 ~/.ssh/id_rsa + ssh-keygen -f ~/.ssh/id_rsa -y # tests key format is correct shell: bash - - name: Add bastion's ssh key to known_hosts - run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts + - name: Add bastion ssh fingerprints to known_hosts + run: | + cat >> ~/.ssh/known_hosts << 'EOF' + ${{ vars.BASTION_FINGERPRINTS }} + EOF shell: bash - name: Install ansible etc diff --git a/.github/workflows/stackhpc.yml b/.github/workflows/stackhpc.yml index 29726442e..4f297379e 100644 --- a/.github/workflows/stackhpc.yml +++ b/.github/workflows/stackhpc.yml @@ -69,10 +69,20 @@ jobs: mkdir ~/.ssh echo "${{ secrets[format('{0}_SSH_KEY', env.CI_CLOUD)] }}" > ~/.ssh/id_rsa chmod 0600 ~/.ssh/id_rsa + ssh-keygen -f ~/.ssh/id_rsa -y # tests key format is correct shell: bash - - name: Add bastion's ssh key to known_hosts - run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts + - name: Add bastion ssh fingerprints to known_hosts + run: | + cat >> ~/.ssh/known_hosts << 'EOF' + ${{ vars.BASTION_FINGERPRINTS }} + EOF + shell: bash + + - name: Ensure Ansible bastion definitions are from **current** branch + run: | + git fetch origin ${{ github.head_ref || github.ref_name }} + git checkout origin/${{ github.head_ref || github.ref_name }} -- environments/.stackhpc/inventory/group_vars/all/bastion.yml shell: bash - uses: actions/setup-python@v6 @@ -174,7 +184,12 @@ jobs: . environments/.stackhpc/activate cd "$STACKHPC_TF_DIR" tofu init - tofu apply -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars" + max_retries=3 + delay=30 + for i in $(seq 1 $max_retries); do + tofu apply -auto-approve -var-file="${{ env.CI_CLOUD }}.tfvars" && break + [ "$i" -lt "$max_retries" ] && sleep $delay || exit 1 + done - name: Configure cluster using current branch run: | diff --git a/.github/workflows/trivyscan.yml b/.github/workflows/trivyscan.yml index 2cb0602cd..a50f3b933 100644 --- a/.github/workflows/trivyscan.yml +++ b/.github/workflows/trivyscan.yml @@ -50,18 +50,6 @@ jobs: run: | echo CI_CLOUD: ${{ env.CI_CLOUD }} - - name: Setup ssh - run: | - set -x - mkdir ~/.ssh - echo "${{ secrets[format('{0}_SSH_KEY', env.CI_CLOUD)] }}" > ~/.ssh/id_rsa - chmod 0600 ~/.ssh/id_rsa - shell: bash - - - name: Add bastion's ssh key to known_hosts - run: cat environments/.stackhpc/bastion_fingerprints >> ~/.ssh/known_hosts - shell: bash - - name: setup environment run: | python3 -m venv venv diff --git a/environments/.stackhpc/LEAFCLOUD.pkrvars.hcl b/environments/.stackhpc/LEAFCLOUD.pkrvars.hcl index db0b28b49..af5a26e8a 100644 --- a/environments/.stackhpc/LEAFCLOUD.pkrvars.hcl +++ b/environments/.stackhpc/LEAFCLOUD.pkrvars.hcl @@ -6,5 +6,5 @@ ssh_private_key_file = "~/.ssh/id_rsa" security_groups = ["default", "SSH"] # see environments/.stackhpc/inventory/group_vars/all/bastion.yml: ssh_bastion_username = "slurm-app-ci" -ssh_bastion_host = "195.114.30.222" +ssh_bastion_host = "45.135.59.32" ssh_bastion_private_key_file = "~/.ssh/id_rsa" diff --git a/environments/.stackhpc/bastion_fingerprints b/environments/.stackhpc/bastion_fingerprints deleted file mode 100644 index 8596c1694..000000000 --- a/environments/.stackhpc/bastion_fingerprints +++ /dev/null @@ -1,8 +0,0 @@ -|1|BwhEZQPqvZcdf9Phmh2mTPmIivU=|bHi1Nf8dYI8z1C+qsqQFPAty1xA= ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQChxwhZggdwj55gNzfDBzah0G8IeTPQjgMZrpboxp2BO4J+o1iZSwDj+2fqyhBGTE43vCJR13uEygz49XIy+t17qBNwHz4fVVR7jdMNymtbZoOsq9oAoBdGEICHrMzQsYZmT9+Wt74ZP2PKOOn+a+f2vg7YdeSy1UhT08iJlbXwCx56fCQnMJMOnZM9MXVLd4NUFN1TeOCIBQHwRiMJyJ7S7CdUKpyUqHOG85peKiPJ07C0RZ/W5HkYKqltwtvPGQd262p5eLC9j3nhOYSG2meRV8yTxYz3lDIPDx0+189CZ5NaxFSPCgqSYA24zavhPVLQqoct7nd7fcEw9JiTs+abZC6GckCONSHDLM+iRtWC/i5u21ZZDLxM9SIqPI96cYFszGeqyZoXxS5qPaIDHbQNAEqJp9ygNXgh9vuBo7E+aWYbFDTG0RuvW02fbmFfZw2/yXIr37+cQX+GPOnkfIRuHE3Hx5eN8C04v+BMrAfK2minawhG3A2ONJs9LI6QoeE= -|1|whGSPLhKW4xt/7PWOZ1treg3PtA=|F5gwV8j0JYWDzjb6DvHHaqO+sxs= ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCpCG881Gt3dr+nuVIC2uGEQkeVwG6WDdS1WcCoxXC7AG+Oi5bfdqtf4IfeLpWmeuEaAaSFH48ODFr76ViygSjU= -|1|0V6eQ1FKO5NMKaHZeNFbw62mrJs=|H1vuGTbbtZD2MEgZxQf1PXPk+yU= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEnOtYByM3s2qvRT8SS1sn5z5sbwjzb1alm0B3emPcHJ -|1|u3QVAK9R2x7Z3uKNj+0vDEIekl0=|yy09Q0Kw472+J7bjFkmir28x3lE= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINNuXZkH7ppkTGNGKzmGEvAnvlLO2D+YtlJw1m3P16FV -|1|nOHeibGxhsIFnhW0flRwnirJjlg=|IJ8nJB355LGI+1U3Wpvdcgdf0ek= ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGG6DieKAdgiTCqRmF2HD0dJi9DuORblPzbridniICsw -185.45.78.150 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDkOPL7fQiLFrg+/mDbff+jr+mQkI8pAkS5aBKOaknKuzTGrxILO5XSbyTJxyEwIKzZHBCUH2w99yv3oCqiphYp7iLLdPKl98RRnAXneJ1mo7nJfaTOSj5FGFf/AeHFZFa18B8zZrfFOOTGdEXeQpcik6R2A0/o4ZGE9rUg/dEoLQpFp8z+XRhsbNWgZ4a63oWrt02p+zdXPZ+Plir56j0qyQXoOo/BjEoLHs0aah61jfEOcJAcgpTrev/vdhBqJCgEXkf6AhiKidTnQxw7G/5C/BKtJbtuBWMgWZKcDf/uCzRkXaHNEggcJi1e6jvpUkvPLUfpRnNiBWLzehw3xZL4NicMM6D2TU0TSpB+UfEOLR0jyhCGKRQQN4jnj8ll0h+JBE6a0KnyKG+B5mXrD7THYu848jXUmBnxIaeor/NUItKEnCL0hzvAygOnniBN6uvtszSJHoGe8WbChLYJcoH3mOQTUH0k9RhXSEe90gSlLfRQInU+uzf2/qc6pffcKuc= -185.45.78.150 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCB8R1BElOz4geGfCcb/ObF5n4Par+g9AaXQW5FU1ccgnPA59uJeOEALPeXAgJijVOhwqTdIkIoWYWeGdlud9Wc= -185.45.78.150 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINNuXZkH7ppkTGNGKzmGEvAnvlLO2D+YtlJw1m3P16FV diff --git a/environments/.stackhpc/inventory/group_vars/all/bastion.yml b/environments/.stackhpc/inventory/group_vars/all/bastion.yml index ea2ad00e5..9f75cea1c 100644 --- a/environments/.stackhpc/inventory/group_vars/all/bastion.yml +++ b/environments/.stackhpc/inventory/group_vars/all/bastion.yml @@ -4,13 +4,13 @@ bastion_config: ARCUS: user: slurm-app-ci ip: 128.232.222.183 - LEAFCLOUD: + LEAFCLOUD: # https://github.com/stackhpc/leafcloud-slurm-jumphost user: slurm-app-ci - ip: 195.114.30.222 + ip: 45.135.59.32 SMS: user: slurm-app-ci ip: 185.45.78.150 # NB: The bastion_{user,ip} variables are used directly in the CI workflow too bastion_user: "{{ bastion_config[ci_cloud].user }}" bastion_ip: "{{ bastion_config[ci_cloud].ip }}" -ansible_ssh_common_args: '-o ProxyCommand="ssh {{ bastion_user }}@{{ bastion_ip }} -W %h:%p"' +ansible_ssh_common_args: "-o ProxyCommand='ssh {{ bastion_user }}@{{ bastion_ip }} -W %h:%p'" diff --git a/packer/openstack.pkr.hcl b/packer/openstack.pkr.hcl index b6f570e22..82749ade7 100644 --- a/packer/openstack.pkr.hcl +++ b/packer/openstack.pkr.hcl @@ -94,7 +94,7 @@ variable "ssh_bastion_username" { variable "ssh_bastion_private_key_file" { type = string - default = "~/.ssh/id_rsa" + default = null } variable "floating_ip_network" {