Enable linting

.ansible-lint.yml

@@ -0,0 +1,22 @@
+---
+skip_list:
+  - role-name
+  # Unresolved issues with parsing jinja in multiline strings
+  # https://github.com/ansible/ansible-lint/issues/3935
+  - jinja[spacing]
+  - galaxy[no-changelog]
+  - meta-runtime[unsupported-version]
+
+warn_list:
+  - name[missing]
+  - name[play]
+  - var-naming
+
+exclude_paths:
+  - actionlint.yml
+  - .ansible/
+  - .github/
+  # The following are files with syntax errors.
+  # Rule 'syntax-check' is unskippable, you cannot use it in 'skip_list' or 'warn_list'. Still, you could exclude the file.
+  - ansible/roles/filebeat/tasks/runtime.yml
+  - environments/common/files/filebeat/filebeat.yml

.checkov.yaml

@@ -0,0 +1,4 @@
+---
+skip-check:
+  # Requires all blocks to have rescue: - not considered appropriate
+  - CKV2_ANSIBLE_3

.editorconfig

@@ -0,0 +1,8 @@
+# The is primarily used to alter the behaviour of linters executed by super-linter.
+# See https://editorconfig.org/
+
+# shfmt will default to indenting shell scripts with tabs,
+# define the indent as 2 spaces
+[{.github/bin,dev}/*.sh]
+indent_style = space
+indent_size = 2

.github/bin/create-merge-branch.sh

@@ -44,7 +44,7 @@ if git show-branch "remotes/origin/$BRANCH_NAME" >/dev/null 2>&1; then
 fi
 
 echo "[INFO] Merging release tag - $RELEASE_TAG"
-git merge --strategy recursive -X theirs --no-commit $RELEASE_TAG
+git merge --strategy recursive -X theirs --no-commit "$RELEASE_TAG"
 
 # Check if the merge resulted in any changes being staged
 if [ -n "$(git status --short)" ]; then
@@ -54,7 +54,7 @@ if [ -n "$(git status --short)" ]; then
   # NOTE(scott): The GitHub create-pull-request action does
   # the commiting for us, so we only need to make branches
   # and commits if running outside of GitHub actions.
-  if [ ! $GITHUB_ACTIONS ]; then
+  if [ ! "$GITHUB_ACTIONS" ]; then
     echo "[INFO] Checking out temporary branch '$BRANCH_NAME'..."
     git checkout -b "$BRANCH_NAME"
 
@@ -74,8 +74,8 @@ if [ -n "$(git status --short)" ]; then
 
   # Write a file containing the branch name and tag
   # for automatic PR or MR creation that follows
-  echo "BRANCH_NAME=\"$BRANCH_NAME\"" > .mergeenv
-  echo "RELEASE_TAG=\"$RELEASE_TAG\"" >> .mergeenv
+  echo "BRANCH_NAME=\"$BRANCH_NAME\"" >.mergeenv
+  echo "RELEASE_TAG=\"$RELEASE_TAG\"" >>.mergeenv
 else
   echo "[INFO] Merge resulted in no changes"
-fi
+fi

.github/bin/get-s3-image.sh

@@ -13,14 +13,14 @@ echo "Checking if image $image_name exists in OpenStack"
 image_exists=$(openstack image list --name "$image_name" -f value -c Name)
 
 if [ -n "$image_exists" ]; then
-    echo "Image $image_name already exists in OpenStack."
+  echo "Image $image_name already exists in OpenStack."
 else
-    echo "Image $image_name not found in OpenStack. Getting it from S3."
+  echo "Image $image_name not found in OpenStack. Getting it from S3."
 
-    wget https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_3a06571936a0424bb40bc5c672c4ccb1/$bucket_name/$image_name --progress=dot:giga
+  wget "https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_3a06571936a0424bb40bc5c672c4ccb1/$bucket_name/$image_name --progress=dot:giga"
 
-    echo "Uploading image $image_name to OpenStack..."
-    openstack image create --file $image_name --disk-format qcow2 $image_name --progress
+  echo "Uploading image $image_name to OpenStack..."
+  openstack image create --file "$image_name" --disk-format qcow2 "$image_name" --progress
 
-    echo "Image $image_name has been uploaded to OpenStack."
-fi
+  echo "Image $image_name has been uploaded to OpenStack."
+fi

.github/linters/.checkov.yaml

@@ -0,0 +1 @@
+../../.checkov.yaml

.github/linters/.python-lint

@@ -0,0 +1 @@
+../../.python-lint

.github/linters/.shellcheckrc

@@ -0,0 +1 @@
+../../.shellcheckrc

.github/linters/.yamllint.yml

@@ -0,0 +1 @@
+../../.yamllint.yml

.github/linters/actionlint.yml

@@ -0,0 +1 @@
+../../actionlint.yml

.github/workflows/extra.yml

@@ -1,22 +1,13 @@
+---
 name: Test extra build
 on:
   workflow_dispatch:
-  push:
-    branches:
-      - main
-    paths:
-      - 'environments/.stackhpc/tofu/cluster_image.auto.tfvars.json'
-      - 'ansible/roles/doca/**'
-      - 'ansible/roles/cuda/**'
-      - 'ansible/roles/lustre/**'
-      - '.github/workflows/extra.yml'
-  pull_request:
-    paths:
-      - 'environments/.stackhpc/tofu/cluster_image.auto.tfvars.json'
-      - 'ansible/roles/doca/**'
-      - 'ansible/roles/cuda/**'
-      - 'ansible/roles/lustre/**'
-      - '.github/workflows/extra.yml'
+
+permissions:
+  contents: read
+  packages: write
+  # To report GitHub Actions status checks
+  statuses: write
 
 jobs:
   doca:
@@ -44,7 +35,7 @@ jobs:
       ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Load current fat images into GITHUB_ENV
         # see https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#example-of-a-multiline-string
@@ -58,7 +49,7 @@ jobs:
       - name: Record settings
         run: |
           echo CI_CLOUD: ${{ env.CI_CLOUD }}
-          echo FAT_IMAGES: ${FAT_IMAGES}
+          echo "FAT_IMAGES: ${FAT_IMAGES}"
 
       - name: Setup ssh
         run: |
@@ -97,7 +88,7 @@ jobs:
 
           PACKER_LOG=1 packer build \
           -on-error=${{ vars.PACKER_ON_ERROR }} \
-          -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
+          -var-file="$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl" \
           -var "source_image_name=${{ fromJSON(env.FAT_IMAGES)['cluster_image'][matrix.build.source_image_name_key] }}" \
           -var "image_name=${{ matrix.build.image_name }}" \
           -var "inventory_groups=${{ matrix.build.inventory_groups }}" \
@@ -109,14 +100,14 @@ jobs:
         run: |
           . venv/bin/activate
           IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
-          while ! openstack image show -f value -c name $IMAGE_ID; do
+          while ! openstack image show -f value -c name "$IMAGE_ID"; do
             sleep 5
           done
-          IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
+          IMAGE_NAME=$(openstack image show -f value -c name "$IMAGE_ID")
           echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
           echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
-          echo $IMAGE_ID > image-id.txt
-          echo $IMAGE_NAME > image-name.txt
+          echo "$IMAGE_ID" > image-id.txt
+          echo "$IMAGE_NAME" > image-name.txt
 
       - name: Make image usable for further builds
         run: |

.github/workflows/fatimage.yml

@@ -1,6 +1,7 @@
 name: Build fat image
 on:
   workflow_dispatch:
+      # checkov:skip=CKV_GHA_7: "The build output cannot be affected by user parameters other than the build entry point and the top-level source location. GitHub Actions workflow_dispatch inputs MUST be empty. "
       inputs:
         ci_cloud:
           description: 'Select the CI_CLOUD'
@@ -16,6 +17,12 @@ on:
           required: true
           default: true
 
+permissions:
+  contents: read
+  packages: write
+  # To report GitHub Actions status checks
+  statuses: write
+
 jobs:
   openstack:
     name: openstack-imagebuild
@@ -41,7 +48,7 @@ jobs:
       LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Record settings for CI cloud
         run: |
@@ -84,7 +91,7 @@ jobs:
 
           PACKER_LOG=1 packer build \
           -on-error=${{ github.event.inputs.cleanup_on_failure && 'cleanup' || 'abort' }} \
-          -var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
+          -var-file="$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl" \
           -var "source_image_name=${{ matrix.build.source_image_name }}" \
           -var "image_name=${{ matrix.build.image_name }}" \
           -var "inventory_groups=${{ matrix.build.inventory_groups }}" \
@@ -95,14 +102,14 @@ jobs:
         run: |
           . venv/bin/activate
           IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
-          while ! openstack image show -f value -c name $IMAGE_ID; do
+          while ! openstack image show -f value -c name "$IMAGE_ID"; do
             sleep 5
           done
-          IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
+          IMAGE_NAME=$(openstack image show -f value -c name "$IMAGE_ID")
           echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
           echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
-          echo $IMAGE_ID > image-id.txt
-          echo $IMAGE_NAME > image-name.txt
+          echo "$IMAGE_ID" > image-id.txt
+          echo "$IMAGE_NAME" > image-name.txt
 
       - name: Make image usable for further builds
         run: |

.github/workflows/lint.yml

@@ -0,0 +1,47 @@
+---
+name: Lint
+
+on:  # yamllint disable-line rule:truthy
+  workflow_call:
+
+permissions:
+  contents: read
+  packages: read
+  # To report GitHub Actions status checks
+  statuses: write
+
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
+      # To report GitHub Actions status checks
+      statuses: write
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # super-linter needs the full git history to get the
+          # list of files that changed across commits
+          fetch-depth: 0
+          submodules: true
+
+      - name: Run ansible-lint
+        uses: ansible/[email protected]
+
+      - name: Load super-linter configuration
+        # Use grep inverse matching to exclude eventual comments in the .env file
+        # because the GitHub Actions command to set environment variables doesn't
+        # support comments.
+        # yamllint disable-line rule:line-length
+        # Ref: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-environment-variable
+        run: grep -v '^#' super-linter.env >> "$GITHUB_ENV"
+        if: always()
+
+      - name: Run super-linter
+        uses: super-linter/[email protected]
+        if: always()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/main.yml

@@ -0,0 +1,105 @@
+---
+name: Test on push and pull request
+
+permissions:
+  actions: write
+  contents: read
+  packages: write
+  # To report GitHub Actions status checks
+  statuses: write
+  id-token: write
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build.image_name }} # to branch/PR + OS
+    cancel-in-progress: true
+
+jobs:
+  lint:
+    name: Lint
+    uses: ./.github/workflows/lint.yml
+
+  files_changed:
+    name: Determine files changed
+    needs: lint
+    # continue-on-error: true # Uncomment once integration is finished
+    runs-on: ubuntu-latest
+    # Map a step output to a job output
+    outputs:
+      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+      paths_result: ${{ steps.skip_check.outputs.paths_result }}
+    steps:
+      - id: skip_check
+        # For security we use the commit of fkirc/skip-duplicate-actions@v5
+        uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf
+        with:
+          cancel_others: true
+          paths_filter: |
+            extra:
+              paths:
+                - 'environments/.stackhpc/tofu/cluster_image.auto.tfvars.json'
+                - 'ansible/roles/doca/**'
+                - 'ansible/roles/cuda/**'
+                - 'ansible/roles/lustre/**'
+                - '.github/workflows/extra.yml'
+            stackhpc:
+              paths:
+                - '**'
+                - '!dev/**'
+                - 'dev/setup-env.sh'
+                - '!docs/**'
+                - '!README.md'
+                - '!.gitignore'
+                - '!.github/workflows/'
+                - '.github/workflows/stackhpc'
+            trivvyscan:
+              paths:
+                - 'environments/.stackhpc/tofu/cluster_image.auto.tfvars.json'
+
+  extra:
+    name: Test extra build
+    needs: files_changed
+    if: needs.files_changed.outputs.should_skip != 'true' &&
+        !fromJSON(needs.files_changed.outputs.paths_result).extra.should_skip
+    #uses: ./.github/workflows/extra.yml
+    steps: # TEST
+      - name: Test extra build...
+        uses: jakejarvis/wait-action@master
+        with:
+          time: '120s'
+    runs-on: ubuntu-latest # TEST
+    #secrets: inherit
+
+  stackhpc:
+    name: Test deployment and reimage on OpenStack
+    needs: files_changed
+    if: needs.files_changed.outputs.should_skip != 'true' &&
+        !fromJSON(needs.files_changed.outputs.paths_result).stackhpc.should_skip
+    #uses: ./.github/workflows/stackhpc.yml
+    steps: # TEST
+      - name: Test deployment and reimage on OpenStack...
+        uses: jakejarvis/wait-action@master
+        with:
+          time: '120s'
+    runs-on: ubuntu-latest # TEST
+    #secrets: inherit
+
+  trivvyscan:
+    name: Trivy scan image for vulnerabilities
+    needs: files_changed
+    if: github.event_name == 'pull_request' &&
+        needs.files_changed.outputs.should_skip != 'true' &&
+        !fromJSON(needs.files_changed.outputs.paths_result).trivvyscan.should_skip
+    #uses: ./.github/workflows/trivvyscan.yml
+    steps: # TEST
+      - name: Trivy scan image for vulnerabilities...
+        uses: jakejarvis/wait-action@master
+        with:
+          time: '120s'
+    runs-on: ubuntu-latest # TEST
+    #secrets: inherit