Skip to content

Enable linting #732

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
22 changes: 22 additions & 0 deletions .ansible-lint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
skip_list:
- role-name
# Unresolved issues with parsing jinja in multiline strings
# https://github.com/ansible/ansible-lint/issues/3935
- jinja[spacing]
- galaxy[no-changelog]
- meta-runtime[unsupported-version]

warn_list:
- name[missing]
- name[play]
- var-naming

exclude_paths:
- actionlint.yml
- .ansible/
- .github/
# The following are files with syntax errors.
# Rule 'syntax-check' is unskippable, you cannot use it in 'skip_list' or 'warn_list'. Still, you could exclude the file.
- ansible/roles/filebeat/tasks/runtime.yml
- environments/common/files/filebeat/filebeat.yml
4 changes: 4 additions & 0 deletions .checkov.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
skip-check:
# Requires all blocks to have rescue: - not considered appropriate
- CKV2_ANSIBLE_3
8 changes: 8 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# The is primarily used to alter the behaviour of linters executed by super-linter.
# See https://editorconfig.org/

# shfmt will default to indenting shell scripts with tabs,
# define the indent as 2 spaces
[{.github/bin,dev}/*.sh]
indent_style = space
indent_size = 2
10 changes: 5 additions & 5 deletions .github/bin/create-merge-branch.sh
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ if git show-branch "remotes/origin/$BRANCH_NAME" >/dev/null 2>&1; then
fi

echo "[INFO] Merging release tag - $RELEASE_TAG"
git merge --strategy recursive -X theirs --no-commit $RELEASE_TAG
git merge --strategy recursive -X theirs --no-commit "$RELEASE_TAG"

# Check if the merge resulted in any changes being staged
if [ -n "$(git status --short)" ]; then
Expand All @@ -54,7 +54,7 @@ if [ -n "$(git status --short)" ]; then
# NOTE(scott): The GitHub create-pull-request action does
# the commiting for us, so we only need to make branches
# and commits if running outside of GitHub actions.
if [ ! $GITHUB_ACTIONS ]; then
if [ ! "$GITHUB_ACTIONS" ]; then
echo "[INFO] Checking out temporary branch '$BRANCH_NAME'..."
git checkout -b "$BRANCH_NAME"

Expand All @@ -74,8 +74,8 @@ if [ -n "$(git status --short)" ]; then

# Write a file containing the branch name and tag
# for automatic PR or MR creation that follows
echo "BRANCH_NAME=\"$BRANCH_NAME\"" > .mergeenv
echo "RELEASE_TAG=\"$RELEASE_TAG\"" >> .mergeenv
echo "BRANCH_NAME=\"$BRANCH_NAME\"" >.mergeenv
echo "RELEASE_TAG=\"$RELEASE_TAG\"" >>.mergeenv
else
echo "[INFO] Merge resulted in no changes"
fi
fi
14 changes: 7 additions & 7 deletions .github/bin/get-s3-image.sh
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ echo "Checking if image $image_name exists in OpenStack"
image_exists=$(openstack image list --name "$image_name" -f value -c Name)

if [ -n "$image_exists" ]; then
echo "Image $image_name already exists in OpenStack."
echo "Image $image_name already exists in OpenStack."
else
echo "Image $image_name not found in OpenStack. Getting it from S3."
echo "Image $image_name not found in OpenStack. Getting it from S3."

wget https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_3a06571936a0424bb40bc5c672c4ccb1/$bucket_name/$image_name --progress=dot:giga
wget "https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_3a06571936a0424bb40bc5c672c4ccb1/$bucket_name/$image_name --progress=dot:giga"

echo "Uploading image $image_name to OpenStack..."
openstack image create --file $image_name --disk-format qcow2 $image_name --progress
echo "Uploading image $image_name to OpenStack..."
openstack image create --file "$image_name" --disk-format qcow2 "$image_name" --progress

echo "Image $image_name has been uploaded to OpenStack."
fi
echo "Image $image_name has been uploaded to OpenStack."
fi
1 change: 1 addition & 0 deletions .github/linters/.checkov.yaml
1 change: 1 addition & 0 deletions .github/linters/.python-lint
1 change: 1 addition & 0 deletions .github/linters/.shellcheckrc
1 change: 1 addition & 0 deletions .github/linters/.yamllint.yml
1 change: 1 addition & 0 deletions .github/linters/actionlint.yml
37 changes: 14 additions & 23 deletions .github/workflows/extra.yml
Original file line number Diff line number Diff line change
@@ -1,22 +1,13 @@
---
name: Test extra build
on:
workflow_dispatch:
push:
branches:
- main
paths:
- 'environments/.stackhpc/tofu/cluster_image.auto.tfvars.json'
- 'ansible/roles/doca/**'
- 'ansible/roles/cuda/**'
- 'ansible/roles/lustre/**'
- '.github/workflows/extra.yml'
pull_request:
paths:
- 'environments/.stackhpc/tofu/cluster_image.auto.tfvars.json'
- 'ansible/roles/doca/**'
- 'ansible/roles/cuda/**'
- 'ansible/roles/lustre/**'
- '.github/workflows/extra.yml'

permissions:
contents: read
packages: write
# To report GitHub Actions status checks
statuses: write

jobs:
doca:
Expand Down Expand Up @@ -44,7 +35,7 @@ jobs:
ARK_PASSWORD: ${{ secrets.ARK_PASSWORD }}

steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4

- name: Load current fat images into GITHUB_ENV
# see https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#example-of-a-multiline-string
Expand All @@ -58,7 +49,7 @@ jobs:
- name: Record settings
run: |
echo CI_CLOUD: ${{ env.CI_CLOUD }}
echo FAT_IMAGES: ${FAT_IMAGES}
echo "FAT_IMAGES: ${FAT_IMAGES}"

- name: Setup ssh
run: |
Expand Down Expand Up @@ -97,7 +88,7 @@ jobs:

PACKER_LOG=1 packer build \
-on-error=${{ vars.PACKER_ON_ERROR }} \
-var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
-var-file="$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl" \
-var "source_image_name=${{ fromJSON(env.FAT_IMAGES)['cluster_image'][matrix.build.source_image_name_key] }}" \
-var "image_name=${{ matrix.build.image_name }}" \
-var "inventory_groups=${{ matrix.build.inventory_groups }}" \
Expand All @@ -109,14 +100,14 @@ jobs:
run: |
. venv/bin/activate
IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
while ! openstack image show -f value -c name $IMAGE_ID; do
while ! openstack image show -f value -c name "$IMAGE_ID"; do
sleep 5
done
IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
IMAGE_NAME=$(openstack image show -f value -c name "$IMAGE_ID")
echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
echo $IMAGE_ID > image-id.txt
echo $IMAGE_NAME > image-name.txt
echo "$IMAGE_ID" > image-id.txt
echo "$IMAGE_NAME" > image-name.txt

- name: Make image usable for further builds
run: |
Expand Down
19 changes: 13 additions & 6 deletions .github/workflows/fatimage.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
name: Build fat image
on:
workflow_dispatch:
# checkov:skip=CKV_GHA_7: "The build output cannot be affected by user parameters other than the build entry point and the top-level source location. GitHub Actions workflow_dispatch inputs MUST be empty. "
inputs:
ci_cloud:
description: 'Select the CI_CLOUD'
Expand All @@ -16,6 +17,12 @@ on:
required: true
default: true

permissions:
contents: read
packages: write
# To report GitHub Actions status checks
statuses: write

jobs:
openstack:
name: openstack-imagebuild
Expand All @@ -41,7 +48,7 @@ jobs:
LEAFCLOUD_PULP_PASSWORD: ${{ secrets.LEAFCLOUD_PULP_PASSWORD }}

steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v4

- name: Record settings for CI cloud
run: |
Expand Down Expand Up @@ -84,7 +91,7 @@ jobs:

PACKER_LOG=1 packer build \
-on-error=${{ github.event.inputs.cleanup_on_failure && 'cleanup' || 'abort' }} \
-var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
-var-file="$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl" \
-var "source_image_name=${{ matrix.build.source_image_name }}" \
-var "image_name=${{ matrix.build.image_name }}" \
-var "inventory_groups=${{ matrix.build.inventory_groups }}" \
Expand All @@ -95,14 +102,14 @@ jobs:
run: |
. venv/bin/activate
IMAGE_ID=$(jq --raw-output '.builds[-1].artifact_id' packer/packer-manifest.json)
while ! openstack image show -f value -c name $IMAGE_ID; do
while ! openstack image show -f value -c name "$IMAGE_ID"; do
sleep 5
done
IMAGE_NAME=$(openstack image show -f value -c name $IMAGE_ID)
IMAGE_NAME=$(openstack image show -f value -c name "$IMAGE_ID")
echo "image-name=${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
echo "image-id=$IMAGE_ID" >> "$GITHUB_OUTPUT"
echo $IMAGE_ID > image-id.txt
echo $IMAGE_NAME > image-name.txt
echo "$IMAGE_ID" > image-id.txt
echo "$IMAGE_NAME" > image-name.txt

- name: Make image usable for further builds
run: |
Expand Down
47 changes: 47 additions & 0 deletions .github/workflows/lint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
---
name: Lint

on: # yamllint disable-line rule:truthy
workflow_call:

permissions:
contents: read
packages: read
# To report GitHub Actions status checks
statuses: write

jobs:
lint:
name: Lint
runs-on: ubuntu-latest
permissions:
contents: read
packages: read
# To report GitHub Actions status checks
statuses: write

steps:
- uses: actions/checkout@v4
with:
# super-linter needs the full git history to get the
# list of files that changed across commits
fetch-depth: 0
submodules: true

- name: Run ansible-lint
uses: ansible/[email protected]

- name: Load super-linter configuration
# Use grep inverse matching to exclude eventual comments in the .env file
# because the GitHub Actions command to set environment variables doesn't
# support comments.
# yamllint disable-line rule:line-length
# Ref: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-environment-variable
run: grep -v '^#' super-linter.env >> "$GITHUB_ENV"
if: always()

- name: Run super-linter
uses: super-linter/[email protected]
if: always()
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
105 changes: 105 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
---
name: Test on push and pull request

permissions:
actions: write
contents: read
packages: write
# To report GitHub Actions status checks
statuses: write
id-token: write

on:
push:
branches:
- main
pull_request:

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build.image_name }} # to branch/PR + OS
cancel-in-progress: true

jobs:
lint:
name: Lint
uses: ./.github/workflows/lint.yml

files_changed:
name: Determine files changed
needs: lint
# continue-on-error: true # Uncomment once integration is finished
runs-on: ubuntu-latest
# Map a step output to a job output
outputs:
should_skip: ${{ steps.skip_check.outputs.should_skip }}
paths_result: ${{ steps.skip_check.outputs.paths_result }}
steps:
- id: skip_check
# For security we use the commit of fkirc/skip-duplicate-actions@v5
uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf
with:
cancel_others: true
paths_filter: |
extra:
paths:
- 'environments/.stackhpc/tofu/cluster_image.auto.tfvars.json'
- 'ansible/roles/doca/**'
- 'ansible/roles/cuda/**'
- 'ansible/roles/lustre/**'
- '.github/workflows/extra.yml'
stackhpc:
paths:
- '**'
- '!dev/**'
- 'dev/setup-env.sh'
- '!docs/**'
- '!README.md'
- '!.gitignore'
- '!.github/workflows/'
- '.github/workflows/stackhpc'
trivvyscan:
paths:
- 'environments/.stackhpc/tofu/cluster_image.auto.tfvars.json'

extra:
name: Test extra build
needs: files_changed
if: needs.files_changed.outputs.should_skip != 'true' &&
!fromJSON(needs.files_changed.outputs.paths_result).extra.should_skip
#uses: ./.github/workflows/extra.yml
steps: # TEST
- name: Test extra build...
uses: jakejarvis/wait-action@master
with:
time: '120s'
runs-on: ubuntu-latest # TEST
#secrets: inherit

stackhpc:
name: Test deployment and reimage on OpenStack
needs: files_changed
if: needs.files_changed.outputs.should_skip != 'true' &&
!fromJSON(needs.files_changed.outputs.paths_result).stackhpc.should_skip
#uses: ./.github/workflows/stackhpc.yml
steps: # TEST
- name: Test deployment and reimage on OpenStack...
uses: jakejarvis/wait-action@master
with:
time: '120s'
runs-on: ubuntu-latest # TEST
#secrets: inherit

trivvyscan:
name: Trivy scan image for vulnerabilities
needs: files_changed
if: github.event_name == 'pull_request' &&
needs.files_changed.outputs.should_skip != 'true' &&
!fromJSON(needs.files_changed.outputs.paths_result).trivvyscan.should_skip
#uses: ./.github/workflows/trivvyscan.yml
steps: # TEST
- name: Trivy scan image for vulnerabilities...
uses: jakejarvis/wait-action@master
with:
time: '120s'
runs-on: ubuntu-latest # TEST
#secrets: inherit
Loading