Turso compare all images with oldest python #12
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #This workflow will install python and run the full analysator testpackage with image comparison for the results | |
| name: Turso compare all images with oldest python | |
| on: | |
| workflow_dispatch: | |
| jobs: | |
| turso_system: | |
| if: github.repository_owner == 'fmihpc' | |
| runs-on: carrington | |
| timeout-minutes: 120 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 2 | |
| matrix: | |
| extras: ["testpackage"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| timeout-minutes: 5 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| timeout-minutes: 5 | |
| - name: Get oldest allowed python version from pyproject.toml | |
| id: pyversion | |
| run: | | |
| module purge | |
| PYTHON=$(cat pyproject.toml | grep -Po '^requires-python\W*=\W*".+\K\d.+\d') | |
| #PYTHON="3.7" | |
| MAJOR=$(echo $PYTHON | grep -Po '^\d+') | |
| MINOR=$(echo $PYTHON | grep -Po '\.\K\d+([\.]|$)') | |
| AVAILABLE_VERSIONS=$(module spider Python) | |
| N=0 | |
| while [[ ! $VERSIONS_MATCH ]] | |
| do | |
| echo "Trying to find module for Python/$PYTHON" | |
| VERSIONS_MATCH=$(echo $AVAILABLE_VERSIONS | grep -Po 'Python/.+?-GCC.+' | grep -Po 'Python'/$PYTHON | head -n1) | |
| if [[ ! $VERSIONS_MATCH ]]; then | |
| MINOR=$(( $MINOR + 1 )) | |
| PYTHON=$MAJOR.$MINOR | |
| N=$(( N + 1)) | |
| if (( N> 20 )); then | |
| echo "::error:: Could not find a python version to use." | |
| exit 1 | |
| fi | |
| fi | |
| done | |
| echo "PYTHON=$PYTHON" >> $GITHUB_OUTPUT | |
| module load Python/$PYTHON | |
| module list Python | |
| - name: Install dependencies | |
| timeout-minutes: 5 | |
| run: | | |
| export TMPDIR=$RUNNER_TEMP | |
| export UV_LINK_MODE=copy | |
| module purge | |
| module load Python/${{ steps.pyversion.outputs.PYTHON }} | |
| module list | |
| uv venv CI_env | |
| . CI_env/bin/activate | |
| uv pip install --editable ../analysator[${{ matrix.extras }}] | |
| - name: Produce plots | |
| id: run_cl | |
| run: | | |
| export TMPDIR=$RUNNER_TEMP | |
| module purge | |
| module load Python/${{ steps.pyversion.outputs.PYTHON }} | |
| module list | |
| verf_loc="/wrk-kappa/group/spacephysics/analysator/CI/verification_sets" | |
| verfset=$(ls -lth $verf_loc | grep ^d | head -n1 | grep -Po '\w+$') | |
| if [[ -f $verf_loc/$verfset/.lockfile ]]; then | |
| echo -e ".lockfile found in $verf_loc/$verfset, not running test, as the verification set generation is likely still ongoing\n Check ongoing actions and/or re-run verification set generation." | |
| exit 1 | |
| fi | |
| sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh > jobid.txt | |
| export JOBID=$(grep -Po '\d+' jobid.txt) | |
| export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED) | |
| if [[ $SACCT_LOG ]]; then | |
| echo "Some job failed on a node" | |
| echo $SACCT_LOG | |
| exit 1 | |
| fi | |
| cat testpackage_run.txt | |
| . CI_env/bin/activate | |
| python ./testpackage/testpackage_get_job_error.py testpackage_run.txt | |
| - name: Comparing plotted data | |
| run: | | |
| export TMPDIR=$RUNNER_TEMP | |
| module purge | |
| module load Python/${{ steps.pyversion.outputs.PYTHON }} | |
| module list | |
| . CI_env/bin/activate | |
| sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh | |
| cat testpackage_compare.txt | |
| python ./testpackage/testpackage_get_job_error.py testpackage_compare.txt | |
| - name: scancel dangling job upon cancellation | |
| if: cancelled() | |
| run: | | |
| scancel ${{ steps.run_cl.outputs.SLURM_JOB_ID }} | |