Skip to content

Commit ccfbe0b

Browse files
lassejsclassejsc
authored
Merge pull request #431 from lassejsc/dev_ci_fix
CI fix and changes from master to dev
2 parents e5738ed + f914afb commit ccfbe0b

8 files changed

Lines changed: 80 additions & 61 deletions

‎.github/workflows/make_sbatch_job.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/usr/bin/env bash
2+
if [[ ! $ARRAY_SIZE ]]; then
3+
ARRAY_SIZE=14
4+
fi
5+
# cmd argument $1 is used for the file name of the script to run, log file, jobid
6+
7+
#if sbatch fails as it gets the error code from python it tries to print the log
8+
# first with srun (in case the file has not yet updated on the front end)
9+
# second if srun fails (in case communication failure) it tries to cat it on the frontend (may be empty if file has not been updated pyproject)
10+
11+
sbatch -W --array=1-$ARRAY_SIZE -o "$1" ./testpackage/$1.sh $2 > jobid_$1 || (srun cat $1 || cat $1 || echo "cat failed exit code $?" && exit 1)
12+
13+
#in case we do exit 0 successfully
14+
LOG=$(srun cat $1 || cat $1)
15+
if [[ ! $LOG ]]; then
16+
echo "::warning::Log file could not be read: srun failed and cat returned and empty logfile. Exit code $?"
17+
exit 1
18+
fi
19+
echo "$LOG"
20+
#It is possible that the sbatch command above returns exit 0 if only for example 1 of the array jobs failed but not all, in such a case we check sacct
21+
# It is also possible that the node never ran it and silently failed which is visible on sacct
22+
#Additionally note that JOBID is saved by the run scripts HOWEVER if a nodes silently fails it may not get passed to the github output
23+
JOBID=$(srun grep -Po '\d+' jobid_$1 || grep -Po '\d+' jobid_$1)
24+
if [[ $? -ne 0 ]] || [[ ! $JOBID ]]; then
25+
echo "::error::Jobid could not be found, exiting. Exit code $?"
26+
exit 1
27+
fi
28+
SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED)
29+
if [[ $SACCT_LOG ]]; then
30+
echo "::error::Some job failed on a node, try to take a look at the slurm log."
31+
echo "$SACCT_LOG"
32+
exit 1
33+
fi

‎.github/workflows/test_compare_image_oldest_python.yml‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -69,23 +69,15 @@ jobs:
6969
module purge
7070
module load ${{ steps.pyversion.outputs.PYTHON }}
7171
module list
72-
verf_loc="/wrk-kappa/group/spacephysics/analysator/CI/verification_sets"
72+
verf_loc="/turso/group/spacephysics/analysator/CI/verification_sets"
7373
verfset=$(ls -lth $verf_loc | grep ^d | head -n1 | grep -Po '\w+$')
7474
if [[ -f $verf_loc/$verfset/.lockfile ]]; then
7575
echo -e ".lockfile found in $verf_loc/$verfset, not running test, as the verification set generation is likely still ongoing\n Check ongoing actions and/or re-run verification set generation."
7676
exit 1
7777
fi
78-
sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh old_python ${{ steps.pyversion.outputs.PYTHON }} > jobid.txt
79-
export JOBID=$(grep -Po '\d+' jobid.txt)
80-
export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED)
81-
if [[ $SACCT_LOG ]]; then
82-
echo "Some job failed on a node"
83-
echo $SACCT_LOG
84-
exit 1
85-
fi
86-
cat testpackage_run.txt
87-
. CI_env/bin/activate
88-
python ./testpackage/testpackage_get_job_error.py testpackage_run.txt
78+
79+
.github/workflows/make_sbatch_job.sh run_testpackage_workflow old_python ${{ steps.pyversion.outputs.PYTHON }}
80+
8981
9082
- name: Comparing plotted data
9183
run: |
@@ -94,9 +86,8 @@ jobs:
9486
module load ${{ steps.pyversion.outputs.PYTHON }}
9587
module list
9688
. CI_env/bin/activate
97-
sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh old_python ${{ steps.pyversion.outputs.PYTHON }}
98-
cat testpackage_compare.txt
99-
python ./testpackage/testpackage_get_job_error.py testpackage_compare.txt
89+
.github/workflows/make_sbatch_job.sh run_compare old_python ${{ steps.pyversion.outputs.PYTHON }}
90+
10091
- name: scancel dangling job upon cancellation
10192
if: cancelled()
10293
run: |

‎.github/workflows/test_compare_images.yml‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ jobs:
4646
export TMPDIR=$RUNNER_TEMP
4747
module purge
4848
module load Python/3.10.4-GCCcore-11.3.0
49-
verf_loc="/wrk-kappa/group/spacephysics/analysator/CI/verification_sets"
49+
verf_loc="/turso/group/spacephysics/analysator/CI/verification_sets"
5050
verfset=$(ls -lth $verf_loc | grep ^d | head -n1 | grep -Po '\w+$')
5151
if [[ -f $verf_loc/$verfset/.lockfile ]]; then
5252
echo -e ".lockfile found in $verf_loc/$verfset, not running test, as the verification set generation is likely still ongoing\n Check ongoing actions and/or re-run verification set generation."
@@ -57,27 +57,27 @@ jobs:
5757
cat diff_log.txt
5858
fi
5959
echo "DIFFRESULT=$DIFFRESULT" >> $GITHUB_OUTPUT
60-
echo "Running $DIFFRESULT"
61-
sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT > jobid.txt
62-
export JOBID=$(grep -Po '\d+' jobid.txt)
63-
export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED)
64-
if [[ $SACCT_LOG ]]; then
65-
echo "Some job failed on a node"
66-
echo $SACCT_LOG
67-
exit 1
60+
MAX_ARRAY=14
61+
TOTAL_TESTS=$(ls ./testpackage/testpackage_definitions/ | wc -l)
62+
if [[ $DIFFRESULT && $DIFFRESULT != "pass" ]]; then
63+
NUM_OF_TESTS=$(echo $DIFFRESULT | sed -e 's/ /\n/g' | wc -l)
64+
ARRAY_SIZE=$(( NUM_OF_TESTS*((MAX_ARRAY+TOTAL_TESTS-1)/TOTAL_TESTS) )) #Rounding up
65+
ARRAY_SIZE=$(( ARRAY_SIZE < MAX_ARRAY ? ARRAY_SIZE : MAX_ARRAY ))
66+
else
67+
ARRAY_SIZE=$MAX_ARRAY
6868
fi
69-
cat testpackage_run.txt
70-
. CI_env/bin/activate
71-
python ./testpackage/testpackage_get_job_error.py testpackage_run.txt
69+
export ARRAY_SIZE=$ARRAY_SIZE
70+
echo "Running $DIFFRESULT, ARRAY_SIZE $ARRAY_SIZE"
71+
.github/workflows/make_sbatch_job.sh run_testpackage_workflow $DIFFRESULT
72+
7273
7374
- name: Comparing plotted data
7475
run: |
7576
export TMPDIR=$RUNNER_TEMP
7677
module load Python/3.10.4-GCCcore-11.3.0
7778
. CI_env/bin/activate
78-
sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh ${{ steps.run_cl.outputs.DIFFRESULT }}
79-
cat testpackage_compare.txt
80-
python ./testpackage/testpackage_get_job_error.py testpackage_compare.txt
79+
.github/workflows/make_sbatch_job.sh run_compare ${{ steps.run_cl.outputs.DIFFRESULT }}
80+
8181
- name: scancel dangling job upon cancellation
8282
if: cancelled()
8383
run: |

‎.github/workflows/test_compare_images_full.yml‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -43,32 +43,26 @@ jobs:
4343
export TMPDIR=$RUNNER_TEMP
4444
module purge
4545
module load Python/3.10.4-GCCcore-11.3.0
46-
verf_loc="/wrk-kappa/group/spacephysics/analysator/CI/verification_sets"
46+
verf_loc="/turso/group/spacephysics/analysator/CI/verification_sets"
4747
verfset=$(ls -lth $verf_loc | grep ^d | head -n1 | grep -Po '\w+$')
4848
if [[ -f $verf_loc/$verfset/.lockfile ]]; then
4949
echo -e ".lockfile found in $verf_loc/$verfset, not running test, as the verification set generation is likely still ongoing\n Check ongoing actions and/or re-run verification set generation."
5050
exit 1
5151
fi
52-
sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh > jobid.txt
53-
export JOBID=$(grep -Po '\d+' jobid.txt)
54-
export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED)
55-
if [[ $SACCT_LOG ]]; then
56-
echo "Some job failed on a node"
57-
echo $SACCT_LOG
58-
exit 1
59-
fi
60-
cat testpackage_run.txt
61-
. CI_env/bin/activate
62-
python ./testpackage/testpackage_get_job_error.py testpackage_run.txt
52+
53+
.github/workflows/make_sbatch_job.sh run_testpackage_workflow
54+
6355
6456
- name: Comparing plotted data
6557
run: |
6658
export TMPDIR=$RUNNER_TEMP
6759
module load Python/3.10.4-GCCcore-11.3.0
6860
. CI_env/bin/activate
69-
sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh
70-
cat testpackage_compare.txt
71-
python ./testpackage/testpackage_get_job_error.py testpackage_compare.txt
61+
62+
.github/workflows/make_sbatch_job.sh run_compare
63+
64+
65+
7266
- name: scancel dangling job upon cancellation
7367
if: cancelled()
7468
run: |

‎testpackage/run_compare.sh‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash -l
22
#SBATCH -t 00:30:00
33
#SBATCH -J analysator_testpackage_compare
4-
#SBATCH --constraint="carrington"
4+
#SBATCH --constraint="ukko|carrington"
55
#SBATCH -p short
66
#SBATCH -n 1
77
#SBATCH --array=1-10
@@ -26,7 +26,8 @@ module load libglvnd/1.7.0-GCCcore-13.3.0
2626
module list
2727

2828
echo "SLURM_JOB_ID=$SLURM_ARRAY_JOB_ID" >> $GITHUB_OUTPUT
29-
verf_loc="/wrk-kappa/group/spacephysics/analysator/CI/verification_sets"
29+
echo $SLURM_ARRAY_JOB_ID
30+
verf_loc="/turso/group/spacephysics/analysator/CI/verification_sets"
3031

3132
#if pass we do not check for anything
3233
if echo $@ | grep -q -P "\spass$|\spass\s|pass"; then
@@ -50,13 +51,12 @@ if [[ $@ == 'verf_set' ]]; then
5051
folder_1="$verf_loc/$verfset/"
5152
folder_2="$verf_loc/$verfset2/"
5253
python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index 0 && echo "No differences found"
53-
echo "EXIT_CODE_FROM_JOB $?"
54-
exit 0
54+
exit $?
5555
#If old_python is used, run all
5656
elif [[ $1 == 'old_python' ]]; then
5757
check=false
5858
#Do selective compare if other arguments
59-
elif [ $@ ]; then
59+
elif [[ $@ ]]; then
6060
check=true
6161
fi
6262

@@ -71,13 +71,13 @@ then
7171
folder_1="$verf_loc/$verfset/$i/"
7272
folder_2="${PWD}/produced_plots/$i/"
7373
python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
74-
echo "EXIT_CODE_FROM_JOB $?"
74+
exit $?
7575
done
7676
exit 0
7777
else
7878
echo "Comparing all"
7979
folder_1="$verf_loc/$verfset/"
8080
folder_2="${PWD}/produced_plots/"
8181
python3 ./testpackage/testpackage_compare.py ${folder_1} ${folder_2} $jobcount $index && echo "No differences found in produced images"
82-
echo "EXIT_CODE_FROM_JOB $?"
82+
exit $?
8383
fi

‎testpackage/run_testpackage_generate_verf_set.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,14 @@ index=$(( $SLURM_ARRAY_TASK_ID - $SLURM_ARRAY_TASK_MIN ))
1414
hostname
1515

1616
echo "SLURM_JOB_ID=$SLURM_ARRAY_JOB_ID" >> $GITHUB_OUTPUT
17+
echo $SLURM_ARRAY_JOB_ID
1718
source CI_env/bin/activate
18-
export PATH=/wrk-vakka/group/spacephysics/proj/appl/tex-basic/texlive/2023/bin/x86_64-linux:$PATH
19+
export PATH=/turso/group/spacephysics/analysator/CI/tex-basic/texlive/2023/bin/x86_64-linux:$PATH
1920

2021
export PTNONINTERACTIVE=1
2122
export PTOUTPUTDIR=$1
2223

2324
python ./testpackage/testpackage_commons.py $jobcount $index
2425

25-
echo "EXIT_CODE_FROM_JOB $?"
26-
26+
exit $?
2727

‎testpackage/run_testpackage_workflow.sh‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash -l
22
#SBATCH -t 01:00:00
33
#SBATCH -J analysator_testpackage
4-
#SBATCH --constraint="carrington"
4+
#SBATCH --constraint="carrington|ukko"
55
#SBATCH -p short
66
#SBATCH -n 1
77
#SBATCH --array=1-14
@@ -18,9 +18,11 @@ hostname
1818

1919

2020
module purge
21-
export PATH=/wrk-vakka/group/spacephysics/proj/appl/tex-basic/texlive/2023/bin/x86_64-linux:$PATH
21+
export PATH=/turso/group/spacephysics/analysator/CI/tex-basic/texlive/2023/bin/x86_64-linux:$PATH
2222
older_python=false
2323
echo "SLURM_JOB_ID=$SLURM_ARRAY_JOB_ID" >> $GITHUB_OUTPUT
24+
echo $SLURM_ARRAY_JOB_ID
25+
2426
if [[ $1 == "old_python" ]]; then
2527
module load $2
2628
older_python=true
@@ -40,8 +42,7 @@ export PTOUTPUTDIR=$PWD/produced_plots/
4042
if $older_python; then
4143
echo "::warning:: Running with older python version $2"
4244
python ./testpackage/testpackage_commons.py $jobcount $index
43-
echo "EXIT_CODE_FROM_JOB $?"
44-
exit 0
45+
exit $?
4546
fi
4647
python ./testpackage/testpackage_commons.py $jobcount $index $@
47-
echo "EXIT_CODE_FROM_JOB $?"
48+
exit $?

‎testpackage/testpackage_commons.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ def source_file_name(filename,fileLocation,time):
3131
print("No significant changes made, the testpackage will not be run!")
3232
quit()
3333

34-
datalocation = "/wrk-kappa/group/spacephysics/analysator/CI/analysator-test-data/vlasiator"
34+
datalocation = "/turso/group/spacephysics/analysator/CI/analysator-test-data/vlasiator"
3535
runs = []
3636

3737

0 commit comments

Comments
 (0)