Skip to content

Commit 0bb9f4e

Browse files
lassejsclassejsc
committed
Merge branch 'master' into wrapper
2 parents b8236dc + c00e9c4 commit 0bb9f4e

6 files changed

Lines changed: 51 additions & 68 deletions

File tree

‎.github/workflows/generate_reference_plots.yml‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -51,20 +51,11 @@ jobs:
5151
echo $SACCT_LOG
5252
exit 1
5353
fi
54-
54+
.github/workflows/make_sbatch_job.sh run_testpackage_generate_verf_set $DATAPATH
5555
rm $DATAPATH/.lockfile
5656
- name: Comparing plotted data
5757
run: |
5858
export TMPDIR=$RUNNER_TEMP
5959
module load Python/3.10.4-GCCcore-11.3.0
6060
. CI_env/bin/activate
61-
sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh verf_set > jobid1.txt
62-
cat testpackage_compare.txt
63-
64-
export JOBID=$(grep -Po '\d+' jobid1.txt)
65-
export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED)
66-
if [[ $SACCT_LOG ]]; then
67-
echo "Some job failed on a node, try to take a look at the slurm log."
68-
echo "$SACCT_LOG"
69-
exit 1
70-
fi
61+
.github/workflows/make_sbatch_job.sh run_compare verf_set

‎.github/workflows/make_sbatch_job.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/usr/bin/env bash
2+
if [[ ! $ARRAY_SIZE ]]; then
3+
ARRAY_SIZE=14
4+
fi
5+
# cmd argument $1 is used for the file name of the script to run, log file, jobid
6+
7+
#if sbatch fails as it gets the error code from python it tries to print the log
8+
# first with srun (in case the file has not yet updated on the front end)
9+
# second if srun fails (in case communication failure) it tries to cat it on the frontend (may be empty if file has not been updated pyproject)
10+
11+
sbatch -W --array=1-$ARRAY_SIZE -o "$1" ./testpackage/$1.sh $2 > jobid_$1 || srun cat $1 || cat $1 || echo "cat failed exit code $?"
12+
13+
#in case we do exit 0 successfully
14+
LOG=$(srun cat $1 || cat $1)
15+
if [[ ! $LOG ]]; then
16+
echo "::warning::Log file could not be read: srun failed and cat returned and empty logfile. Exit code $?"
17+
exit 1
18+
fi
19+
echo "$LOG"
20+
#It is possible that the sbatch command above returns exit 0 if only for example 1 of the array jobs failed but not all, in such a case we check sacct
21+
# It is also possible that the node never ran it and silently failed which is visible on sacct
22+
#Additionally note that JOBID is saved by the run scripts HOWEVER if a nodes silently fails it may not get passed to the github output
23+
JOBID=$(srun grep -Po '\d+' jobid_$1 || grep -Po '\d+' jobid_$1)
24+
if [[ $? -ne 0 ]] || [[ ! $JOBID ]]; then
25+
echo "::error::Jobid could not be found, exiting. Exit code $?"
26+
exit 1
27+
fi
28+
SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED)
29+
if [[ $SACCT_LOG ]]; then
30+
echo "::error::Some job failed on a node, try to take a look at the slurm log."
31+
echo "$SACCT_LOG"
32+
exit 1
33+
fi

‎.github/workflows/test_compare_image_oldest_python.yml‎

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -75,15 +75,9 @@ jobs:
7575
echo -e ".lockfile found in $verf_loc/$verfset, not running test, as the verification set generation is likely still ongoing\n Check ongoing actions and/or re-run verification set generation."
7676
exit 1
7777
fi
78-
sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh old_python ${{ steps.pyversion.outputs.PYTHON }} > jobid.txt
79-
cat testpackage_run.txt
80-
export JOBID=$(grep -Po '\d+' jobid.txt)
81-
export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED)
82-
if [[ $SACCT_LOG ]]; then
83-
echo "Some job failed on a node, try to take a look at the slurm log."
84-
echo "$SACCT_LOG"
85-
exit 1
86-
fi
78+
79+
.github/workflows/make_sbatch_job.sh run_testpackage_workflow old_python ${{ steps.pyversion.outputs.PYTHON }}
80+
8781
8882
- name: Comparing plotted data
8983
run: |
@@ -92,15 +86,7 @@ jobs:
9286
module load ${{ steps.pyversion.outputs.PYTHON }}
9387
module list
9488
. CI_env/bin/activate
95-
sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh old_python ${{ steps.pyversion.outputs.PYTHON }} > jobid1.txt
96-
cat testpackage_compare.txt
97-
export JOBID=$(grep -Po '\d+' jobid1.txt)
98-
export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED)
99-
if [[ $SACCT_LOG ]]; then
100-
echo "Some job failed on a node, try to take a look at the slurm log."
101-
echo "$SACCT_LOG"
102-
exit 1
103-
fi
89+
.github/workflows/make_sbatch_job.sh run_compare old_python ${{ steps.pyversion.outputs.PYTHON }}
10490
10591
- name: scancel dangling job upon cancellation
10692
if: cancelled()

‎.github/workflows/test_compare_images.yml‎

Lines changed: 5 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@ jobs:
5757
cat diff_log.txt
5858
fi
5959
echo "DIFFRESULT=$DIFFRESULT" >> $GITHUB_OUTPUT
60-
echo "Running $DIFFRESULT"
6160
MAX_ARRAY=14
6261
TOTAL_TESTS=$(ls ./testpackage/testpackage_definitions/ | wc -l)
6362
if [[ $DIFFRESULT && $DIFFRESULT != "pass" ]]; then
@@ -67,32 +66,18 @@ jobs:
6766
else
6867
ARRAY_SIZE=$MAX_ARRAY
6968
fi
70-
sbatch -W -o testpackage_run.txt --array=1-$ARRAY_SIZE --job-name gen_plots ./testpackage/run_testpackage_workflow.sh $DIFFRESULT > jobid.txt
71-
cat testpackage_run.txt
72-
export JOBID=$(grep -Po '\d+' jobid.txt)
73-
export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED)
74-
if [[ $SACCT_LOG ]]; then
75-
echo "Some job failed on a node, try to take a look at the slurm log."
76-
echo "$SACCT_LOG"
77-
exit 1
78-
fi
69+
export ARRAY_SIZE=$ARRAY_SIZE
70+
echo "Running $DIFFRESULT, ARRAY_SIZE $ARRAY_SIZE"
71+
.github/workflows/make_sbatch_job.sh run_testpackage_workflow $DIFFRESULT
72+
7973
8074
- name: Comparing plotted data
8175
run: |
8276
export TMPDIR=$RUNNER_TEMP
8377
module load Python/3.10.4-GCCcore-11.3.0
8478
. CI_env/bin/activate
85-
sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh ${{ steps.run_cl.outputs.DIFFRESULT }} > jobid1.txt
86-
cat testpackage_compare.txt
87-
export JOBID=$(grep -Po '\d+' jobid1.txt)
88-
export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED)
89-
if [[ $SACCT_LOG ]]; then
90-
echo "Some job failed on a node, try to take a look at the slurm log."
91-
echo "$SACCT_LOG"
92-
exit 1
93-
fi
79+
.github/workflows/make_sbatch_job.sh run_compare ${{ steps.run_cl.outputs.DIFFRESULT }}
9480
95-
9681
- name: scancel dangling job upon cancellation
9782
if: cancelled()
9883
run: |

‎.github/workflows/test_compare_images_full.yml‎

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -49,30 +49,18 @@ jobs:
4949
echo -e ".lockfile found in $verf_loc/$verfset, not running test, as the verification set generation is likely still ongoing\n Check ongoing actions and/or re-run verification set generation."
5050
exit 1
5151
fi
52-
sbatch -W -o testpackage_run.txt --job-name gen_plots ./testpackage/run_testpackage_workflow.sh > jobid.txt
53-
cat testpackage_run.txt
54-
export JOBID=$(grep -Po '\d+' jobid.txt)
55-
export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED)
56-
if [[ $SACCT_LOG ]]; then
57-
echo "Some job failed on a node, try to take a look at the slurm log."
58-
echo "$SACCT_LOG"
59-
exit 1
60-
fi
52+
53+
.github/workflows/make_sbatch_job.sh run_testpackage_workflow
54+
6155
6256
- name: Comparing plotted data
6357
run: |
6458
export TMPDIR=$RUNNER_TEMP
6559
module load Python/3.10.4-GCCcore-11.3.0
6660
. CI_env/bin/activate
67-
sbatch -W -o "testpackage_compare.txt" ./testpackage/run_compare.sh > jobid1.txt
68-
cat testpackage_compare.txt
69-
export JOBID=$(grep -Po '\d+' jobid1.txt)
70-
export SACCT_LOG=$(sacct -j $JOBID -o job,state,node | grep FAILED)
71-
if [[ $SACCT_LOG ]]; then
72-
echo "Some job failed on a node, try to take a look at the slurm log."
73-
echo "$SACCT_LOG"
74-
exit 1
75-
fi
61+
62+
.github/workflows/make_sbatch_job.sh run_compare
63+
7664
7765
7866
- name: scancel dangling job upon cancellation

‎testpackage/run_compare.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ if [[ $@ == 'verf_set' ]]; then
5656
elif [[ $1 == 'old_python' ]]; then
5757
check=false
5858
#Do selective compare if other arguments
59-
elif [ $@ ]; then
59+
elif [[ $@ ]]; then
6060
check=true
6161
fi
6262

0 commit comments

Comments
 (0)