Skip to content

Commit 73905db

Browse files
boegelboegel
authored
Merge pull request #467 from casparvl/run_test_suite_based_on_pr366
let bot run EESSI test suite during test phase
2 parents b9d32f0 + c53f578 commit 73905db

6 files changed

+686
-5
lines changed

‎bot/check-build.sh

+1
Original file line numberDiff line numberDiff line change
@@ -400,6 +400,7 @@ comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${succ
400400
success_msg="found message matching <code>${GP_tgz_created}</code>"
401401
failure_msg="no message matching <code>${GP_tgz_created}</code>"
402402
comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
403+
403404
# Now, do the actual replacement of __DETAILS_FMT__
404405
comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
405406
comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"

‎bot/check-test.sh

+187-2
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,193 @@ job_dir=${PWD}
1313
job_out="slurm-${SLURM_JOB_ID}.out"
1414
job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
1515

16+
# Check that job output file is found
17+
[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
18+
if [[ -f ${job_out} ]]; then
19+
SLURM_OUTPUT_FOUND=1
20+
[[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'"
21+
else
22+
SLURM_OUTPUT_FOUND=0
23+
[[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found"
24+
fi
25+
26+
27+
# ReFrame prints e.g.
28+
#[----------] start processing checks
29+
#[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
30+
#[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default
31+
#[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default
32+
#[ FAIL ] (1/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default
33+
#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_d597cff4'
34+
#[ OK ] (2/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
35+
#P: perf: 8.441 ns/day (r:0, l:None, u:None)
36+
#[ FAIL ] (3/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default
37+
#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_f4194106'
38+
#[----------] all spawned checks have finished
39+
#[ FAILED ] Ran 3/3 test case(s) from 2 check(s) (2 failure(s), 0 skipped, 0 aborted)
40+
41+
# We will grep for the last and final line, since this reflects the overall result
42+
# Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
43+
FAILED=-1
44+
if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
45+
GP_failed='\[\s*FAILED\s*\].*Ran .* test case'
46+
grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
47+
[[ $? -eq 0 ]] && FAILED=1 || FAILED=0
48+
# have to be careful to not add searched for pattern into slurm out file
49+
[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
50+
[[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_failed}"
51+
fi
52+
53+
# Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
54+
# I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests
55+
ERROR=-1
56+
if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
57+
GP_error='ERROR: '
58+
grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
59+
[[ $? -eq 0 ]] && ERROR=1 || ERROR=0
60+
# have to be careful to not add searched for pattern into slurm out file
61+
[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'"
62+
[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
63+
fi
64+
65+
SUCCESS=-1
66+
# Grep for the success pattern, so we can report the amount of tests run
67+
if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
68+
GP_success='\[\s*PASSED\s*\].*Ran .* test case'
69+
grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
70+
[[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
71+
# have to be careful to not add searched for pattern into slurm out file
72+
[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
73+
[[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_success}"
74+
fi
75+
76+
if [[ ! -z ${grep_reframe_failed} ]]; then
77+
grep_reframe_result=${grep_reframe_failed}
78+
else
79+
grep_reframe_result=${grep_reframe_success}
80+
fi
81+
1682
echo "[TEST]" > ${job_test_result_file}
17-
echo "comment_description = <em>(no tests yet)</em>" >> ${job_test_result_file}
18-
echo "status = SUCCESS" >> ${job_test_result_file}
83+
if [[ ${SLURM_OUTPUT_FOUND} -eq 0 ]]; then
84+
summary=":cry: FAILURE"
85+
reason="Job output file not found, cannot check test results."
86+
status="FAILURE"
87+
# Should come before general errors: if SUCCESS==1, it indicates the test suite ran succesfully
88+
# regardless of other things that might have gone wrong
89+
elif [[ ${SUCCESS} -eq 1 ]]; then
90+
summary=":grin: SUCCESS"
91+
reason=""
92+
status="SUCCESS"
93+
# Should come before general errors: if FAILED==1, it indicates the test suite ran
94+
# otherwise the pattern wouldn't have been there
95+
elif [[ ${FAILED} -eq 1 ]]; then
96+
summary=":cry: FAILURE"
97+
reason="EESSI test suite produced failures."
98+
status="FAILURE"
99+
elif [[ ${ERROR} -eq 1 ]]; then
100+
summary=":cry: FAILURE"
101+
reason="EESSI test suite was not run, test step itself failed to execute."
102+
status="FAILURE"
103+
else
104+
summary=":grin: FAILURE"
105+
reason="Failed for unknown reason"
106+
status="FAILURE"
107+
fi
108+
109+
110+
echo "[TEST]" > ${job_test_result_file}
111+
echo -n "comment_description = " >> ${job_test_result_file}
112+
113+
# Use template for writing PR comment with details
114+
# construct and write complete PR comment details: implements third alternative
115+
comment_template="<details>__SUMMARY_FMT__<dl>__REASON_FMT____REFRAME_FMT____DETAILS_FMT__</dl></details>"
116+
comment_success_item_fmt=":white_check_mark: __ITEM__"
117+
comment_failure_item_fmt=":x: __ITEM__"
118+
119+
# Initialize comment_description
120+
comment_description=${comment_template}
121+
122+
# Now, start replacing template items one by one
123+
comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
124+
comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
125+
comment_description=${comment_description/__SUMMARY_FMT__/${comment_summary}}
126+
127+
128+
# Only add if there is a reason (e.g. no reason for successful runs)
129+
if [[ ! -z ${reason} ]]; then
130+
comment_reason_fmt="<dt>_Reason_</dt><dd>__REASONS__</dd>"
131+
reason_details="${comment_reason_fmt/__REASONS__/${reason}}"
132+
comment_description=${comment_description/__REASON_FMT__/${reason_details}}
133+
else
134+
comment_description=${comment_description/__REASON_FMT__/""}
135+
fi
136+
137+
# Only add if there is a reframe summary (e.g. no reframe summary if reframe wasn't launched succesfully)
138+
echo "ReFrame result:"
139+
echo "${grep_reframe_result}"
140+
if [[ ! -z ${grep_reframe_result} ]]; then
141+
comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
142+
reframe_summary=${comment_reframe_fmt/__REFRAME_SUMMARY__/${grep_reframe_result}}
143+
comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}}
144+
else
145+
comment_description=${comment_description/__REFRAME_FMT__/""}
146+
fi
147+
148+
# Declare functions
149+
function print_br_item() {
150+
format="${1}"
151+
item="${2}"
152+
echo -n "${format//__ITEM__/${item}}<br/>"
153+
}
154+
155+
function success() {
156+
format="${comment_success_item_fmt}"
157+
item="$1"
158+
print_br_item "${format}" "${item}"
159+
}
160+
161+
function failure() {
162+
format="${comment_failure_item_fmt}"
163+
item="$1"
164+
print_br_item "${format}" "${item}"
165+
}
166+
167+
function add_detail() {
168+
actual=${1}
169+
expected=${2}
170+
success_msg="${3}"
171+
failure_msg="${4}"
172+
if [[ ${actual} -eq ${expected} ]]; then
173+
success "${success_msg}"
174+
else
175+
failure "${failure_msg}"
176+
fi
177+
}
178+
179+
# first construct comment_details_list, abbreviated comment_details_list
180+
# then use it to set comment_details
181+
comment_details_list=""
182+
183+
success_msg="job output file <code>${job_out}</code>"
184+
failure_msg="no job output file <code>${job_out}</code>"
185+
comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
186+
187+
success_msg="no message matching <code>${GP_error}</code>"
188+
failure_msg="found message matching <code>${GP_error}</code>"
189+
comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
190+
191+
# Add an escape character to every *, for it to be printed correctly in the comment on GitHub
192+
GP_failed="${GP_failed//\*/\\*}"
193+
success_msg="no message matching <code>""${GP_failed}""</code>"
194+
failure_msg="found message matching <code>""${GP_failed}""</code>"
195+
comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
196+
197+
comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
198+
comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
199+
comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
200+
201+
# Actually writing the comment description to the result file
202+
echo "${comment_description}" >> ${job_test_result_file}
203+
echo "status = ${status}" >> ${job_test_result_file}
19204

20205
exit 0

0 commit comments

Comments
 (0)