From 202c3ada1266cff2bb71c96e43d3c2117044e732 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 16 Jun 2025 11:19:36 +0200 Subject: [PATCH 01/10] Make sure the full directory name is replaced in eb_hooks.py --- install_scripts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index 5ac2940..0376c3c 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -204,4 +204,4 @@ done sed -i "s@//@/${EESSI_VERSION}/@g" ${INSTALL_PREFIX}/init/modules/EESSI/${EESSI_VERSION}.lua # replace EESSI version used in EasyBuild hooks -sed -i "s@//@/${EESSI_VERSION}/@g" ${INSTALL_PREFIX}/init/easybuild/eb_hooks.py +sed -i "s@/eessi-/@/eessi-${EESSI_VERSION}/@g" ${INSTALL_PREFIX}/init/easybuild/eb_hooks.py From bda6d6046369080c8eb09619cf93a9be64eb35fb Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 16 Jun 2025 11:42:16 +0200 Subject: [PATCH 02/10] Compare files with placeholders removed --- install_scripts.sh | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/install_scripts.sh b/install_scripts.sh index 0376c3c..d43a535 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -134,6 +134,9 @@ init_files=( minimal_eessi_env README.md test.py lmod_eessi_archdetect_wrapper.sh lmod_eessi_archdetect_wrapper_accel.sh ) +# make sure that scripts in init/ and scripts/ use correct EESSI version +sed -i "s/__EESSI_VERSION_DEFAULT__/${EESSI_VERSION}/g" ${TOPDIR}/init/eessi_defaults + copy_files_by_list ${TOPDIR}/init ${INSTALL_PREFIX}/init "${init_files[@]}" # Copy for the init/arch_specs directory @@ -152,12 +155,16 @@ copy_files_by_list ${TOPDIR}/init/Magic_Castle ${INSTALL_PREFIX}/init/Magic_Cast mc_files=( 2023.06.lua ) +# replace EESSI version used in comments in EESSI module +sed -i "s@//@/${EESSI_VERSION}/@g" ${TOPDIR}/init/modules/EESSI/${EESSI_VERSION}.lua copy_files_by_list ${TOPDIR}/init/modules/EESSI ${INSTALL_PREFIX}/init/modules/EESSI "${mc_files[@]}" # Copy for init/lmod directory -init_script_files=( - bash zsh ksh fish csh -) +init_script_files=$(ls ${TOPDIR}/init/lmod) +# replace placeholder for default EESSI version in Lmod init scripts +for shell in $init_script_files; do + sed -i "s/__EESSI_VERSION_DEFAULT__/${EESSI_VERSION}/g" ${TOPDIR}/init/lmod/${shell} +done copy_files_by_list ${TOPDIR}/init/lmod ${INSTALL_PREFIX}/init/lmod "${init_script_files[@]}" # Copy for the scripts directory @@ -186,22 +193,6 @@ ${INSTALL_PREFIX}/scripts/gpu_support/nvidia/easystacks "${host_injections_easys hook_files=( eb_hooks.py ) -copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/init/easybuild "${hook_files[@]}" - -# replace version placeholders in scripts; -# note: the commands below are always run, regardless of whether the scripts were changed, -# but that should be fine (no changes are made if version placeholder is not present anymore) - -# make sure that scripts in init/ and scripts/ use correct EESSI version -sed -i "s/__EESSI_VERSION_DEFAULT__/${EESSI_VERSION}/g" ${INSTALL_PREFIX}/init/eessi_defaults - -# replace placeholder for default EESSI version in Lmod init scripts -for shell in $(ls ${INSTALL_PREFIX}/init/lmod); do - sed -i "s/__EESSI_VERSION_DEFAULT__/${EESSI_VERSION}/g" ${INSTALL_PREFIX}/init/lmod/${shell} -done - -# replace EESSI version used in comments in EESSI module -sed -i "s@//@/${EESSI_VERSION}/@g" ${INSTALL_PREFIX}/init/modules/EESSI/${EESSI_VERSION}.lua - # replace EESSI version used in EasyBuild hooks -sed -i "s@/eessi-/@/eessi-${EESSI_VERSION}/@g" ${INSTALL_PREFIX}/init/easybuild/eb_hooks.py +sed -i "s@/eessi-/@/eessi-${EESSI_VERSION}/@g" ${TOPDIR}/eb_hooks.py +copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/init/easybuild "${hook_files[@]}" From 8f1f6ed86551fa760ad4145aeb1a7cafdedb8aa7 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 16 Jun 2025 13:12:39 +0200 Subject: [PATCH 03/10] Make sure test step is using right EESSI version --- bot/test.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bot/test.sh b/bot/test.sh index b615be0..168b0d5 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -151,12 +151,13 @@ EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} echo "bot/test.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" -# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} -# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# determine EESSI version to be used from .repository.repo_version in ${JOB_CFG_FILE} +# here, just set & export EESSI_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) -export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") -echo "bot/test.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" +REPOSITORY_VERSION=$(cfg_get_value "repository" "repo_version") +export EESSI_VERSION_OVERRIDE=${REPOSITORY_VERSION} +echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down From 140a20dcba513167fb65748e88a16b31d135857f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 16 Jun 2025 13:43:48 +0200 Subject: [PATCH 04/10] Make trivial change to trigger deploy of file --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4775bd4..552ba46 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -405,7 +405,7 @@ def parse_hook_zen4_module_only(ec, eprefix): env_varname = EESSI_IGNORE_ZEN4_GCC1220_ENVVAR # TODO: create a docs page to which we can refer for more info here # TODO: then update the link to the known issues page to the _specific_ issue - # Need to escape newline character so that the newline character actually ends up in the module file + # Need to escape the newline character so that the newline character actually ends up in the module file # (otherwise, it splits the string, and a 2-line string ends up in the modulefile, resulting in syntax error) errmsg = "EasyConfigs using toolchains based on GCCcore-12.2.0 are not supported for the Zen4 architecture.\\n" errmsg += "See https://www.eessi.io/docs/known_issues/eessi-/#gcc-1220-and-foss-2022b-based-modules-cannot-be-loaded-on-zen4-architecture" From 0b50d5b5eedd092e319b8cb1c3e9598e0701433c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 16 Jun 2025 14:02:35 +0200 Subject: [PATCH 05/10] Remove unnecessary space --- install_scripts.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index d43a535..4a52734 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -136,7 +136,6 @@ init_files=( ) # make sure that scripts in init/ and scripts/ use correct EESSI version sed -i "s/__EESSI_VERSION_DEFAULT__/${EESSI_VERSION}/g" ${TOPDIR}/init/eessi_defaults - copy_files_by_list ${TOPDIR}/init ${INSTALL_PREFIX}/init "${init_files[@]}" # Copy for the init/arch_specs directory From 2a287804b79ea7b1ba72d1c528a2d824dd81736b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 16 Jun 2025 14:14:16 +0200 Subject: [PATCH 06/10] Only retain the sed change if it is needed --- install_scripts.sh | 56 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/install_scripts.sh b/install_scripts.sh index 4a52734..d116a36 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -37,6 +37,30 @@ file_changed_in_pr() { ) && return 0 || return 1 } +sed_update_if_changed() { + # Usage: sed_update_if_changed 's/foo/bar/' file.txt + if [ "$#" -ne 2 ]; then + echo "Usage: sed_update_if_changed 'sed_command' file" >&2 + return 1 + fi + + local sed_command="$1" + local file="$2" + local tmp_file="$(mktemp "${file}.XXXXXX")" + + sed "$sed_command" "$file" > "$tmp_file" || { + rm -f "$tmp_file" + echo "sed command failed" >&2 + return 1 + } + + if ! cmp -s "$file" "$tmp_file"; then + mv "$tmp_file" "$file" + else + rm -f "$tmp_file" + fi +} + compare_and_copy() { if [ "$#" -ne 2 ]; then echo "Usage of function: compare_and_copy " @@ -134,8 +158,6 @@ init_files=( minimal_eessi_env README.md test.py lmod_eessi_archdetect_wrapper.sh lmod_eessi_archdetect_wrapper_accel.sh ) -# make sure that scripts in init/ and scripts/ use correct EESSI version -sed -i "s/__EESSI_VERSION_DEFAULT__/${EESSI_VERSION}/g" ${TOPDIR}/init/eessi_defaults copy_files_by_list ${TOPDIR}/init ${INSTALL_PREFIX}/init "${init_files[@]}" # Copy for the init/arch_specs directory @@ -154,16 +176,12 @@ copy_files_by_list ${TOPDIR}/init/Magic_Castle ${INSTALL_PREFIX}/init/Magic_Cast mc_files=( 2023.06.lua ) -# replace EESSI version used in comments in EESSI module -sed -i "s@//@/${EESSI_VERSION}/@g" ${TOPDIR}/init/modules/EESSI/${EESSI_VERSION}.lua copy_files_by_list ${TOPDIR}/init/modules/EESSI ${INSTALL_PREFIX}/init/modules/EESSI "${mc_files[@]}" # Copy for init/lmod directory -init_script_files=$(ls ${TOPDIR}/init/lmod) -# replace placeholder for default EESSI version in Lmod init scripts -for shell in $init_script_files; do - sed -i "s/__EESSI_VERSION_DEFAULT__/${EESSI_VERSION}/g" ${TOPDIR}/init/lmod/${shell} -done +init_script_files=( + bash zsh ksh fish csh +) copy_files_by_list ${TOPDIR}/init/lmod ${INSTALL_PREFIX}/init/lmod "${init_script_files[@]}" # Copy for the scripts directory @@ -192,6 +210,22 @@ ${INSTALL_PREFIX}/scripts/gpu_support/nvidia/easystacks "${host_injections_easys hook_files=( eb_hooks.py ) -# replace EESSI version used in EasyBuild hooks -sed -i "s@/eessi-/@/eessi-${EESSI_VERSION}/@g" ${TOPDIR}/eb_hooks.py copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/init/easybuild "${hook_files[@]}" + +# replace version placeholders in scripts; +# note: the commands below are always run, regardless of whether the scripts were changed, +# but that should be fine (no changes are made if version placeholder is not present anymore) + +# make sure that scripts in init/ and scripts/ use correct EESSI version +sed_update_if_changed "s/__EESSI_VERSION_DEFAULT__/${EESSI_VERSION}/g" ${INSTALL_PREFIX}/init/eessi_defaults + +# replace placeholder for default EESSI version in Lmod init scripts +for shell in $(ls ${INSTALL_PREFIX}/init/lmod); do + sed_update_if_changed "s/__EESSI_VERSION_DEFAULT__/${EESSI_VERSION}/g" ${INSTALL_PREFIX}/init/lmod/${shell} +done + +# replace EESSI version used in comments in EESSI module +sed_update_if_changed "s@//@/${EESSI_VERSION}/@g" ${INSTALL_PREFIX}/init/modules/EESSI/${EESSI_VERSION}.lua + +# replace EESSI version used in EasyBuild hooks +sed_update_if_changed "s@//@/${EESSI_VERSION}/@g" ${INSTALL_PREFIX}/init/easybuild/eb_hooks.py From b8c6600a2e18c1173deb8cbd8c04fa337c43440a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 16 Jun 2025 14:16:48 +0200 Subject: [PATCH 07/10] Use diff rather than cmp --- install_scripts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index d116a36..dc3356e 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -54,7 +54,7 @@ sed_update_if_changed() { return 1 } - if ! cmp -s "$file" "$tmp_file"; then + if ! diff -q "$file" "$tmp_file" > /dev/null; then mv "$tmp_file" "$file" else rm -f "$tmp_file" From 0afa994b6058510271f695f5be69f94846aab9ec Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 16 Jun 2025 14:20:26 +0200 Subject: [PATCH 08/10] Fix the original problem! --- install_scripts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index dc3356e..15675b2 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -228,4 +228,4 @@ done sed_update_if_changed "s@//@/${EESSI_VERSION}/@g" ${INSTALL_PREFIX}/init/modules/EESSI/${EESSI_VERSION}.lua # replace EESSI version used in EasyBuild hooks -sed_update_if_changed "s@//@/${EESSI_VERSION}/@g" ${INSTALL_PREFIX}/init/easybuild/eb_hooks.py +sed_update_if_changed "s@/eessi-/@/eessi-${EESSI_VERSION}/@g" ${INSTALL_PREFIX}/init/easybuild/eb_hooks.py From 4ca595fb19d52bcb0b2ee6a862e119c100eea51c Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 17 Jun 2025 12:29:04 +0200 Subject: [PATCH 09/10] Apply suggestions from code review --- install_scripts.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install_scripts.sh b/install_scripts.sh index 15675b2..adf9bca 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -55,7 +55,8 @@ sed_update_if_changed() { } if ! diff -q "$file" "$tmp_file" > /dev/null; then - mv "$tmp_file" "$file" + # Use cat to retain existing permissions, set umask to world readable in case the target file does not yet exist. + (umask 022 && cat "$tmp_file" > "$file") else rm -f "$tmp_file" fi From a8fc637f4658b410c5be1f1d70bc2409b07ab060 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 17 Jun 2025 12:49:20 +0200 Subject: [PATCH 10/10] Update install_scripts.sh --- install_scripts.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install_scripts.sh b/install_scripts.sh index adf9bca..cbe9bdb 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -57,9 +57,9 @@ sed_update_if_changed() { if ! diff -q "$file" "$tmp_file" > /dev/null; then # Use cat to retain existing permissions, set umask to world readable in case the target file does not yet exist. (umask 022 && cat "$tmp_file" > "$file") - else - rm -f "$tmp_file" fi + # Remove the temporary file + rm -f "$tmp_file" } compare_and_copy() {