Merge pull request #11 from boegel/init_scripts_fix_version

make sure that copied scripts in `init/` and `scripts/` use right EESSI version

Author: ocaisa

.github/workflows/test-eb-hooks.yml

@@ -6,11 +6,13 @@ on:
   workflow_dispatch:
 permissions:
   contents: read # to fetch code (actions/checkout)
-env:
-  EESSI_VERSION: '2023.06'
 jobs:
   check_eb_hooks:
     runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        EESSI_VERSION:
+          - '2023.06'
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -50,7 +52,10 @@ jobs:
                 git show origin/${{ github.base_ref }}:$FILE > "$TEMP_FILE"
               fi
 
+              # replace <EESSI_VERSION> placeholder (as is also done in install_scripts.sh)
+              sed -i "s/<EESSI_VERSION>/${{matrix.EESSI_VERSION}}/g" "${TEMP_FILE}"
+
               # Compare the hooks to what is shipped in the repository
-              source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash
+              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EESSI-extend
               diff "$TEMP_FILE" "$EASYBUILD_HOOKS"

.github/workflows/tests_archdetect_nvidia_gpu.yml

@@ -17,6 +17,8 @@ jobs:
           - 2xa100 # cc80, supported with (atleast) zen2 CPU
           - 4xa100 # cc80, supported with (atleast) zen2 CPU
           - cc01 # non-existing GPU
+        EESSI_VERSION:
+          - '2023.06'
       fail-fast: false
     steps:
     - name: checkout
@@ -31,6 +33,10 @@ jobs:
           cvmfs_http_proxy: DIRECT
           cvmfs_repositories: software.eessi.io
 
+    - name: Fix EESSI version in init scripts
+      run: |
+        sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/eessi_defaults
+
     - name: test accelerator detection
       run: |
           export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/amd/zen2'
@@ -68,7 +74,7 @@ jobs:
                   match=$(grep "${pattern}" init.out || true)
                   test "x${match}" = "x" || (echo "unexpected match found for '${pattern}' in init output" && exit 1)
 
-                  pattern="Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/.*/accel/.*/modules/all to \$MODULEPATH"
+                  pattern="Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/.*/accel/.*/modules/all to \$MODULEPATH"
                   echo ">>> checking for lack of pattern '${pattern}' in init output..."
                   match=$(grep "${pattern}" init.out || true)
                   test "x${match}" = "x" || (echo "unexpected match found for '${pattern}' in init output" && exit 1)
@@ -79,19 +85,19 @@ jobs:
                   echo ">>> checking for pattern '${pattern}' in init output..."
                   grep "${pattern}" init.out || (echo "FAILED 1" || exit 1)
 
-                  pattern="Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/.*/accel/.*/modules/all to \$MODULEPATH"
+                  pattern="Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/.*/accel/.*/modules/all to \$MODULEPATH"
                   echo ">>> checking for lack of pattern '${pattern}' in init output..."
                   match=$(grep "${pattern}" init.out || true)
                   test "x${match}" = "x" || (echo "unexpected match found for '${pattern}' in init output" && exit 1)
 
               else
                   echo ">>> checking for 'accel/nvidia/cc80' in init output..."
                   grep "archdetect found supported accelerator for CPU target x86_64/amd/zen2: accel/nvidia/cc80" init.out || (echo "FAILED 2" && exit 1)
-                  grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1)
+                  grep "Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/x86_64/amd/zen2/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1)
               fi
 
               echo ">>> checking last line of init output..."
-              tail -1 init.out | grep "Environment set up to use EESSI (2023.06), have fun!" || (echo "FAILED, full init utput:" && cat init.out && exit 1)
+              tail -1 init.out | grep "Environment set up to use EESSI (${{matrix.EESSI_VERSION}}), have fun!" || (echo "FAILED, full init utput:" && cat init.out && exit 1)
 
               echo "All checks on init output PASSED"
           else
@@ -118,7 +124,7 @@ jobs:
           echo ">>> checking for 'accel/nvidia/cc80' in init output..."
           grep "archdetect found supported accelerator for CPU target x86_64/amd/zen3: accel/nvidia/cc80" init.out || (echo "FAILED 1" && exit 1)
           grep "Using x86_64/amd/zen2 as software subdirectory" init.out || (echo "FAILED 2" && exit 1)
-          grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1)
-          grep "Prepending /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 4" && exit 1)
+          grep "Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/x86_64/amd/zen2/modules/all to \$MODULEPATH" init.out || (echo "FAILED 3" && exit 1)
+          grep "Prepending /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/software/linux/x86_64/amd/zen3/accel/nvidia/cc80/modules/all to \$MODULEPATH" init.out || (echo "FAILED 4" && exit 1)
 
           echo "All checks on init output PASSED"

.github/workflows/tests_eessi_module.yml

@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         EESSI_VERSION:
-          - 2023.06
+          - '2023.06'
     steps:
       - name: Check out software-layer repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -68,7 +68,7 @@ jobs:
       fail-fast: false
       matrix:
         EESSI_VERSION:
-          - 2023.06
+          - '2023.06'
         EESSI_SOFTWARE_SUBDIR_OVERRIDE:
           - x86_64/amd/zen3
           - x86_64/amd/zen4
@@ -84,6 +84,10 @@ jobs:
           cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
           cvmfs_http_proxy: DIRECT
           cvmfs_repositories: software.eessi.io
+
+      - name: Fix EESSI version in init scripts
+        run: |
+          sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/eessi_defaults
               
       - name: Test for expected variables match between Lmod init script and original bash script
         run: |
@@ -141,7 +145,7 @@ jobs:
       fail-fast: false
       matrix:
         EESSI_VERSION:
-          - 2023.06
+          - '2023.06'
         EESSI_SOFTWARE_SUBDIR_OVERRIDE:
           - none
           - x86_64/amd/zen2

.github/workflows/tests_init_module.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         EESSI_VERSION:
-          - 2023.06
+          - '2023.06'
         EESSI_SOFTWARE_SUBDIR_OVERRIDE:
           - x86_64/intel/haswell
     steps:
@@ -27,6 +27,12 @@ jobs:
           cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
           cvmfs_http_proxy: DIRECT
           cvmfs_repositories: software.eessi.io
+
+      - name: Fix EESSI version in lmod init scripts
+        run: |
+          for shell in $(ls init/lmod); do
+            sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/lmod/${shell}
+          done
          
       - name: Clone assert.sh script
         run: git clone https://github.com/lehmannro/assert.sh.git

eb_hooks.py

@@ -408,7 +408,7 @@ def parse_hook_zen4_module_only(ec, eprefix):
         # Need to escape newline character so that the newline character actually ends up in the module file
         # (otherwise, it splits the string, and a 2-line string ends up in the modulefile, resulting in syntax error)
         errmsg = "EasyConfigs using toolchains based on GCCcore-12.2.0 are not supported for the Zen4 architecture.\\n"
-        errmsg += "See https://www.eessi.io/docs/known_issues/eessi-2023.06/#gcc-1220-and-foss-2022b-based-modules-cannot-be-loaded-on-zen4-architecture"
+        errmsg += "See https://www.eessi.io/docs/known_issues/eessi-<EESSI_VERSION>/#gcc-1220-and-foss-2022b-based-modules-cannot-be-loaded-on-zen4-architecture"
         ec['modluafooter'] = 'if (not os.getenv("%s")) then LmodError("%s") end' % (env_varname, errmsg)
 
 

init/eessi_defaults

@@ -8,6 +8,8 @@
 # license: GPLv2
 #
 
+export EESSI_VERSION_DEFAULT='__EESSI_VERSION_DEFAULT__'
+
 # use different defaults for RISC-V, as we want to redirect to the riscv.eessi.io repo
 if [[ $(uname -m) == "riscv64" ]]; then
     export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/riscv.eessi.io}"
@@ -20,7 +22,7 @@ if [[ $(uname -m) == "riscv64" ]]; then
     fi
 else
     export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/software.eessi.io}"
-    export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=2023.06}"
+    export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=${EESSI_VERSION_DEFAULT}}"
 fi
 # use archdetect by default, unless otherwise specified
 export EESSI_USE_ARCHDETECT="${EESSI_USE_ARCHDETECT:=1}"

init/lmod/bash

@@ -1,7 +1,8 @@
 # Choose an EESSI CVMFS repository
 EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO:-/cvmfs/software.eessi.io}"
 # Choose an EESSI version
-EESSI_VERSION="${EESSI_VERSION:-2023.06}"
+EESSI_VERSION_DEFAULT="__EESSI_VERSION_DEFAULT__"
+EESSI_VERSION="${EESSI_VERSION:-${EESSI_VERSION_DEFAULT}}"
 # Path to top-level module tree
 export MODULEPATH="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/modules"
 . "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash"

init/lmod/csh

@@ -1,7 +1,8 @@
 # Choose an EESSI CVMFS repository
 if (! $?EESSI_CVMFS_REPO) then; set EESSI_CVMFS_REPO = "/cvmfs/software.eessi.io"; endif
 # Choose an EESSI version
-if (! $?EESSI_VERSION) then; set EESSI_VERSION = "2023.06"; endif
+setenv EESSI_VERSION_DEFAULT "__EESSI_VERSION_DEFAULT__"
+if (! $?EESSI_VERSION) then; set EESSI_VERSION = "${EESSI_VERSION_DEFAULT}"; endif
 # Path to top-level module tree
 setenv MODULEPATH "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/modules"
 source "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)/usr/share/Lmod/init/csh"

init/lmod/fish

@@ -1,7 +1,8 @@
 # Choose an EESSI CVMFS repository
 set EESSI_CVMFS_REPO (set -q EESSI_CVMFS_REPO; and echo "$EESSI_CVMFS_REPO"; or echo "/cvmfs/software.eessi.io")
 # Choose an EESSI version
-set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "2023.06")
+set EESSI_VERSION_DEFAULT "__EESSI_VERSION_DEFAULT__"
+set EESSI_VERSION (set -q EESSI_VERSION; and echo "$EESSI_VERSION"; or echo "$EESSI_VERSION_DEFAULT")
 # Path to top-level module tree
 set -x MODULEPATH "$EESSI_CVMFS_REPO"/versions/"$EESSI_VERSION"/init/modules
 . "$EESSI_CVMFS_REPO"/versions/"$EESSI_VERSION"/compat/linux/(uname -m)/usr/share/Lmod/init/fish

init/lmod/ksh

@@ -1,7 +1,8 @@
 # Choose an EESSI CVMFS repository
 EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO:-/cvmfs/software.eessi.io}"
 # Choose an EESSI version
-EESSI_VERSION="${EESSI_VERSION:-2023.06}"
+EESSI_VERSION_DEFAULT="__EESSI_VERSION_DEFAULT__"
+EESSI_VERSION="${EESSI_VERSION:-${EESSI_VERSION_DEFAULT}}"
 # Path to top-level module tree
 export MODULEPATH="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/modules"
 . "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)/usr/share/Lmod/init/ksh"

init/lmod/zsh

@@ -1,7 +1,8 @@
 # Choose an EESSI CVMFS repository
 EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO:-/cvmfs/software.eessi.io}"
 # Choose an EESSI version
-EESSI_VERSION="${EESSI_VERSION:-2023.06}"
+EESSI_VERSION_DEFAULT="__EESSI_VERSION_DEFAULT__"
+EESSI_VERSION="${EESSI_VERSION:-${EESSI_VERSION_DEFAULT}}"
 # Path to top-level module tree
 export MODULEPATH="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/modules"
 . "${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)/usr/share/Lmod/init/zsh"

init/modules/EESSI/2023.06.lua

@@ -21,6 +21,7 @@ if (subprocess("uname -m"):gsub("\n$","") == "riscv64") then
 end
 local eessi_prefix = pathJoin(eessi_repo, "versions", eessi_version)
 local eessi_os_type = "linux"
+setenv("EESSI_VERSION_DEFAULT", eessi_version)
 setenv("EESSI_VERSION", eessi_version)
 setenv("EESSI_CVMFS_REPO", eessi_repo)
 setenv("EESSI_OS_TYPE", eessi_os_type)
@@ -80,18 +81,18 @@ local archdetect_accel = archdetect_accel()
 -- eessi_cpu_family is derived from  the archdetect match, e.g., x86_64
 local eessi_cpu_family = archdetect:match("([^/]+)")
 local eessi_software_subdir = archdetect
--- eessi_eprefix is the base location of the compat layer, e.g., /cvmfs/software.eessi.io/versions/2023.06/compat/linux/x86_64
+-- eessi_eprefix is the base location of the compat layer, e.g., /cvmfs/software.eessi.io/versions/<EESSI_VERSION>/compat/linux/x86_64
 local eessi_eprefix = pathJoin(eessi_prefix, "compat", eessi_os_type, eessi_cpu_family)
 -- eessi_software_path is the location of the software installations, e.g.,
--- /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3
+-- /cvmfs/software.eessi.io/versions/<EESSI_VERSION>/software/linux/x86_64/amd/zen3
 local eessi_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_software_subdir)
 local eessi_modules_subdir = pathJoin("modules", "all")
 -- eessi_module_path is the location of the _CPU_ module files, e.g.,
--- /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/modules/all
+-- /cvmfs/software.eessi.io/versions/<EESSI_VERSION>/software/linux/x86_64/amd/zen3/modules/all
 local eessi_module_path = pathJoin(eessi_software_path, eessi_modules_subdir)
 local eessi_site_software_path = string.gsub(eessi_software_path, "versions", "host_injections")
 -- Site module path is the same as the EESSI one, but with `versions` changed to `host_injections`, e.g.,
---  /cvmfs/software.eessi.io/host_injections/2023.06/software/linux/x86_64/amd/zen3/modules/all
+--  /cvmfs/software.eessi.io/host_injections/<EESSI_VERSION>/software/linux/x86_64/amd/zen3/modules/all
 local eessi_site_module_path = pathJoin(eessi_site_software_path, eessi_modules_subdir)
 setenv("EPREFIX",  eessi_eprefix)
 eessiDebug("Setting EPREFIX to " .. eessi_eprefix)
@@ -134,10 +135,10 @@ if not (archdetect_accel == nil or archdetect_accel == '') then
     -- The CPU subdirectory of the accelerator installations is _usually_ the same as host CPU, but this can be overridden
     eessi_accel_software_subdir = os.getenv("EESSI_ACCEL_SOFTWARE_SUBDIR_OVERRIDE") or eessi_software_subdir
     -- CPU location of the accelerator installations, e.g.,
-    -- /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3
+    -- /cvmfs/software.eessi.io/versions/<EESSI_VERSION>/software/linux/x86_64/amd/zen3
     eessi_accel_software_path = pathJoin(eessi_prefix, "software", eessi_os_type, eessi_accel_software_subdir)
     -- location of the accelerator modules, e.g.,
-    -- /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen3/accel/nvidia/cc80/modules/all
+    -- /cvmfs/software.eessi.io/versions/<EESSI_VERSION>/software/linux/x86_64/amd/zen3/accel/nvidia/cc80/modules/all
     eessi_module_path_accel = pathJoin(eessi_accel_software_path, archdetect_accel, eessi_modules_subdir)
     eessiDebug("Checking if " .. eessi_module_path_accel .. " exists")
     if isDir(eessi_module_path_accel) then

install_scripts.sh

@@ -187,3 +187,21 @@ hook_files=(
     eb_hooks.py
 )
 copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/init/easybuild "${hook_files[@]}"
+
+# replace version placeholders in scripts;
+# note: the commands below are always run, regardless of whether the scripts were changed,
+# but that should be fine (no changes are made if version placeholder is not present anymore)
+
+# make sure that scripts in init/ and scripts/ use correct EESSI version
+sed -i "s/__EESSI_VERSION_DEFAULT__/${EESSI_VERSION}/g" ${INSTALL_PREFIX}/init/eessi_defaults
+
+# replace placeholder for default EESSI version in Lmod init scripts
+for shell in $(ls ${INSTALL_PREFIX}/init/lmod); do
+    sed -i "s/__EESSI_VERSION_DEFAULT__/${EESSI_VERSION}/g" ${INSTALL_PREFIX}/init/lmod/${shell}
+done
+
+# replace EESSI version used in comments in EESSI module
+sed -i "s@/<EESSI_VERSION>/@/${EESSI_VERSION}/@g" ${INSTALL_PREFIX}/init/modules/EESSI/${EESSI_VERSION}.lua
+
+# replace EESSI version used in EasyBuild hooks
+sed -i "s@/<EESSI_VERSION>/@/${EESSI_VERSION}/@g" ${INSTALL_PREFIX}/init/easybuild/eb_hooks.py