Merge branch 'nessi.no-2023.06' into nessi.no-2023.06-QuantumESPRESSO/6.7-foss/2021a

Author: TopRichard

.github/workflows/test_eessi.yml

@@ -13,11 +13,9 @@ jobs:
         - 2023.06
         EESSI_SOFTWARE_SUBDIR:
         - aarch64/generic
-        - aarch64/graviton2
-        - aarch64/graviton3
         - x86_64/amd/zen2
-        - x86_64/amd/zen3
-        - x86_64/intel/haswell
+        - x86_64/intel/broadwell
+        - x86_64/intel/cascadelake
         - x86_64/intel/skylake_avx512
         - x86_64/generic
         EASYSTACK_FILE:
@@ -33,24 +31,24 @@ jobs:
               cvmfs_http_proxy: DIRECT
               cvmfs_repositories: pilot.eessi-hpc.org
 
-#        - name: Test check_missing_installations.sh script
-#          run: |
-#              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
-#              module load EasyBuild
-#              eb --version
-#              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
-#              export EESSI_OS_TYPE=linux
-#              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
-#              env | grep ^EESSI | sort
-#              echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
-#              ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}
+        - name: Test check_missing_installations.sh script
+          run: |
+              source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash
+              module load EasyBuild
+              eb --version
+              export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+              env | grep ^EESSI | sort
+              echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
+              ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}
 
         - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
           run: |
-              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+              source /cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
               eb --version
-              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_PREFIX=/cvmfs/pilot.nessi.no/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort

.github/workflows/tests_readme.yml

@@ -7,8 +7,6 @@ on:
       - init/eessi_defaults
 
   pull_request:
-    branches:
-      - main
     paths:
       - README.md
       - init/eessi_defaults

.github/workflows/tests_scripts.yml

@@ -13,8 +13,6 @@ on:
       - update_lmod_cache.sh
 
   pull_request:
-    branches:
-      - main
     paths:
       - build_container.sh
       - create_directory_tarballs.sh
@@ -42,7 +40,10 @@ jobs:
         # bind current directory into container as /software-layer
         export SINGULARITY_BIND="${PWD}:/software-layer"
 
-        for EB_VERSION in '4.5.0' '4.5.1' '4.7.2'; do
+        # can't test with EasyBuild versions older than v4.5.2 when using EESSI pilot 2023.06,
+        # since Python in compat layer is Python 3.11.x;
+        # testing with a single EasyBuild version takes a while in GitHub Actions, so stick to a single sensible version
+        for EB_VERSION in '4.6.0'; do
           # Create script that uses load_easybuild_module.sh which we can run in compat layer environment
           # note: Be careful with single vs double quotes below!
           #       ${EB_VERSION} should be expanded, so use double quotes;

EESSI-pilot-install-software.sh

@@ -50,59 +50,59 @@ set -- "${POSITIONAL_ARGS[@]}"
 
 TOPDIR=$(dirname $(realpath $0))
 
-source ${TOPDIR}/scripts/utils.sh
+source $TOPDIR/scripts/utils.sh
 
-# honor ${TMPDIR} if it is already defined, use /tmp otherwise
-if [ -z ${TMPDIR} ]; then
-    export WORKDIR=/tmp/${USER}
+# honor $TMPDIR if it is already defined, use /tmp otherwise
+if [ -z $TMPDIR ]; then
+    export WORKDIR=/tmp/$USER
 else
-    export WORKDIR=${TMPDIR}/${USER}
+    export WORKDIR=$TMPDIR/$USER
 fi
 
 TMPDIR=$(mktemp -d)
 
 echo ">> Setting up environment..."
 
-source ${TOPDIR}/init/minimal_eessi_env
+source $TOPDIR/init/minimal_eessi_env
 
-if [ -d ${EESSI_CVMFS_REPO} ]; then
-    echo_green "${EESSI_CVMFS_REPO} available, OK!"
+if [ -d $EESSI_CVMFS_REPO ]; then
+    echo_green "$EESSI_CVMFS_REPO available, OK!"
 else
-    fatal_error "${EESSI_CVMFS_REPO} is not available!"
+    fatal_error "$EESSI_CVMFS_REPO is not available!"
 fi
 
-# make sure we're in Prefix environment by checking ${SHELL}
+# make sure we're in Prefix environment by checking $SHELL
 if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then
     echo_green ">> It looks like we're in a Gentoo Prefix environment, good!"
 else
     fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
 fi
 
 # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
-export PYTHONPYCACHEPREFIX=${TMPDIR}/pycache
+export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
 
 DETECTION_PARAMETERS=''
 GENERIC=0
 EB='eb'
-if [[ "${EASYBUILD_OPTARCH}" == "GENERIC" ]]; then
+if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
     echo_yellow ">> GENERIC build requested, taking appropriate measures!"
-    DETECTION_PARAMETERS="${DETECTION_PARAMETERS} --generic"
+    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
     GENERIC=1
     EB='eb --optarch=GENERIC'
 fi
 
 echo ">> Determining software subdirectory to use for current build host..."
-if [ -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
-  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 ${TOPDIR}/eessi_software_subdir.py ${DETECTION_PARAMETERS})
-  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py ${DETECTION_PARAMETERS}' script"
+if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
 else
   echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
 fi
 
 # Set all the EESSI environment variables (respecting ${EESSI_SOFTWARE_SUBDIR_OVERRIDE})
 # $EESSI_SILENT - don't print any messages
 # $EESSI_BASIC_ENV - give a basic set of environment variables
-EESSI_SILENT=1 EESSI_BASIC_ENV=1 source ${TOPDIR}/init/eessi_environment_variables
+EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
 
 if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
     fatal_error "Failed to determine software subdirectory?!"
@@ -113,39 +113,38 @@ else
 fi
 
 echo ">> Initializing Lmod..."
-source ${EPREFIX}/usr/share/Lmod/init/bash
-ml_version_out=${TMPDIR}/ml.out
-ml --version &> ${ml_version_out}
+source $EPREFIX/usr/share/Lmod/init/bash
+ml_version_out=$TMPDIR/ml.out
+ml --version &> $ml_version_out
 if [[ $? -eq 0 ]]; then
     echo_green ">> Found Lmod ${LMOD_VERSION}"
 else
     fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
 fi
 
 echo ">> Configuring EasyBuild..."
-source ${TOPDIR}/configure_easybuild
+source $TOPDIR/configure_easybuild
 
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
 module --force purge
 # ignore current $MODULEPATH entirely
-module unuse ${MODULEPATH}
-module use ${EASYBUILD_INSTALLPATH}/modules/all
+module unuse $MODULEPATH
+module use $EASYBUILD_INSTALLPATH/modules/all
 if [[ -z ${MODULEPATH} ]]; then
     fatal_error "Failed to set up \$MODULEPATH?!"
 else
     echo_green ">> MODULEPATH set up: ${MODULEPATH}"
 fi
 
 for eb_version in '4.7.2'; do
+
     # load EasyBuild module (will be installed if it's not available yet)
     source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
 
     echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
 
-    for gen in '2021a'; do
-
-        es="eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-${gen}.yml"
+    for es in $(ls eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-*.yml); do
 
         if [ -f ${es} ]; then
             echo_green "Feeding easystack file ${es} to EasyBuild..."
@@ -157,16 +156,19 @@ for eb_version in '4.7.2'; do
             fatal_error "Easystack file ${es} not found!"
         fi
     done
+
 done
 
+### add packages here
+
 echo ">> Creating/updating Lmod cache..."
 export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua"
-if [ ! -f ${LMOD_RC} ]; then
-    python3 ${TOPDIR}/create_lmodrc.py ${EASYBUILD_INSTALLPATH}
-    check_exit_code $? "${LMOD_RC} created" "Failed to create ${LMOD_RC}"
+if [ ! -f $LMOD_RC ]; then
+    python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH}
+    check_exit_code $? "$LMOD_RC created" "Failed to create $LMOD_RC"
 fi
 
-${TOPDIR}/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
+$TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
 
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}

README.md

@@ -9,14 +9,14 @@ See also https://eessi.github.io/docs/software_layer.
 You can set up your environment by sourcing the init script:
 
 ```
-$ source /cvmfs/pilot.eessi-hpc.org/versions/2021.12/init/bash
-Found EESSI pilot repo @ /cvmfs/pilot.eessi-hpc.org/versions/2021.12!
-Derived subdirectory for software layer: x86_64/intel/haswell
-Using x86_64/intel/haswell subdirectory for software layer (HARDCODED)
+$ source /cvmfs/pilot.nessi.no/versions/2023.06/init/bash
+Found EESSI pilot repo @ /cvmfs/pilot.nessi.no/versions/2023.06!
+Derived subdirectory for software layer: x86_64/intel/broadwell
+Using x86_64/intel/broadwell subdirectory for software layer (HARDCODED)
 Initializing Lmod...
-Prepending /cvmfs/pilot.eessi-hpc.org/versions/2021.12/software/x86_64/intel/haswell/modules/all to $MODULEPATH...
+Prepending /cvmfs/pilot.nessi.no/versions/2023.06/software/x86_64/intel/broadwell/modules/all to $MODULEPATH...
 Environment set up to use EESSI pilot software stack, have fun!
-[EESSI pilot 2021.12] $
+[EESSI pilot 2023.06] $
 ```
 
 ### Accessing EESSI via a container