diff --git a/bot/build.sh b/bot/build.sh index de73faef0b..70d30a8c10 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -276,9 +276,12 @@ fi # Retain location for host injections so we don't reinstall CUDA # (Always need to run the driver installation as available driver may change) -if [[ ! -z ${SHARED_FS_PATH} ]]; then - BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections") -fi +# use $STORAGE for host-injections (maybe shared fs is not behaving well) +mkdir -p ${STORAGE}/host_injections +BUILD_STEP_ARGS+=("--host-injections" "${STORAGE}/host-injections") +#if [[ ! -z ${SHARED_FS_PATH} ]]; then +# BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections") +#fi # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index e2a6978e10..0ba57b5dd0 100644 --- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -82,3 +82,7 @@ easyconfigs: - BLAST+-2.14.1-gompi-2023a.eb: options: from-pr: 20784 + - PyTorch-bundle-2.1.2-foss-2023a.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20382 + options: + from-pr: 20382 diff --git a/eb_hooks.py b/eb_hooks.py index 849a135fa8..976c1ba777 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -5,8 +5,8 @@ import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import obtain_config_guess -from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES +from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option from easybuild.tools.filetools import apply_regex_substitutions, copy_file, remove_file, symlink, which @@ -299,6 +299,46 @@ def parse_hook_pybind11_replace_catch2(ec, eprefix): build_deps[idx] = (catch2_name, catch2_version) +def parse_hook_pytorch_bundle_torchvision_setenv(ec, eprefix): + """ + Set TORCHVISION_{INCLUDE,LIBRARY}, initially for non-CUDA version only + """ + if ec.name == 'PyTorch-bundle' and ec.version in ['2.1.2']: + if not hasattr(ec, 'versionsuffix') or (ec.versionsuffix and not 'CUDA' in ec.versionsuffix): + print_msg("parse_hook for PyTorch-bundle without CUDA: extslist '%s'", ec['exts_list']) + print_msg("New exts_list: '%s'", ec['exts_list']) + ec_dict = ec.asdict() + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') + libpng_root = os.path.join(eessi_software_path, "software", "libpng", "1.6.39-GCCcore-12.3.0") + libpng_include = os.path.join(libpng_root, 'include') + libpng_lib = os.path.join(libpng_root, 'lib') + libjpeg_turbo_root = os.path.join(eessi_software_path, "software", "libjpeg-turbo", "2.1.5.1-GCCcore-12.3.0") + libjpeg_turbo_include = os.path.join(libjpeg_turbo_root, 'include') + libjpeg_turbo_lib = os.path.join(libjpeg_turbo_root, 'lib') + exts_list_new = [] + torchvision_include = 'export TORCHVISION_INCLUDE=%s:%s' % (libpng_include, libjpeg_turbo_include) + torchvision_library = 'export TORCHVISION_LIBRARY=%s:%s' % (libpng_lib, libjpeg_turbo_lib) + for item in ec_dict['exts_list']: + if item[0] != 'torchvision': + exts_list_new.append(item) + else: + ext_dict = item[2] + if 'preinstallopts' in ext_dict: + raise EasyBuildError("found value for 'preinstallopts' for extension 'torchvision'," + " but expected NONE") + else: + # add preinstallopts + ext_dict['preinstallopts'] = torchvision_include + ' && ' + torchvision_library + ' && ' + exts_list_new.append((item[0], item[1], ext_dict)) + ec['exts_list'] = exts_list_new + print_msg("New exts_list: '%s'", ec['exts_list']) + else: + print_msg("parse_hook for PyTorch-bundle for CUDA -> leaving preinstallopts unchanged") + else: + raise EasyBuildError("PyTorch-bundle-specific hook triggered for non-PyTorch-bundle easyconfig?!") + + def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): """ Disable check for QtWebEngine in Qt5 as workaround for problem with determining glibc version. @@ -312,7 +352,6 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): raise EasyBuildError("Qt5-specific hook triggered for non-Qt5 easyconfig?!") - def parse_hook_sentencepiece_disable_tcmalloc_aarch64(ec, eprefix): """ Disable using TC_Malloc on 'aarch64/generic' @@ -943,6 +982,7 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'Pillow-SIMD' : parse_hook_Pillow_SIMD_harcoded_paths, 'pybind11': parse_hook_pybind11_replace_catch2, + 'PyTorch-bundle': parse_hook_pytorch_bundle_torchvision_setenv, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, 'SentencePiece': parse_hook_sentencepiece_disable_tcmalloc_aarch64, 'UCX': parse_hook_ucx_eprefix,