diff --git a/.gitignore b/.gitignore index b434e939d..92c37ac37 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ .a bin/ build/ -build-* +build-*/ .cache/* .cproject *.dSYM/ diff --git a/ci/baseimage.cuda.Dockerfile b/ci/baseimage.cuda.Dockerfile deleted file mode 100644 index 5fe7c1ea1..000000000 --- a/ci/baseimage.cuda.Dockerfile +++ /dev/null @@ -1,120 +0,0 @@ -FROM ubuntu:22.04 as builder - -ARG CUDA_ARCH=60 - -ENV DEBIAN_FRONTEND noninteractive - -ENV FORCE_UNSAFE_CONFIGURE 1 - -ENV PATH="/spack/bin:${PATH}" - -ENV MPICH_VERSION=3.4.3 - -ENV CMAKE_VERSION=3.26.3 - -RUN apt-get -y update - -RUN apt-get install -y apt-utils - -# install basic tools -RUN apt-get install -y --no-install-recommends gcc g++ gfortran clang libomp-14-dev git make unzip file \ - vim wget pkg-config python3-pip python3-dev cython3 python3-pythran curl tcl m4 cpio automake meson \ - xz-utils patch patchelf apt-transport-https ca-certificates gnupg software-properties-common perl tar bzip2 \ - liblzma-dev libbz2-dev - -# install CMake -RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz -O cmake.tar.gz && \ - tar zxvf cmake.tar.gz --strip-components=1 -C /usr - -# get latest version of spack -RUN git clone -b v0.21.2 https://github.com/spack/spack.git - -# add local repo to spack -COPY ./spack /opt/spack -RUN spack repo add --scope system /opt/spack - -# set the location of packages built by spack -RUN spack config add config:install_tree:root:/opt/local -# set cuda_arch for all packages -RUN spack config add packages:all:variants:cuda_arch=${CUDA_ARCH} - -# find all external packages -RUN spack external find --all - -# find compilers -RUN spack compiler find - -# install yq (utility to manipulate the yaml files) -RUN wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_386 && chmod a+x /usr/local/bin/yq - -# change the fortran compilers: for gcc the gfortran is already properly set and the change has no effect; add it for clang -RUN yq -i '.compilers[0].compiler.paths.f77 = "/usr/bin/gfortran"' /root/.spack/linux/compilers.yaml && \ - yq -i '.compilers[0].compiler.paths.fc = "/usr/bin/gfortran"' /root/.spack/linux/compilers.yaml && \ - yq -i '.compilers[1].compiler.paths.f77 = "/usr/bin/gfortran"' /root/.spack/linux/compilers.yaml && \ - yq -i '.compilers[1].compiler.paths.fc = "/usr/bin/gfortran"' /root/.spack/linux/compilers.yaml - -# install MPICH -RUN spack install mpich@${MPICH_VERSION} %gcc - -# install openmpi -RUN spack install openmpi %gcc - -# install libvdwxc -RUN spack install libvdwxc %gcc +mpi ^mpich@${MPICH_VERSION} - -# install openblas -RUN spack install openblas %gcc +fortran - -RUN spack install magma %gcc +cuda +fortran ^openblas - -RUN spack install nlcglib@master %gcc +cuda - -# for the MPI hook -RUN echo $(spack find --format='{prefix.lib}' mpich) > /etc/ld.so.conf.d/mpich.conf -RUN ldconfig - -# create environments for several configurations and install dependencies -RUN spack env create -d /sirius-env-clang && \ - spack -e /sirius-env-clang add "sirius@develop %clang build_type=RelWithDebInfo ~cuda ~fortran +tests +pugixml ^openblas%gcc ^libxc%gcc ^mpich%gcc " && \ - spack -e /sirius-env-clang develop -p /sirius-src sirius@develop && \ - spack -e /sirius-env-clang install --only=dependencies --fail-fast - -RUN spack env create -d /sirius-env-cuda && \ - spack -e /sirius-env-cuda add "sirius@develop %gcc build_type=RelWithDebInfo +scalapack +tests +pugixml +apps +cuda +magma +python ^netlib-scalapack ^mpich ^openblas threads=openmp" && \ - spack -e /sirius-env-cuda develop -p /sirius-src sirius@develop && \ - spack -e /sirius-env-cuda install --only=dependencies --fail-fast - -RUN spack env create -d /sirius-env-cuda-mkl-mpich && \ - spack -e /sirius-env-cuda-mkl-mpich add "sirius@develop %gcc build_type=RelWithDebInfo +tests +pugixml +apps +cuda +scalapack +magma ^mpich ^intel-oneapi-mkl+cluster" && \ - spack -e /sirius-env-cuda-mkl-mpich develop -p /sirius-src sirius@develop && \ - spack -e /sirius-env-cuda-mkl-mpich install --only=dependencies --fail-fast - -RUN spack env create -d /sirius-env-elpa && \ - spack -e /sirius-env-elpa add "sirius@develop %gcc build_type=RelWithDebInfo +tests +pugixml +apps +cuda +scalapack +elpa ^netlib-scalapack ^mpich ^openblas ^elpa+cuda" && \ - spack -e /sirius-env-elpa develop -p /sirius-src sirius@develop && \ - spack -e /sirius-env-elpa install --only=dependencies --fail-fast - -RUN spack env create -d /sirius-env-fp32 && \ - spack -e /sirius-env-fp32 add "sirius@develop %gcc build_type=RelWithDebInfo +tests +pugixml +apps +cuda ^mpich ^openblas ^elpa+cuda ^spfft+single_precision+cuda" && \ - spack -e /sirius-env-fp32 develop -p /sirius-src sirius@develop && \ - spack -e /sirius-env-fp32 install --only=dependencies --fail-fast - -RUN spack env create -d /sirius-env-nlcg && \ - spack -e /sirius-env-nlcg add "sirius@develop %gcc build_type=RelWithDebInfo +fortran +tests +pugixml +apps +cuda +nlcglib ^openblas ^mpich" && \ - spack -e /sirius-env-nlcg develop -p /sirius-src sirius@develop && \ - spack -e /sirius-env-nlcg install --only=dependencies --fail-fast - -RUN spack env create -d /sirius-env-openmpi && \ - spack -e /sirius-env-openmpi add "sirius@develop %gcc +tests +pugixml +apps +scalapack +fortran build_type=RelWithDebInfo ^netlib-scalapack ^openblas ^openmpi" && \ - spack -e /sirius-env-openmpi develop -p /sirius-src sirius@develop && \ - spack -e /sirius-env-openmpi install --only=dependencies --fail-fast - -RUN spack env create -d /sirius-env-cuda-sequential && \ - spack -e /sirius-env-cuda-sequential add "sirius@develop %gcc +cuda +tests +pugixml +apps +fortran build_type=RelWithDebInfo ^openblas ^openmpi" && \ - spack -e /sirius-env-cuda-sequential develop -p /sirius-src sirius@develop && \ - spack -e /sirius-env-cuda-sequential install --only=dependencies --fail-fast - -RUN spack env create -d /sirius-env-vdwxc-cuda && \ - spack -e /sirius-env-vdwxc-cuda add "sirius@develop %gcc build_type=RelWithDebInfo +fortran +tests +pugixml +apps +vdwxc +cuda +nlcglib ^openblas ^mpich +cuda" && \ - spack -e /sirius-env-vdwxc-cuda develop -p /sirius-src sirius@develop && \ - spack -e /sirius-env-vdwxc-cuda install --only=dependencies --fail-fast diff --git a/ci/baseimage.rocm.Dockerfile b/ci/baseimage.rocm.Dockerfile deleted file mode 100644 index 621235c9b..000000000 --- a/ci/baseimage.rocm.Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -FROM ubuntu:22.04 - -ARG ROCM_ARCH=gfx90a - -ENV DEBIAN_FRONTEND=noninteractive - -ENV FORCE_UNSAFE_CONFIGURE 1 - -ENV PATH="/spack/bin:${PATH}" - -ENV CMAKE_VERSION=3.26.3 - -RUN apt-get -y update && apt-get install -y apt-utils - -# install basic tools -RUN apt-get install -y gcc g++ gfortran clang git make unzip \ - vim wget pkg-config python3-pip python3-venv curl tcl m4 cpio automake \ - apt-transport-https ca-certificates gnupg software-properties-common \ - patchelf meson liblzma-dev libbz2-dev - -# install CMake -RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz -O cmake.tar.gz && \ - tar zxvf cmake.tar.gz --strip-components=1 -C /usr - -# get latest version of spack -RUN git clone -b v0.22.2 https://github.com/spack/spack.git - -# add local repo to spack -COPY ./spack /opt/spack -RUN spack repo add --scope system /opt/spack - -# set the location of packages built by spack -RUN spack config add config:install_tree:root:/opt/local -# set amdgpu_target for all packages -RUN spack config add packages:all:variants:amdgpu_target=${ROCM_ARCH} -# set basic x86_64 architecture -RUN spack config add packages:all:target:x86_64 - -# find gcc and clang compilers -RUN spack compiler find -RUN spack external find --all --exclude ncurses -# workaround hip wants to call /usr/bin/llvm-config, but ubuntu renamed it to /usr/bin/llvm-config-14 -RUN ln -s /usr/bin/llvm-config-14 /usr/bin/llvm-config - -# install big packages -RUN spack install --fail-fast hip%gcc -RUN spack install --fail-fast rocblas%gcc -RUN spack install --fail-fast rocsolver%gcc -RUN spack install --fail-fast hipfft%gcc - -RUN spack env create -d /sirius-env-rocm && \ - spack -e /sirius-env-rocm add "sirius@develop %gcc build_type=Release +scalapack +fortran +tests +pugixml +rocm ^openblas ^mpich ^spfft ^umpire+rocm~device_alloc" && \ - spack -e /sirius-env-rocm develop -p /sirius-src sirius@develop && \ - spack -e /sirius-env-rocm install --only=dependencies --fail-fast diff --git a/ci/build-cpu.sh b/ci/build-cpu.sh new file mode 100755 index 000000000..2117439df --- /dev/null +++ b/ci/build-cpu.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -xeuo pipefail + +export SPACK_SYSTEM_CONFIG_PATH=/user-environment/config + +# make sure we keep the stage direcorty +spack config --scope=user add config:build_stage:/dev/shm/spack-stage + +spack env create -d ./spack-env +# add local repository with current sirius recipe +spack -e ./spack-env repo add $REPO + +spack -e ./spack-env add $SPEC + +# build sirius from source +spack -e ./spack-env develop -p $PWD sirius@develop + +# display spack.yaml +cat ./spack-env/spack.yaml + +spack -e ./spack-env concretize +spack -e ./spack-env install + +# the tar pipe below expects a relative path +builddir=$(spack -e ./spack-env location -b sirius) +# create a symlink to spack build directory (keep in artifacts) +tar -cf builddir.tar $builddir diff --git a/ci/build-cuda.sh b/ci/build-cuda.sh new file mode 100755 index 000000000..39cb7b8b1 --- /dev/null +++ b/ci/build-cuda.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -xeuo pipefail + +export SPACK_SYSTEM_CONFIG_PATH=/user-environment/config + +# make sure we keep the stage direcorty +spack config --scope=user add config:build_stage:/dev/shm/spack-stage + +spack env create -d ./spack-env +# add local repository with current sirius recipe +spack -e ./spack-env repo add $REPO + +spack -e ./spack-env config add "packages:all:variants:[cuda_arch=${CUDA_ARCH},+cuda]" + +spack -e ./spack-env add $SPEC + +# build sirius from source +spack -e ./spack-env develop -p $PWD sirius@develop + +# display spack.yaml +cat ./spack-env/spack.yaml + +spack -e ./spack-env concretize +spack -e ./spack-env install + +# the tar pipe below expects a relative path +builddir=$(spack -e ./spack-env location -b sirius) +# create a symlink to spack build directory (keep in artifacts) +tar -cf builddir.tar $builddir diff --git a/ci/cscs-daint.yml b/ci/cscs-daint.yml index 8b92820fd..47dc10ee6 100644 --- a/ci/cscs-daint.yml +++ b/ci/cscs-daint.yml @@ -6,239 +6,155 @@ stages: - build - test -build base cuda image: - extends: .container-builder-dynamic-name +build base uenv gh200: + extends: .uenv-builder-daint-gh200 stage: baseimage - timeout: 10h + timeout: 2h variables: - DOCKERFILE: ci/baseimage.cuda.Dockerfile - WATCH_FILECHANGES: ci/baseimage.cuda.Dockerfile - PERSIST_IMAGE_NAME: $CSCS_REGISTRY_PATH/base/sirius-ci - DOCKER_BUILD_ARGS: '["CUDA_ARCH=60"]' + UENV_NAME: sirius-gh200 + UENV_RECIPE: ci/sirius-uenv-recipe-gh200/ + UENV_VERSION: v1 -build base rocm image: - extends: .container-builder-dynamic-name +build base uenv zen2: + extends: .uenv-builder-eiger-zen2 stage: baseimage - # rocm takes long to build - timeout: 4h + timeout: 2h variables: - DOCKERFILE: ci/baseimage.rocm.Dockerfile - WATCH_FILECHANGES: ci/baseimage.rocm.Dockerfile - PERSIST_IMAGE_NAME: $CSCS_REGISTRY_PATH/base/sirius-ci - DOCKER_BUILD_ARGS: '["ROCM_ARCH=gfx90a"]' + UENV_NAME: sirius-zen2 + UENV_RECIPE: ci/sirius-uenv-recipe-cpu/ + UENV_VERSION: v1 -build cuda image mkl: - extends: .container-builder - needs: ["build base cuda image"] +build sirius cuda: + extends: .uenv-runner-daint-gh200 stage: build - variables: - DOCKERFILE: ci/build.Dockerfile - PERSIST_IMAGE_NAME: discard - ENVPATH: "/sirius-env-cuda-mkl-mpich" - DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]' - -build cuda image: - extends: .container-builder - needs: ["build base cuda image"] - stage: build - variables: - CSCS_REBUILD_POLICY: always - DOCKERFILE: ci/build.Dockerfile - PERSIST_IMAGE_NAME: $CSCS_REGISTRY_PATH/sirius/sirius-ci:$CI_COMMIT_SHA - ENVPATH: "/sirius-env-cuda" - DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]' - -build rocm image: - extends: .container-builder - needs: ["build base rocm image"] - stage: build - variables: - DOCKERFILE: ci/build.Dockerfile - PERSIST_IMAGE_NAME: discard - ENVPATH: "/sirius-env-rocm" - DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]' - -build elpa cuda image: - extends: .container-builder - needs: ["build base cuda image"] - stage: build - variables: - DOCKERFILE: ci/build.Dockerfile - PERSIST_IMAGE_NAME: discard - ENVPATH: "/sirius-env-elpa" - DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]' - -build sequential eigen-solver cuda image: - extends: .container-builder - needs: ["build base cuda image"] - stage: build - variables: - DOCKERFILE: ci/build.Dockerfile - PERSIST_IMAGE_NAME: discard - ENVPATH: "/sirius-env-cuda-sequential" - DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]' - -build fp32 cuda image: - extends: .container-builder - needs: ["build base cuda image"] - stage: build - variables: - DOCKERFILE: ci/build.Dockerfile - PERSIST_IMAGE_NAME: discard - ENVPATH: "/sirius-env-fp32" - DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]' - -build vdwxc cuda image: - extends: .container-builder - needs: ["build base cuda image"] - stage: build - variables: - DOCKERFILE: ci/build.Dockerfile - PERSIST_IMAGE_NAME: discard - # we can't use MKL here because vdwxc needs parallel FFT and MKL doesn't provide it - ENVPATH: "/sirius-env-vdwxc-cuda" - DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]' - -build nlcg image: - extends: .container-builder - needs: ["build base cuda image"] - stage: build - variables: - DOCKERFILE: ci/build.Dockerfile - PERSIST_IMAGE_NAME: discard - ENVPATH: "/sirius-env-nlcg" - DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]' - -build clang image: - extends: .container-builder - needs: ["build base cuda image"] - stage: build - variables: - DOCKERFILE: ci/build.Dockerfile - PERSIST_IMAGE_NAME: discard - # reuse openblas, libxc and spfft from gcc build - ENVPATH: "/sirius-env-clang" - DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]' - -build openmpi image: - extends: .container-builder - needs: ["build base cuda image"] + needs: ["build base uenv gh200"] + image: sirius-gh200/v1:$CI_PIPELINE_ID + artifacts: + paths: + - builddir.tar + variables: + SPEC: sirius@develop +cuda +apps +tests +scalapack ~python + CUDA_ARCH: 90 + REPO: ./ci/sirius-uenv-recipe-gh200/repo + script: + - | + # + git clone --filter=tree:0 $(jq -r .spack.repo /user-environment/meta/configure.json) /dev/shm/spack-clone + git -C /dev/shm/spack-clone checkout $(jq -r .spack.commit /user-environment/meta/configure.json) + source /dev/shm/spack-clone/share/spack/setup-env.sh + ./ci/build-cuda.sh + +build sirius eiger: + extends: .uenv-runner-eiger-zen2 stage: build - variables: - DOCKERFILE: ci/build.Dockerfile - PERSIST_IMAGE_NAME: discard - ENVPATH: "/sirius-env-openmpi" - DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]' + needs: ["build base uenv zen2"] + image: sirius-zen2/v1:$CI_PIPELINE_ID + artifacts: + paths: + - builddir.tar + variables: + SPEC: sirius@develop +apps +tests +scalapack ~python + REPO: ./ci/sirius-uenv-recipe-cpu/repo + script: + - | + # + git clone --filter=tree:0 $(jq -r .spack.repo /user-environment/meta/configure.json) /dev/shm/spack-clone + git -C /dev/shm/spack-clone checkout $(jq -r .spack.commit /user-environment/meta/configure.json) + source /dev/shm//spack-clone/share/spack/setup-env.sh + ./ci/build-cpu.sh .run_tests: - extends: .container-runner-daint-gpu - needs: ["build cuda image"] - stage: test script: - - cd /sirius-src/spack-build - | + if [ $SLURM_LOCALID -eq 0 ]; then + tar xf ./builddir.tar -C / + touch /dev/shm/unpack_done_$CI_JOB_ID + fi + - | + # wait for tar unpack + while [ ! -f /dev/shm/unpack_done_$CI_JOB_ID ]; do + sleep 0.2 + done + + # ls -alhrt /dev/shm/ + # ls -alhrt /dev/shm//spack-stage/antonk/* + + stagedir=$(find /dev/shm/spack-stage -type d -name spack-stage-sirius-*) + echo "stagedir: $stagedir" + cd $stagedir/spack-build-* if [ "$SLURM_PROCID" == "0" ]; then - $TEST_COMMAND -V + $TEST_COMMAND -V else - $TEST_COMMAND --output-on-failure + $TEST_COMMAND --output-on-failure fi - image: $CSCS_REGISTRY_PATH/sirius/sirius-ci:$CI_COMMIT_SHA - variables: - CRAY_CUDA_MPS: 1 - GIT_STRATEGY: none - MPICH_MAX_THREAD_SAFETY: multiple - CSCS_REGISTRY_LOGIN: 'YES' - PULL_IMAGE: 'YES' - SLURM_HINT: nomultithread - SLURM_JOB_NUM_NODES: 1 - SLURM_UNBUFFEREDIO: '' - SLURM_WAIT: 0 +.run_tests_daint: + extends: [.uenv-runner-daint-gh200, .run_tests] + image: sirius-gh200/v1:$CI_PIPELINE_ID + stage: test + needs: ["build sirius cuda"] +.run_tests_eiger: + extends: [.uenv-runner-eiger-zen2, .run_tests] + image: sirius-zen2/v1:$CI_PIPELINE_ID + stage: test + needs: ["build sirius eiger"] + variables: + SLURM_HINT: nomultithread + SLURM_CPU_BIND: verbose -gpu serial: - extends: .run_tests +daint test gpu serial: + extends: [.run_tests_daint] variables: - OMP_NUM_THREADS: 12 - SLURM_CONSTRAINT: gpu - SLURM_CPUS_PER_TASK: 12 + OMP_NUM_THREADS: 20 + SLURM_TIMELIMIT: '60:00' + SLURM_GPUS_PER_TASK: 1 + SLURM_GPUS: 1 SLURM_NTASKS: 1 - SLURM_TIMELIMIT: "60:00" + SLURM_CPU_BIND: 'socket' + USE_MPI: 'YES' TEST_COMMAND: ctest -L gpu_serial -gpu band parallel: - extends: .run_tests +daint test gpu band parallel: + extends: [.run_tests_daint] variables: - OMP_NUM_THREADS: 3 - SLURM_CONSTRAINT: gpu - SLURM_CPUS_PER_TASK: 3 + OMP_NUM_THREADS: 20 + SLURM_TIMELIMIT: '60:00' + SLURM_GPUS_PER_TASK: 1 SLURM_NTASKS: 4 - SLURM_TIMELIMIT: "60:00" - TEST_COMMAND: ctest -L gpu_band_parallel --timeout 3600 + SLURM_GPUS: 4 + SLURM_CPU_BIND: 'socket' USE_MPI: 'YES' + TEST_COMMAND: ctest -L gpu_band_parallel --timeout 3600 -gpu k-point parallel: - extends: .run_tests +daint test gpu k-point parallel: + extends: [.run_tests_daint] variables: - OMP_NUM_THREADS: 3 + OMP_NUM_THREADS: 20 SLURM_CONSTRAINT: gpu - SLURM_CPUS_PER_TASK: 3 SLURM_NTASKS: 4 - SLURM_TIMELIMIT: "60:00" - TEST_COMMAND: ctest -L gpu_k_point_parallel + SLURM_GPUS: 4 + SLURM_GPUS_PER_TASK: 1 + SLURM_CPU_BIND: 'socket' USE_MPI: 'YES' + TEST_COMMAND: ctest -L gpu_k_point_parallel -cpu single: - extends: .run_tests +eiger test cpu serial: + extends: [.run_tests_eiger] variables: - OMP_NUM_THREADS: 12 - SLURM_CONSTRAINT: gpu - SLURM_CPU_BIND: sockets - SLURM_CPUS_PER_TASK: 12 + OMP_NUM_THREADS: 8 + SLURM_TIMELIMIT: '60:00' SLURM_NTASKS: 1 - SLURM_TIMELIMIT: "60:00" + SRUN_CPUS_PER_TASK: 8 + USE_MPI: 'YES' TEST_COMMAND: ctest -L cpu_serial -cpu band parallel: - extends: .run_tests +eiger test cpu band parallel: + extends: [.run_tests_eiger] variables: - OMP_NUM_THREADS: 3 - SLURM_CONSTRAINT: gpu - SLURM_CPU_BIND: sockets - SLURM_CPUS_PER_TASK: 3 + OMP_NUM_THREADS: 8 + SLURM_TIMELIMIT: '60:00' SLURM_NTASKS: 4 - SLURM_TIMELIMIT: "60:00" - TEST_COMMAND: ctest -L cpu_band_parallel --timeout 3600 - USE_MPI: 'YES' - -python: - extends: .container-runner-daint-gpu - needs: ["build cuda image"] - stage: test - script: - - | - # spack load etc fails, - PYTHONPATH=$(spack -C /root/.spack find --format={prefix} sirius)/lib/python3.10/site-packages - PYTHONPATH=/sirius-env-cuda/.spack-env/view/local/lib/python3.10/dist-packages/:$PYTHONPATH - export PYTHONPATH - pip install pytest - cd /sirius-src/python_module/test/ && pytest . - image: $CSCS_REGISTRY_PATH/sirius/sirius-ci:$CI_COMMIT_SHA - variables: - CRAY_CUDA_MPS: 1 - GIT_STRATEGY: none - MPICH_MAX_THREAD_SAFETY: multiple - CSCS_REGISTRY_LOGIN: 'YES' - PULL_IMAGE: 'YES' - SLURM_HINT: nomultithread - SLURM_JOB_NUM_NODES: 1 - SLURM_UNBUFFEREDIO: '' - SLURM_WAIT: 0 - OMP_NUM_THREADS: 3 - SLURM_CONSTRAINT: gpu - SLURM_CPU_BIND: sockets - SLURM_CPUS_PER_TASK: 3 - SLURM_NTASKS: 1 - SLURM_TIMELIMIT: "60:00" - TEST_COMMAND: pytest + SRUN_CPUS_PER_TASK: 8 USE_MPI: 'YES' + TEST_COMMAND: ctest -L cpu_band_parallel diff --git a/ci/sirius-uenv-recipe-cpu/compilers.yaml b/ci/sirius-uenv-recipe-cpu/compilers.yaml new file mode 100644 index 000000000..281fae8b9 --- /dev/null +++ b/ci/sirius-uenv-recipe-cpu/compilers.yaml @@ -0,0 +1,5 @@ +bootstrap: + spec: gcc@12.3 +gcc: + specs: + - gcc@13 diff --git a/ci/sirius-uenv-recipe-cpu/config.yaml b/ci/sirius-uenv-recipe-cpu/config.yaml new file mode 100644 index 000000000..2c23bb673 --- /dev/null +++ b/ci/sirius-uenv-recipe-cpu/config.yaml @@ -0,0 +1,6 @@ +name: sirius-uenv-base +spack: + commit: releases/v0.23 + repo: https://github.com/spack/spack.git +store: /user-environment +description: SIRIUS dependencies for SIRIUS CI diff --git a/ci/sirius-uenv-recipe-cpu/environments.yaml b/ci/sirius-uenv-recipe-cpu/environments.yaml new file mode 100644 index 000000000..c490180a4 --- /dev/null +++ b/ci/sirius-uenv-recipe-cpu/environments.yaml @@ -0,0 +1,19 @@ +gcc-env: + compiler: + - toolchain: gcc + spec: gcc + mpi: + spec: cray-mpich@8.1.30 + unify: when_possible + specs: + - sirius@develop ~python +vdwxc +apps +tests +dlaf +scalapack + - sirius@develop +elpa + variants: + - +mpi + views: + sirius: + link: roots + uenv: + add_compilers: true + prefix_paths: + LD_LIBRARY_PATH: [lib, lib64] diff --git a/ci/sirius-uenv-recipe-cpu/modules.yaml b/ci/sirius-uenv-recipe-cpu/modules.yaml new file mode 100644 index 000000000..623307b09 --- /dev/null +++ b/ci/sirius-uenv-recipe-cpu/modules.yaml @@ -0,0 +1,23 @@ +modules: + # Paths to check when creating modules for all module sets + prefix_inspections: + bin: + - PATH + lib: + - LD_LIBRARY_PATH + lib64: + - LD_LIBRARY_PATH + + default: + arch_folder: false + # Where to install modules + roots: + tcl: /user-environment/modules + tcl: + all: + autoload: none + hash_length: 0 + exclude_implicits: true + exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0'] + projections: + all: '{name}/{version}' diff --git a/ci/sirius-uenv-recipe-cpu/repo/packages b/ci/sirius-uenv-recipe-cpu/repo/packages new file mode 120000 index 000000000..fee95275e --- /dev/null +++ b/ci/sirius-uenv-recipe-cpu/repo/packages @@ -0,0 +1 @@ +../../../spack/packages \ No newline at end of file diff --git a/ci/sirius-uenv-recipe-cpu/repo/repo.yaml b/ci/sirius-uenv-recipe-cpu/repo/repo.yaml new file mode 100644 index 000000000..b40ff6a01 --- /dev/null +++ b/ci/sirius-uenv-recipe-cpu/repo/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: sirius-ci diff --git a/ci/sirius-uenv-recipe-gh200/compilers.yaml b/ci/sirius-uenv-recipe-gh200/compilers.yaml new file mode 100644 index 000000000..281fae8b9 --- /dev/null +++ b/ci/sirius-uenv-recipe-gh200/compilers.yaml @@ -0,0 +1,5 @@ +bootstrap: + spec: gcc@12.3 +gcc: + specs: + - gcc@13 diff --git a/ci/sirius-uenv-recipe-gh200/config.yaml b/ci/sirius-uenv-recipe-gh200/config.yaml new file mode 100644 index 000000000..2c23bb673 --- /dev/null +++ b/ci/sirius-uenv-recipe-gh200/config.yaml @@ -0,0 +1,6 @@ +name: sirius-uenv-base +spack: + commit: releases/v0.23 + repo: https://github.com/spack/spack.git +store: /user-environment +description: SIRIUS dependencies for SIRIUS CI diff --git a/ci/sirius-uenv-recipe-gh200/environments.yaml b/ci/sirius-uenv-recipe-gh200/environments.yaml new file mode 100644 index 000000000..68861a14d --- /dev/null +++ b/ci/sirius-uenv-recipe-gh200/environments.yaml @@ -0,0 +1,22 @@ +gcc-env: + compiler: + - toolchain: gcc + spec: gcc + mpi: + spec: cray-mpich@8.1.30 + gpu: cuda + unify: when_possible + specs: + - sirius@develop ~python +vdwxc +apps +tests +dlaf +scalapack + - sirius@develop +elpa + variants: + - +mpi + - +cuda + - cuda_arch=90 + views: + sirius: + link: roots + uenv: + add_compilers: true + prefix_paths: + LD_LIBRARY_PATH: [lib, lib64] diff --git a/ci/sirius-uenv-recipe-gh200/modules.yaml b/ci/sirius-uenv-recipe-gh200/modules.yaml new file mode 100644 index 000000000..623307b09 --- /dev/null +++ b/ci/sirius-uenv-recipe-gh200/modules.yaml @@ -0,0 +1,23 @@ +modules: + # Paths to check when creating modules for all module sets + prefix_inspections: + bin: + - PATH + lib: + - LD_LIBRARY_PATH + lib64: + - LD_LIBRARY_PATH + + default: + arch_folder: false + # Where to install modules + roots: + tcl: /user-environment/modules + tcl: + all: + autoload: none + hash_length: 0 + exclude_implicits: true + exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0'] + projections: + all: '{name}/{version}' diff --git a/ci/sirius-uenv-recipe-gh200/repo/packages b/ci/sirius-uenv-recipe-gh200/repo/packages new file mode 120000 index 000000000..fee95275e --- /dev/null +++ b/ci/sirius-uenv-recipe-gh200/repo/packages @@ -0,0 +1 @@ +../../../spack/packages \ No newline at end of file diff --git a/ci/sirius-uenv-recipe-gh200/repo/repo.yaml b/ci/sirius-uenv-recipe-gh200/repo/repo.yaml new file mode 100644 index 000000000..b40ff6a01 --- /dev/null +++ b/ci/sirius-uenv-recipe-gh200/repo/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: sirius-ci