Images update with VeloC

.github/actions/build/action.yml

@@ -134,3 +134,4 @@ runs:
         sudo swapoff /swapfile || true
         sudo rm -f /swapfile || true
       fi
+

.github/workflows/debuntu_docker_images.yml

@@ -28,14 +28,16 @@ jobs:
         echo "distro_version=${BASE##*/}" >> "${GITHUB_OUTPUT}"
     - uses: actions/checkout@v6
       with: { submodules: 'recursive' }
-    - name: "Build and push"
+    - name: "Build and no push"
       uses: pdidev/test_env/.github/actions/build@v4
       with:
         GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         context: "./"
         path: debuntu/${{ matrix.variant }}
         image-name: ${{ matrix.base }}/${{ matrix.mpi }}/${{ matrix.variant }}
         test: "${{ matrix.variant != 'pdi' }}" # should be true, except for "pdi" variant
+        push : "false"
+        cache-to: "type=registry,ref=ghcr.io/iole-bolognesi/test_env/cache/${{ matrix.base }}/${{ matrix.mpi }}/${{ matrix.variant }},mode=max"
         build-args: |
           MPI=${{ matrix.mpi }}
           DISTRO=${{ steps.info.outputs.distro }}

.github/workflows/run_clang_format.yml

@@ -27,4 +27,4 @@ jobs:
         context: ./
         path: run_clang_format
         image-name: run-clang-format
-        tags: ${{ steps.info.outputs.tags }}
+        tags: ${{ steps.info.outputs.tags }}

.github/workflows/spack_docker_images.yml

@@ -27,6 +27,8 @@ jobs:
         context: "./"
         path: spack/1-base
         image-name: base
+        push : "false"
+        cache-to: "type=registry,ref=ghcr.io/iole-bolognesi/test_env/cache/${{ matrix.base }}/${{ matrix.mpi }}/${{ matrix.variant }},mode=max"
         cache: ${{ github.event_name != 'schedule' && ( github.event_name != 'workflow_dispatch' || inputs.cache ) }} # cache for every run but scheduled ones
     - name: "Build and push 2- spackbase"
       uses: pdidev/test_env/.github/actions/build@v4
@@ -35,6 +37,8 @@ jobs:
         context: "./"
         path: spack/2-spackbase
         image-name: spackbase
+        push : "false"
+        cache-to: "type=registry,ref=ghcr.io/iole-bolognesi/test_env/cache/${{ matrix.base }}/${{ matrix.mpi }}/${{ matrix.variant }},mode=max"
   step_3_baselibs:
     strategy:
       fail-fast: false
@@ -53,6 +57,8 @@ jobs:
         context: "./"
         path: spack/3-baselibs
         image-name: builder:${{ matrix.version }}-${{ matrix.compiler }}-baselibs
+        push : "false"
+        cache-to: "type=registry,ref=ghcr.io/iole-bolognesi/test_env/cache/${{ matrix.base }}/${{ matrix.mpi }}/${{ matrix.variant }},mode=max"
         build-args: |
           VERSION=${{ matrix.version }}
           COMPILER=${{ matrix.compiler }}
@@ -75,6 +81,8 @@ jobs:
         context: "./"
         path: spack/4-mini
         image-name: builder:${{ matrix.version }}-${{ matrix.compiler }}-${{ matrix.mpi }}-mini
+        push : "false"
+        cache-to: "type=registry,ref=ghcr.io/iole-bolognesi/test_env/cache/${{ matrix.base }}/${{ matrix.mpi }}/${{ matrix.variant }},mode=max"
         build-args: |
           VERSION=${{ matrix.version }}
           COMPILER=${{ matrix.compiler }}
@@ -86,6 +94,8 @@ jobs:
         context: "./"
         path: spack/5-all
         image-name: builder:${{ matrix.version }}-${{ matrix.compiler }}-${{ matrix.mpi }}-all
+        push : "false"
+        cache-to: "type=registry,ref=ghcr.io/iole-bolognesi/test_env/cache/${{ matrix.base }}/${{ matrix.mpi }}/${{ matrix.variant }},mode=max"
         build-args: |
           VERSION=${{ matrix.version }}
           COMPILER=${{ matrix.compiler }}
@@ -111,14 +121,16 @@ jobs:
           echo "tags=ghcr.io/pdidev/spack/${{ matrix.version }}/${{ matrix.compiler }}/${{ matrix.mpi }}/${{ matrix.level }}:${{github.ref_name}}" >> "${GITHUB_OUTPUT}"
           echo "fullspack=ghcr.io/pdidev/fullspack/${{ matrix.version }}/${{ matrix.compiler }}/${{ matrix.mpi }}/${{ matrix.level }}:${{github.ref_name}}" >> "${GITHUB_OUTPUT}"
         fi
-    - name: "Build and push"
+    - name: "Build and no push"
       uses: pdidev/test_env/.github/actions/build@v4
       with:
         GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
         context: "./"
         path: spack/6-finalize
         image-name: spack/${{ matrix.version }}/${{ matrix.compiler }}/${{ matrix.mpi }}/${{ matrix.level }}
         test: true
+        push : "false"
+        cache-to: "type=registry,ref=ghcr.io/iole-bolognesi/test_env/cache/${{ matrix.base }}/${{ matrix.mpi }}/${{ matrix.variant }},mode=max"
         build-args: |
           VERSION=${{ matrix.version }}
           COMPILER=${{ matrix.compiler }}
@@ -134,10 +146,12 @@ jobs:
         path: spack/6-finalize
         image-name: fullspack/${{ matrix.version }}/${{ matrix.compiler }}/${{ matrix.mpi }}/${{ matrix.level }}
         target: fullspack
+        push : "false"
+        cache-to: "type=registry,ref=ghcr.io/iole-bolognesi/test_env/cache/${{ matrix.base }}/${{ matrix.mpi }}/${{ matrix.variant }},mode=max"
         build-args: |
           VERSION=${{ matrix.version }}
           COMPILER=${{ matrix.compiler }}
           MPI=${{ matrix.mpi }}
           LIBS=${{ matrix.level }}
         tags: |
-          ${{ steps.info.outputs.fullspack }}
+          ${{ steps.info.outputs.fullspack }}

README.md

@@ -44,7 +44,7 @@ With the following parameters:
 * `variant`:
   - `mini`: dependencies "vendored" in PDI are not included in the image,
   - `all`: dependencies "vendored" in PDI are included in the image,
-  - `pdi`: PDI is included in the image.
+  - `pdi`: PDI is included in the image. The VeloC plugin is not present in this image for the time being. 
 
 
 ## Full Spack images

debuntu/all/Dockerfile

@@ -30,6 +30,7 @@ RUN set -ex \
  && apt-get update -y \
  && apt-get upgrade -y \
  && apt-get install -y --no-install-recommends \
+            git \
             apt-transport-https \
             ca-certificates \
             curl \
@@ -40,6 +41,8 @@ RUN set -ex \
  && apt-get install -y --no-install-recommends \
             build-essential \
             cmake \
+            libboost-dev \
+            libssl-dev \
             doxygen \
             gfortran \
             libhdf5-${MPI}-dev \
@@ -70,6 +73,68 @@ RUN set -ex \
  && rm -rf /root/* \
  && [ -f ~/.profile ] && ( sed -i 's/mesg n/( tty -s \&\& mesg n || true )/g' ~/.profile || true )
 
+# stage : veloc download 
+
+FROM main AS velocdl
+
+RUN if pip3 install --help | grep -q -- --break-system-packages; then \
+      pip3 install --break-system-packages beautifulsoup4 wget; \
+    else \
+      pip3 install beautifulsoup4 wget; \
+    fi
+
+WORKDIR /usr/veloc-src
+RUN git clone https://github.com/ECP-VeloC/VELOC.git . \
+    && mkdir -p /usr/local \
+    && python3 auto-install.py /usr/local
+
+# stage : mpich download enabling options for shared memory support 
+# shared mem support is needed for VeloC  
+FROM main AS mpichdl
+ARG MPI
+ARG DISTRO
+ARG DISTRO_VERSION
+
+RUN set -ex \
+ && if [ "${MPI}" = "mpich" ] && [ "${DISTRO}" = "debian" ] && [ "${DISTRO_VERSION}" = "unstable" ]; then \
+      apt-get update -y \
+      && apt-get install -y --no-install-recommends dpkg-dev \
+      && printf "Types: deb-src\nURIs: http://deb.debian.org/debian\nSuites: ${DISTRO_VERSION}\nComponents: main\n" \
+         > /etc/apt/sources.list.d/debian-src.sources \
+      && apt-get update -y \
+      && apt-get source mpich \
+      && cd mpich-* \
+      && ./configure \
+           --prefix=/usr/local \
+           --with-ch4-shmmods=posix \
+           --with-device=ch4:ucx \
+           --enable-shared \
+           --disable-static \
+           FC=gfortran \
+      && make -j$(nproc) \
+      && make install \
+      && ldconfig; \
+    fi
+
+# stage : complete image
+
+FROM main AS complete-main
+ARG DISTRO
+ARG DISTRO_VERSION
+ARG MPI
+COPY --from=velocdl /usr/local /usr/local_veloc
+COPY --from=mpichdl /usr/local /usr/local_mpich
+RUN if [ "${MPI}" = "mpich" ] && [ "${DISTRO}" = "debian" ] && [ "${DISTRO_VERSION}" = "unstable" ]; then \
+      cp -a /usr/local_mpich/. /usr/local/ \
+      && rm -rf /usr/local_mpich; \
+    else \
+      rm -rf /usr/local_mpich; \
+    fi
+RUN printf "/usr/local_veloc/lib" > /etc/ld.so.conf.d/veloc.conf \
+    && ldconfig
+
+ENV CMAKE_PREFIX_PATH=/usr/local_veloc:${CMAKE_PREFIX_PATH}
+
 ENV PDI_SYSTEM=${DISTRO}-${DISTRO_VERSION}
 ENV PDI_COMPILER=gcc
 ENV PDI_MPI=${MPI}
@@ -82,21 +147,19 @@ ENV HOME=/home/ci
 USER ci:ci
 WORKDIR ${HOME}
 
-
+# stage : pdi download 
 
 FROM alpine/git AS pdidl
 
-RUN git -C /root clone --recurse-submodules https://github.com/pdidev/pdi.git /root/pdi
-
+RUN git -C /root clone --recurse-submodules https://github.com/iole-bolognesi/pdi.git /root/pdi
 
+# stage : test 
 
-FROM main AS test
+FROM complete-main AS test
 
 COPY --from=pdidl --chown=ci:ci /root/pdi /home/ci/pdi
 
 RUN export VERBOSE=1 \
  && bash pdi/bin/build_and_run_all_tests
 
-
-
-FROM main
+FROM complete-main

debuntu/mini/Dockerfile

@@ -34,6 +34,9 @@ RUN set -ex \
             build-essential \
             cmake \
             doxygen \
+            git \
+            libboost-dev \
+            libssl-dev \
             gfortran \
             libcurl4-openssl-dev \
             lib${MPI}-dev \
@@ -61,6 +64,68 @@ RUN set -ex \
  && rm -rf /root/* \
  && [ -f ~/.profile ] && ( sed -i 's/mesg n/( tty -s \&\& mesg n || true )/g' ~/.profile || true )
 
+# stage : veloc download 
+
+FROM main AS velocdl
+
+RUN if pip3 install --help | grep -q -- --break-system-packages; then \
+      pip3 install --break-system-packages beautifulsoup4 wget; \
+    else \
+      pip3 install beautifulsoup4 wget; \
+    fi
+
+WORKDIR /usr/veloc-src
+RUN git clone https://github.com/ECP-VeloC/VELOC.git . \
+    && mkdir -p /usr/local \
+    && python3 auto-install.py /usr/local
+
+# stage : mpich download (with options for shared memory support)
+
+FROM main AS mpichdl
+ARG MPI
+ARG DISTRO
+ARG DISTRO_VERSION
+
+RUN set -ex \
+ && if [ "${MPI}" = "mpich" ] && [ "${DISTRO}" = "debian" ] && [ "${DISTRO_VERSION}" = "unstable" ]; then \
+      apt-get update -y \
+      && apt-get install -y --no-install-recommends dpkg-dev \
+      && printf "Types: deb-src\nURIs: http://deb.debian.org/debian\nSuites: ${DISTRO_VERSION}\nComponents: main\n" \
+         > /etc/apt/sources.list.d/debian-src.sources \
+      && apt-get update -y \
+      && apt-get source mpich \
+      && cd mpich-* \
+      && ./configure \
+           --prefix=/usr/local \
+           --with-ch4-shmmods=posix \
+           --with-device=ch4:ucx \
+           --enable-shared \
+           --disable-static \
+           FC=gfortran \
+      && make -j$(nproc) \
+      && make install \
+      && ldconfig; \
+    fi
+
+# stage : complete image
+
+FROM main AS complete-main
+ARG DISTRO
+ARG DISTRO_VERSION
+ARG MPI
+COPY --from=velocdl /usr/local /usr/local_veloc
+COPY --from=mpichdl /usr/local /usr/local_mpich
+RUN if [ "${MPI}" = "mpich" ] && [ "${DISTRO}" = "debian" ] && [ "${DISTRO_VERSION}" = "unstable" ]; then \
+      cp -a /usr/local_mpich/. /usr/local/ \
+      && rm -rf /usr/local_mpich; \
+    else \
+      rm -rf /usr/local_mpich; \
+    fi
+RUN printf "/usr/local_veloc/lib" > /etc/ld.so.conf.d/veloc.conf \
+    && ldconfig
+
+ENV CMAKE_PREFIX_PATH=/usr/local_veloc:${CMAKE_PREFIX_PATH}
+
 ENV PDI_SYSTEM=${DISTRO}-${DISTRO_VERSION}
 ENV PDI_COMPILER=gcc
 ENV PDI_MPI=${MPI}
@@ -72,21 +137,19 @@ ENV HOME=/home/ci
 USER ci:ci
 WORKDIR ${HOME}
 
-
+# stage : pdi download 
 
 FROM alpine/git AS pdidl
 
-RUN git -C /root clone --recurse-submodules https://github.com/pdidev/pdi.git /root/pdi
-
+RUN git -C /root clone --recurse-submodules https://github.com/iole-bolognesi/pdi.git /root/pdi
 
+# stage : test 
 
-FROM main AS test
+FROM complete-main AS test
 
 COPY --from=pdidl --chown=ci:ci /root/pdi /home/ci/pdi
 
 RUN export VERBOSE=1 \
  && bash pdi/bin/build_and_run_all_tests
 
-
-
-FROM main
+FROM complete-main

spack/2-spackbase/Dockerfile

@@ -1,6 +1,7 @@
 ARG IMAGE_TAG
+ARG REGISTRY=ghcr.io/pdidev/test_env
 
-FROM ghcr.io/pdidev/test_env/base:${IMAGE_TAG}
+FROM ${REGISTRY}/base:${IMAGE_TAG}
 
 LABEL "org.opencontainers.image.source"="https://github.com/pdidev/test_env"