Add toolchain for Lassen (#257)

* Add toolchain for lassen with gnu12 compilers

* Define compilers in compile.sh to fix lassen dependency issue

* Minor tweaks to parmetis compile commands

* Add power9 tuning flags for gcc

* Adjust setup.sh to use existing CFLAGS if set

* Update authors in README

* Add section to README for LC machines

README.md

@@ -39,6 +39,12 @@ Compilation options:
 - -t: Use your own cmake/toolchain
 - -u: Update all of libROM's dependencies.
 
+## Compiling on LC Machines
+
+libROM provides several CMake toolchains which can be used to compile on LLNL LC machines. 
+For more information on installing and using libROM on specific LC machines, 
+refer to [this wiki page](https://github.com/LLNL/libROM/wiki/Compiling-on-LC-Machines).
+
 # Installing via Spack
 
 There is a Spack package for libROM; however, the version it installs
@@ -114,6 +120,7 @@ LLNL-CODE-766763
 
 # Authors
 - Robert W. Anderson (LLNL)
+- William Anderson (LLNL)
 - William Arrighi (LLNL)
 - Kyle Chand (LLNL)
 - Siu Wun Cheung (LLNL)
@@ -126,10 +133,12 @@ LLNL-CODE-766763
 - Xiaolong He (UC San Diego)
 - Adrian Humphry (University of Toronto)
 - Kevin Huynh (LLNL)
+- Coleman Kendrick (LLNL)
 - Tanya Kostova-Vassilevska (LLNL)
 - Jessica Lauzon (Stanford)
 - Sean McBane (UT Austin)
-- Yeonjong Shin (KAIST)
 - Geoffrey Oxberry (LLNL)
+- Yeonjong Shin (KAIST)
+- Paul Tranquilli (LLNL)
 - Pranav Vempati (LLNL)
 - Masayuki Yano (University of Toronto)

cmake/toolchains/gnu12-rhel_7_ppc64le_ib_p9-dev.cmake

@@ -0,0 +1,31 @@
+# Toolchain for Lawrence Livermore National Laboratory CORAL machines
+# (e.g., Lassen), assuming the following dependencies:
+#
+# - OpenMPI with GNU 12.2.1 compilers
+# - LAPACK 3.11
+# - HDF5 1.14.0
+
+# Use MPI compiler wrappers because it simplifies detection of MPI
+set(CMAKE_C_COMPILER /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-12.2.1/bin/mpicc)
+set(CMAKE_CXX_COMPILER /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-12.2.1/bin/mpicxx)
+set(CMAKE_Fortran_COMPILER /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-12.2.1/bin/mpif90)
+
+execute_process(
+  COMMAND "echo $(which h5diff) | sed -e s/'/bin/h5diff'//"
+  OUTPUT_VARIABLE hdf5_serial_path
+)
+set(HDF5_ROOT ${hdf5_serial_path})
+
+set(BLAS_ROOT /usr/tcetmp/packages/lapack/lapack-3.11.0-gcc-11.2.1/)
+set(LAPACK_ROOT /usr/tcetmp/packages/lapack/lapack-3.11.0-gcc-11.2.1/)
+
+set(ScaLAPACK_ROOT /usr/tcetmp/packages/lapack/lapack-3.11.0-gcc-11.2.1/)
+set(SCALAPACKDIR /usr/tcetmp/packages/lapack/lapack-3.11.0-gcc-11.2.1/)
+
+set(CMAKE_EXE_LINKER_FLAGS_INIT "-lblas -llapack")
+
+# Set flags to tune for the POWER9 architecture
+set(CMAKE_C_FLAGS_INIT "-mcpu=powerpc64le -mtune=powerpc64le")
+set(CMAKE_CXX_FLAGS_INIT "-mcpu=powerpc64le -mtune=powerpc64le")
+
+set(CMAKE_LIBRARY_ARCHITECTURE ppc64le)

scripts/compile.sh

@@ -77,6 +77,9 @@ do
 done
 shift $((OPTIND-1))
 
+export CC=${CC:="mpicc"}
+export CXX=${CXX:="mpicxx"}
+
 CURR_DIR=$(pwd) 
 if [[ -d "dependencies" ]]; then
     HOME_DIR=$CURR_DIR

scripts/setup.sh

@@ -33,9 +33,9 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 LIB_DIR=$SCRIPT_DIR/../dependencies
 mkdir -p $LIB_DIR
 
-export CFLAGS="-fPIC"
-export CPPFLAGS="-fPIC"
-export CXXFLAGS="-fPIC"
+export CFLAGS="-fPIC ${CFLAGS}"
+export CPPFLAGS="-fPIC ${CPPFLAGS}"
+export CXXFLAGS="-fPIC ${CXXFLAGS}"
 
 # Install ScaLAPACK if specified.
 cd $LIB_DIR
@@ -88,9 +88,9 @@ if [ ! -d "parmetis-4.0.3" ]; then
 
   tar -zxvf parmetis-4.0.3.tar.gz
   cd parmetis-4.0.3
-  make config
+  make config shared=1
   check_result $? parmetis-config
-  make
+  make -j 8
   check_result $? parmetis-installation
   METIS_DIR=$LIB_DIR/parmetis-4.0.3
   METIS_OPT=-I${METIS_DIR}/metis/include
@@ -100,9 +100,6 @@ if [ ! -d "parmetis-4.0.3" ]; then
   check_result $? parmetis-link
 fi
 
-unset CFLAGS
-unset CPPFLAGS
-unset CXXFLAGS
 
 METIS_DIR=$LIB_DIR/parmetis-4.0.3
 METIS_OPT=-I${METIS_DIR}/metis/include