ScaLAPACK dependency installation (#230)

* scalapack dependency files.

* added some instructions.

* added scalapack installation procedure in setup.sh.

* Fix for the case when the hint is not used.

* check if dependencies/scalapack-2.2.0 is already installed.

* minor fix to use manually built scalapack.

* setup.sh: check_result - check the setup command result.

* scripts/compile.sh: get repo prefix without using git command.

* scripts/compile.sh: scalapack installation is optional.

cmake/toolchains/simple.cmake

@@ -0,0 +1,9 @@
+set(CMAKE_C_COMPILER mpicc)
+set(CMAKE_CXX_COMPILER mpicxx)
+set(CMAKE_Fortran_COMPILER mpif90)
+
+# Use environment variables.
+set(MFEM_DIR $ENV{MFEM_DIR})
+set(HYPRE_DIR $ENV{HYPRE_DIR})
+set(PARMETIS_DIR $ENV{PARMETIS_DIR})
+set(METIS_DIR $ENV{METIS_DIR})

dependencies/SLmake.inc

@@ -0,0 +1,65 @@
+############################################################################
+#
+#  Program:         ScaLAPACK
+#
+#  Module:          SLmake.inc
+#
+#  Purpose:         Top-level Definitions
+#
+#  Creation date:   February 15, 2000
+#
+#  Modified:        October 13, 2011
+#
+#  Send bug reports, comments or suggestions to [email protected]
+#
+############################################################################
+#
+#  C preprocessor definitions:  set CDEFS to one of the following:
+#
+#     -DNoChange (fortran subprogram names are lower case without any suffix)
+#     -DUpCase   (fortran subprogram names are upper case without any suffix)
+#     -DAdd_     (fortran subprogram names are lower case with "_" appended)
+
+CDEFS         = -DAdd_
+
+#
+#  The fortran and C compilers, loaders, and their flags
+#
+
+# LIBROM instruction: copy this file to scalapack-2.2.0 directory and type make.
+
+# -fallow-argument-mismatch is needed to compile some legacy fortran code that does not conform to modern gnu compiler standard.
+# -fPIC is needed to build the static scalapack library.
+
+FC            = mpif90
+CC            = mpicc 
+NOOPT         = -O0 -fallow-argument-mismatch -fPIC
+FCFLAGS       = -O3 -fallow-argument-mismatch -fPIC
+CCFLAGS       = -O3 -fPIC
+FCLOADER      = $(FC)
+CCLOADER      = $(CC)
+FCLOADFLAGS   = $(FCFLAGS)
+CCLOADFLAGS   = $(CCFLAGS)
+
+#
+#  The archiver and the flag(s) to use when building archive (library)
+#  Also the ranlib routine.  If your system has no ranlib, set RANLIB = echo
+#
+
+ARCH          = ar
+ARCHFLAGS     = cr
+RANLIB        = ranlib
+
+#
+#  The name of the ScaLAPACK library to be created
+#
+
+SCALAPACKLIB  = libscalapack.a
+
+#
+#  BLAS, LAPACK (and possibly other) libraries needed for linking test programs
+#
+
+BLASLIB       = -lblas
+LAPACKLIB     = -llapack
+LIBS          = $(LAPACKLIB) $(BLASLIB)

lib/CMakeLists.txt

@@ -98,7 +98,15 @@ if (BLAS_LIBRARIES MATCHES ".*mkl.*")
     target_include_directories(ROM PUBLIC ${MKL_INCLUDE_DIRS})
   endif()
 else() # BLAS or LAPACK isn't MKL
-  find_package(ScaLAPACK REQUIRED)
+  find_package(ScaLAPACK)
+  if (NOT ScaLAPACK_FOUND)
+    # Attempt to use manually-built scalapack in dependencies.
+    # CMake files in scalapack directory disrupts find_package using PATHS/HINTS.
+    # Here we prepend environment PATH variable.
+    # This changed PATH variable applies only to this libROM compilation.
+    set(ENV{PATH} "${CMAKE_SOURCE_DIR}/dependencies/scalapack-2.2.0:$ENV{PATH}")
+    find_package(ScaLAPACK REQUIRED)
+  endif()
   target_link_libraries(ROM PUBLIC ${ScaLAPACK_LIBRARIES})
 endif()
 

scripts/compile.sh

@@ -9,11 +9,22 @@
 #  SPDX-License-Identifier: (Apache-2.0 OR MIT)
 #
 ###############################################################################
+check_result () {
+  # $1: Result output of the previous command ($?)
+  # $2: Name of the previous command
+  if [ $1 -eq 0 ]; then
+      echo "$2 succeeded"
+  else
+      echo "$2 failed"
+      exit -1
+  fi
+}
 
 ARDRA=false
 BUILD_TYPE="Optimized"
 USE_MFEM="Off"
 UPDATE_LIBS=false
+INSTALL_SCALAPACK=false
 MFEM_USE_GSLIB="Off"
 
 cleanup_dependencies() {
@@ -34,7 +45,7 @@ cleanup_dependencies() {
 
 
 # Get options
-while getopts "ah:dh:gh:mh:t:uh" o;
+while getopts "ah:dh:gh:mh:t:uh:sh" o;
 do
     case "${o}" in
         a)
@@ -55,6 +66,9 @@ do
         u)
             UPDATE_LIBS=true
             ;;
+        s)
+            INSTALL_SCALAPACK=true
+            ;;
     *)
             echo "Unknown option."
             exit 1
@@ -88,10 +102,12 @@ if [[ $MFEM_USE_GSLIB == "On" ]] && [[ ! -d "$HOME_DIR/dependencies/gslib" ]]; t
 fi
 export MFEM_USE_GSLIB
 
-REPO_PREFIX=$(git rev-parse --show-toplevel)
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+REPO_PREFIX=$( dirname $SCRIPT_DIR )
 
 if [[ $USE_MFEM == "On" ]]; then
-    . ${REPO_PREFIX}/scripts/setup.sh
+    . ${REPO_PREFIX}/scripts/setup.sh ${INSTALL_SCALAPACK}
+    check_result $? scripts-setup
 fi
 
 if [[ $ARDRA == "true" ]]; then
@@ -124,7 +140,9 @@ if [ "$(uname)" == "Darwin" ]; then
         -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
         -DUSE_MFEM=${USE_MFEM} \
         -DMFEM_USE_GSLIB=${MFEM_USE_GSLIB}
+  check_result $? librom-config
   make
+  check_result $? librom-build
 elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
   if [[ $ARDRA == "true" ]]; then
       TOOLCHAIN_FILE=${REPO_PREFIX}/cmake/toolchains/ic18-toss_3_x86_64_ib-ardra.cmake
@@ -136,6 +154,8 @@ elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
         -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
         -DUSE_MFEM=${USE_MFEM} \
         -DMFEM_USE_GSLIB=${MFEM_USE_GSLIB}
+  check_result $? librom-config
   make -j8
+  check_result $? librom-build
 fi
 popd

scripts/setup.sh

@@ -1,4 +1,17 @@
 #!/bin/bash
+check_result () {
+  # $1: Result output of the previous command ($?)
+  # $2: Name of the previous command
+  if [ $1 -eq 0 ]; then
+      echo "$2 succeeded"
+  else
+      echo "$2 failed"
+      exit -1
+  fi
+}
+
+# Take the first argument as option for installing ScaLAPACK.
+INSTALL_SCALAPACK=$1
 
 # Check whether Homebrew or wget is installed
 if [ "$(uname)" == "Darwin" ]; then
@@ -24,6 +37,21 @@ export CFLAGS="-fPIC"
 export CPPFLAGS="-fPIC"
 export CXXFLAGS="-fPIC"
 
+# Install ScaLAPACK if specified.
+cd $LIB_DIR
+if [[ $INSTALL_SCALAPACK == "true" ]]; then
+  if [ -f "scalapack-2.2.0/libscalapack.a" ]; then
+    echo "Using dependencies/scalapack-2.2.0/libscalapack.a"
+  else
+    echo "ScaLAPACK is needed!"
+    tar -zxvf scalapack-2.2.0.tar.gz
+    cp SLmake.inc scalapack-2.2.0/
+    cd scalapack-2.2.0/
+    make
+    check_result $? ScaLAPACK-installation
+  fi
+fi
+
 # Install HYPRE
 cd $LIB_DIR
 if [ ! -d "hypre" ]; then
@@ -33,6 +61,7 @@ if [ ! -d "hypre" ]; then
   cd hypre/src
   ./configure --disable-fortran
   make -j
+  check_result $? hypre-installation
 fi
 
 # Install GSLIB
@@ -48,6 +77,7 @@ if [ $MFEM_USE_GSLIB == "On" ] && [ ! -d "gslib" ]; then
   mv gslib-1.0.7 gslib
   cd gslib
   make CC=mpicc -j
+  check_result $? gslib-installation
 fi
 
 # Install PARMETIS 4.0.3
@@ -57,12 +87,15 @@ if [ ! -d "parmetis-4.0.3" ]; then
   tar -zxvf parmetis-4.0.3.tar.gz
   cd parmetis-4.0.3
   make config
+  check_result $? parmetis-config
   make
+  check_result $? parmetis-installation
   METIS_DIR=$LIB_DIR/parmetis-4.0.3
   METIS_OPT=-I${METIS_DIR}/metis/include
   cd ${METIS_DIR}/build
   MACHINE_ARCH=$(ls)
   ln -s $MACHINE_ARCH lib
+  check_result $? parmetis-link
 fi
 
 unset CFLAGS
@@ -84,10 +117,12 @@ if [[ $BUILD_TYPE == "Debug" ]]; then
         cd mfem_debug
         git pull
         make pdebug -j 8 STATIC=NO SHARED=YES MFEM_USE_MPI=YES MFEM_USE_GSLIB=${MG} MFEM_USE_METIS=YES MFEM_USE_METIS_5=YES METIS_DIR="$METIS_DIR" METIS_OPT="$METIS_OPT" METIS_LIB="$METIS_LIB"
+        check_result $? mfem-debug-installation
     fi
     cd $LIB_DIR
     rm mfem
     ln -s mfem_debug mfem
+    check_result $? mfem-debug-link
 else
     if [ ! -d "mfem_parallel" ]; then
       UPDATE_LIBS=true
@@ -97,8 +132,10 @@ else
         cd mfem_parallel
         git pull
         make parallel -j 8 STATIC=NO SHARED=YES MFEM_USE_MPI=YES MFEM_USE_GSLIB=${MG} MFEM_USE_METIS=YES MFEM_USE_METIS_5=YES METIS_DIR="$METIS_DIR" METIS_OPT="$METIS_OPT" METIS_LIB="$METIS_LIB"
+        check_result $? mfem-parallel-installation
     fi
     cd $LIB_DIR
     rm mfem
     ln -s mfem_parallel mfem
+    check_result $? mfem-parallel-link
 fi