diff --git a/scm/etc/modules/derecho_gnu.lua b/scm/etc/modules/derecho_gnu.lua deleted file mode 100644 index f7f64fb57..000000000 --- a/scm/etc/modules/derecho_gnu.lua +++ /dev/null @@ -1,29 +0,0 @@ -help([[ -This module loads libraries for building the CCPP Single-Column Model on -the CISL machine Derecho (Cray) using GNU 12.2.0 -]]) - -whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Derecho with GNU compilers]===]) - -prepend_path("MODULEPATH","/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra") -prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") - -load("stack-gcc/12.2.0") -load("stack-cray-mpich/8.1.25") -load("stack-python/3.10.13") -load("cmake/3.23.1") - -load("hdf5/1.14.0") -load("netcdf-c/4.9.2") -load("netcdf-fortran/4.6.1") -load("bacio/2.4.1") -load("sp/2.5.0") -load("w3emc/2.10.0") - -load("py-f90nml") -load("py-netcdf4/1.5.8") - -setenv("CMAKE_C_COMPILER","mpicc") -setenv("CMAKE_CXX_COMPILER","mpicxx") -setenv("CMAKE_Fortran_COMPILER","mpif90") -setenv("CMAKE_Platform","derecho.gnu") diff --git a/scm/etc/modules/derecho_gnu.lua b/scm/etc/modules/derecho_gnu.lua new file mode 120000 index 000000000..73e1963d7 --- /dev/null +++ b/scm/etc/modules/derecho_gnu.lua @@ -0,0 +1 @@ +derecho_gnu_spack_stack_1.9.3.lua \ No newline at end of file diff --git a/scm/etc/modules/derecho_gnu_spack_stack_1.6.0.lua b/scm/etc/modules/derecho_gnu_spack_stack_1.6.0.lua new file mode 100644 index 000000000..f7f64fb57 --- /dev/null +++ b/scm/etc/modules/derecho_gnu_spack_stack_1.6.0.lua @@ -0,0 +1,29 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the CISL machine Derecho (Cray) using GNU 12.2.0 +]]) + +whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Derecho with GNU compilers]===]) + +prepend_path("MODULEPATH","/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra") +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") + +load("stack-gcc/12.2.0") +load("stack-cray-mpich/8.1.25") +load("stack-python/3.10.13") +load("cmake/3.23.1") + +load("hdf5/1.14.0") +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.1") +load("bacio/2.4.1") +load("sp/2.5.0") +load("w3emc/2.10.0") + +load("py-f90nml") +load("py-netcdf4/1.5.8") + +setenv("CMAKE_C_COMPILER","mpicc") +setenv("CMAKE_CXX_COMPILER","mpicxx") +setenv("CMAKE_Fortran_COMPILER","mpif90") +setenv("CMAKE_Platform","derecho.gnu") diff --git a/scm/etc/modules/derecho_gnu_spack_stack_1.9.3.lua b/scm/etc/modules/derecho_gnu_spack_stack_1.9.3.lua new file mode 100644 index 000000000..9a114e355 --- /dev/null +++ b/scm/etc/modules/derecho_gnu_spack_stack_1.9.3.lua @@ -0,0 +1,28 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the CISL machine Derecho (Cray) using GNU 12.4.0 +]]) + +whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Derecho with GNU compilers]===]) + +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-gcc-12.4.0/install/modulefiles/Core/") + +load("stack-gcc/12.4.0") +load("stack-cray-mpich/8.1.29") +load("stack-python/3.11.7") +load("cmake/3.27.9") + +load("hdf5/1.14.3") +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.1") +load("bacio/2.4.1") +load("sp/2.5.0") +load("w3emc/2.10.0") + +load("py-f90nml") +load("py-netcdf4/1.7.1.post2") + +setenv("CMAKE_C_COMPILER","mpicc") +setenv("CMAKE_CXX_COMPILER","mpicxx") +setenv("CMAKE_Fortran_COMPILER","mpif90") +setenv("CMAKE_Platform","derecho.gnu") diff --git a/scm/etc/modules/derecho_intel.lua b/scm/etc/modules/derecho_intel.lua deleted file mode 100644 index 07b8d3627..000000000 --- a/scm/etc/modules/derecho_intel.lua +++ /dev/null @@ -1,30 +0,0 @@ -help([[ -This module loads libraries for building the CCPP Single-Column Model on -the CISL machine Derecho (Cray) using Intel-classic-2021.10.0 -]]) - -whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Derecho with Intel compilers]===]) - -setenv("LMOD_TMOD_FIND_FIRST","yes") -load("ncarenv/23.09") - -prepend_path("MODULEPATH","/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra") -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") - -load("stack-intel/2021.10.0") -load("stack-cray-mpich/8.1.25") -load("stack-python/3.10.13") -load("py-f90nml") -load("py-netcdf4/1.5.8") -load("cmake/3.23.1") - -load("netcdf-c/4.9.2") -load("netcdf-fortran/4.6.1") -load("bacio/2.4.1") -load("sp/2.5.0") -load("w3emc/2.10.0") - -setenv("CMAKE_C_COMPILER","cc") -setenv("CMAKE_CXX_COMPILER","CC") -setenv("CMAKE_Fortran_COMPILER","ftn") -setenv("CMAKE_Platform","derecho.intel") diff --git a/scm/etc/modules/derecho_intel.lua b/scm/etc/modules/derecho_intel.lua new file mode 120000 index 000000000..20843592d --- /dev/null +++ b/scm/etc/modules/derecho_intel.lua @@ -0,0 +1 @@ +derecho_intel_spack_stack_1.9.3.lua \ No newline at end of file diff --git a/scm/etc/modules/derecho_intel_spack_stack_1.6.0.lua b/scm/etc/modules/derecho_intel_spack_stack_1.6.0.lua new file mode 100644 index 000000000..07b8d3627 --- /dev/null +++ b/scm/etc/modules/derecho_intel_spack_stack_1.6.0.lua @@ -0,0 +1,30 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the CISL machine Derecho (Cray) using Intel-classic-2021.10.0 +]]) + +whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Derecho with Intel compilers]===]) + +setenv("LMOD_TMOD_FIND_FIRST","yes") +load("ncarenv/23.09") + +prepend_path("MODULEPATH","/lustre/desc1/scratch/epicufsrt/contrib/modulefiles_extra") +prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") + +load("stack-intel/2021.10.0") +load("stack-cray-mpich/8.1.25") +load("stack-python/3.10.13") +load("py-f90nml") +load("py-netcdf4/1.5.8") +load("cmake/3.23.1") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.1") +load("bacio/2.4.1") +load("sp/2.5.0") +load("w3emc/2.10.0") + +setenv("CMAKE_C_COMPILER","cc") +setenv("CMAKE_CXX_COMPILER","CC") +setenv("CMAKE_Fortran_COMPILER","ftn") +setenv("CMAKE_Platform","derecho.intel") diff --git a/scm/etc/modules/derecho_intel_spack_stack_1.9.3.lua b/scm/etc/modules/derecho_intel_spack_stack_1.9.3.lua new file mode 100644 index 000000000..f06a2a7f7 --- /dev/null +++ b/scm/etc/modules/derecho_intel_spack_stack_1.9.3.lua @@ -0,0 +1,41 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the CISL machine Derecho (Cray) using Intel-oneAPI +]]) + +whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Derecho with Intel oneAPI compilers]===]) + +-- setenv("LMOD_TMOD_FIND_FIRST","yes") +-- load("ncarenv/24.12") + +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-oneapi-2024.2.1/install/modulefiles/Core/") + + +load("stack-oneapi/2024.2.1") +load("stack-cray-mpich/8.1.29") +load("stack-python/3.11.7") +load("cmake/3.27.9") + +load("hdf5/1.14.3") +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.1") +load("bacio/2.4.1") +load("sp/2.5.0") +load("w3emc/2.10.0") + +load("py-f90nml") +load("py-netcdf4/1.7.1.post2") + +load("intel-oneapi") + +setenv("CMAKE_C_COMPILER","cc") +setenv("CMAKE_CXX_COMPILER","CC") +setenv("CMAKE_Fortran_COMPILER","ftn") +setenv("CMAKE_Platform","derecho.intel") + +if mode() == "load" then + local red = "\27[31m" + local reset = "\27[0m" + LmodMessage(red .. "WARNING: " .. reset .. "Login node on Derecho does not have enough memory to build SCM, use interactive node") + LmodMessage(red .. "WARNING: " .. reset .. "Debug mode currently broken on Derecho, don't use '-g'") +end diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index b0c68574d..990446e30 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -174,7 +174,7 @@ if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") set(CMAKE_C_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -traceback -fpp -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -sox -align array64byte -qno-opt-dynamic-align") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -traceback -fpp -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -align array64byte -qno-opt-dynamic-align") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align -sox -fp-model source") if(NOT 32BIT) @@ -188,20 +188,10 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") if(32BIT) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3") else() - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3 -no-prec-div -no-prec-sqrt") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3 -no-prec-div") endif() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -debug minimal") set(FAST "-fast-transcendentals") - if(AVX2) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx2") - elseif(SIMDMULTIARCH) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,CORE-AVX2") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,CORE-AVX2") - elseif(AVX) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx-i") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx-i") - endif() endif() set(CMAKE_C_FLAGS_RELEASE "-O2 -fPIC" CACHE STRING "" FORCE) @@ -209,6 +199,20 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") set(CMAKE_C_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) set(CMAKE_Fortran_FLAGS_BITFORBIT "-O2 -fPIC" CACHE STRING "" FORCE) + if(DEFINED ENV{CMAKE_Platform} AND + "$ENV{CMAKE_Platform}" MATCHES "derecho.intel") + set(CMAKE_Fortran_FLAGS "-O0 -fPIC") + set(CMAKE_C_FLAGS "-O0 -fPIC") + set(CMAKE_Fortran_FLAGS_RELEASE "") + set(CMAKE_C_FLAGS_RELEASE "") + set(CMAKE_Fortran_FLAGS_DEBUG "") + set(CMAKE_C_FLAGS_DEBUG "") + if(NOT 32BIT) + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -real-size 64") + endif() + message(WARNING "Building on Derecho, only using CMAKE_Fortran_FLAGS=${CMAKE_Fortran_FLAGS}") + endif() + elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") if(ENABLE_NVIDIA_OPENACC MATCHES "true") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc -Minfo=accel")