diff --git a/.gitmodules b/.gitmodules index 56dee3ba7..93d534346 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,8 +4,10 @@ branch = develop [submodule "ccpp-physics"] path = ccpp/physics - url = https://github.com/NCAR/ccpp-physics - branch = main + # url = https://github.com/NCAR/ccpp-physics + # branch = main + url = https://github.com/scrasmussen/ccpp-physics + branch = enhancement/build-with-ip [submodule "CMakeModules"] path = CMakeModules url = https://github.com/noaa-emc/CMakeModules diff --git a/ccpp/physics b/ccpp/physics index 3f4ad81b7..d755cd7b1 160000 --- a/ccpp/physics +++ b/ccpp/physics @@ -1 +1 @@ -Subproject commit 3f4ad81b726ba03f46160641284a24cf67fe0eaa +Subproject commit d755cd7b15fa3c0f9ec3160bd5f44f41aea131af diff --git a/scm/etc/modules/derecho_gnu_spack_stack_1.9.3.lua b/scm/etc/modules/derecho_gnu_spack_stack_1.9.3.lua new file mode 100644 index 000000000..343c81f5f --- /dev/null +++ b/scm/etc/modules/derecho_gnu_spack_stack_1.9.3.lua @@ -0,0 +1,28 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the CISL machine Derecho (Cray) using GNU 12.4.0 +]]) + +whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Derecho with GNU compilers]===]) + +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.9.3/envs/ue-gcc-12.4.0/install/modulefiles/Core/") + +load("stack-gcc/12.4.0") +load("stack-cray-mpich/8.1.29") +load("stack-python/3.11.7") +load("cmake/3.27.9") + +load("hdf5/1.14.3") +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.1") +load("bacio/2.4.1") +load("ip/5.1.0") +load("w3emc/2.10.0") + +load("py-f90nml") +load("py-netcdf4/1.7.1.post2") + +setenv("CMAKE_C_COMPILER","mpicc") +setenv("CMAKE_CXX_COMPILER","mpicxx") +setenv("CMAKE_Fortran_COMPILER","mpif90") +setenv("CMAKE_Platform","derecho.gnu") diff --git a/scm/etc/modules/derecho_intel_spack_stack_2.0.0.lua b/scm/etc/modules/derecho_intel_spack_stack_2.0.0.lua new file mode 100644 index 000000000..2b8a0d09e --- /dev/null +++ b/scm/etc/modules/derecho_intel_spack_stack_2.0.0.lua @@ -0,0 +1,29 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the CISL machine Derecho (Cray) using Intel oneAPI 2025.2.1 +]]) + +whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Derecho with Intel compilers]===]) + + +prepend_path("MODULEPATH","/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-2.0.0/envs/ue-oneapi-2025.2.1/modules/Core/") + +load("stack-intel-oneapi-compilers/2025.2.1") +load("stack-cray-mpich/8.1.32") +load("python/3.11.11") +load("cmake/3.31.8") + +load("hdf5/1.14.5") +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.1") +load("bacio/2.6.0") +load("ip/5.4.0") +load("w3emc/2.11.0") + +load("py-f90nml") +load("py-netcdf4/1.7.2") + +setenv("CMAKE_C_COMPILER","cc") +setenv("CMAKE_CXX_COMPILER","CC") +setenv("CMAKE_Fortran_COMPILER","ftn") +setenv("CMAKE_Platform","derecho.intel") diff --git a/scm/etc/modules/ursa_gnu_spack_stack_2.0.0.lua b/scm/etc/modules/ursa_gnu_spack_stack_2.0.0.lua new file mode 100644 index 000000000..54ddefdb6 --- /dev/null +++ b/scm/etc/modules/ursa_gnu_spack_stack_2.0.0.lua @@ -0,0 +1,31 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the NOAA RDHPC machine Ursa using GNU 12.4 +]]) + +whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Ursa with Intel compilers ]===]) + +prepend_path("MODULEPATH","/contrib/spack-stack/spack-stack-2.0.0/envs/ue-gcc-12.4.0/modules/Core") + +load("stack-gcc/12.4.0") +load("stack-openmpi/4.1.6") +load("python/3.11.11") +load("cmake/3.31.8") + +load("hdf5/1.14.5") +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.1") +load("bacio/2.6.0") +load("ip/5.4.0") +load("w3emc/2.11.0") +load("esmf/8.8.0") + +load("py-f90nml") +load("py-netcdf4/1.7.2") + +setenv("CMAKE_C_COMPILER","mpicc") +setenv("CMAKE_CXX_COMPILER","mpicxx") +setenv("CMAKE_Fortran_COMPILER","mpif90") +setenv("CMAKE_Platform","ursa.gnu") + +execute{cmd="source /scratch3/BMC/gmtb/ccpp-scm-software/spack-stack-2.0.0-gnu/bin/activate", modeA={"load"}} diff --git a/scm/etc/modules/ursa_intel_spack_stack_2.0.0.lua b/scm/etc/modules/ursa_intel_spack_stack_2.0.0.lua new file mode 100644 index 000000000..ff084526d --- /dev/null +++ b/scm/etc/modules/ursa_intel_spack_stack_2.0.0.lua @@ -0,0 +1,31 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +the NOAA RDHPC machine Ursa using Intel oneAPI 2025.2.1 +]]) + +whatis([===[Loads spack-stack libraries needed for building the CCPP SCM on Ursa with Intel compilers ]===]) + +prepend_path("MODULEPATH","/contrib/spack-stack/spack-stack-2.0.0/envs/ue-oneapi-2025.2.1/modules/Core") + +load("stack-intel-oneapi-compilers/2025.2.1") +load("stack-intel-oneapi-mpi/2021.13") +load("python/3.11.11") +load("cmake/3.31.8") + +load("hdf5/1.14.5") +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.1") +load("bacio/2.6.0") +load("ip/5.4.0") +load("w3emc/2.11.0") +load("esmf/8.8.0") + +load("py-f90nml") +load("py-netcdf4/1.7.2") + +setenv("CMAKE_C_COMPILER","cc") +setenv("CMAKE_CXX_COMPILER","CC") +setenv("CMAKE_Fortran_COMPILER","ftn") +setenv("CMAKE_Platform","ursa.intel") + +execute{cmd="source /scratch3/BMC/gmtb/ccpp-scm-software/spack-stack-2.0.0-intel/bin/activate", modeA={"load"}} diff --git a/scm/src/CMakeLists.txt b/scm/src/CMakeLists.txt index 49a592209..726b5dbfd 100644 --- a/scm/src/CMakeLists.txt +++ b/scm/src/CMakeLists.txt @@ -69,7 +69,11 @@ SET(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}/../../CMakeModule find_package(NetCDF REQUIRED COMPONENTS C Fortran) find_package(bacio REQUIRED) -find_package(sp REQUIRED) +# Use ip@5 or later if available, fall back to sp +find_package(ip 5) +if(NOT ip_FOUND) + find_package(sp REQUIRED) +endif() find_package(w3emc REQUIRED) find_package(MPI REQUIRED) if(NOT MPI_Fortran_HAVE_F08_MODULE) @@ -156,6 +160,7 @@ endif() if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") message(STATUS "Compile using GNU") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ggdb -fbacktrace -cpp -fcray-pointer -ffree-line-length-none -fno-range-check") + set(CMAKE_Fortran_FLAGS_OPENMP_OFF "-fno-openmp") if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz") @@ -182,9 +187,10 @@ if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") #------------------------------------------------------------------------------ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "IntelLLVM") message(STATUS "Compile using Intel OneAPI") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -traceback -fpp -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -align array64byte -qno-opt-dynamic-align") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align -sox -fp-model source") - + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -traceback -fpp -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -align array64byte -qno-opt-dynamic-align") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align -fp-model source") + set(CMAKE_Fortran_FLAGS_OPENMP_OFF "-fno-openmp") + if(NOT 32BIT) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -real-size 64") endif() @@ -195,11 +201,11 @@ if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -ftrapuv") else() if(32BIT) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3") else() - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3 -no-prec-div") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3 -no-prec-div") endif() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -debug minimal") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -debug minimal") set(FAST "-fast-transcendentals") if(AVX2) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2") @@ -221,9 +227,11 @@ if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") #------------------------------------------------------------------------------ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel") message(STATUS "Compile using Intel Classic") - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -traceback -fpp -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -sox -align array64byte -qno-opt-dynamic-align") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align -sox -fp-model source") - + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g -traceback -fpp -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -align array64byte -qno-opt-dynamic-align") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align -fp-model source") + set(CMAKE_Fortran_FLAGS_OPENMP_OFF "-fno-openmp") + + if(NOT 32BIT) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -real-size 64") endif() @@ -233,11 +241,11 @@ if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -ftrapuv") else() if(32BIT) - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3") else() - set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3 -no-prec-div -no-prec-sqrt") + set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3 -no-prec-div") endif() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -debug minimal") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -debug minimal") set(FAST "-fast-transcendentals") if(AVX2) set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2") @@ -261,6 +269,7 @@ if (${CMAKE_Fortran_COMPILER_ID} MATCHES "GNU") #------------------------------------------------------------------------------ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "NVHPC") message(STATUS "Compile using Nvidia HPC Stack") + set(CMAKE_Fortran_FLAGS_OPENMP_OFF "-nomp") if(ENABLE_NVIDIA_OPENACC MATCHES "true") set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc -Minfo=accel") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -acc -Minfo=accel") @@ -302,7 +311,7 @@ if(32BIT) message(STATUS "Compile CCPP slow physics with 64-bit precision, fast physics with 32-bit precision") add_definitions(-DOVERLOAD_R4) if(CMAKE_Fortran_COMPILER_ID MATCHES "Intel") - # set(CMAKE_Fortran_FLAGS_PHYSICS "-real-size 64 -no-prec-div -no-prec-sqrt") + # set(CMAKE_Fortran_FLAGS_PHYSICS "-real-size 64 -no-prec-div") elseif(CMAKE_Fortran_COMPILER_ID MATCHES "GNU") # set(CMAKE_Fortran_FLAGS_PHYSICS "-fdefault-real-8 -fdefault-double-8") endif() @@ -367,11 +376,23 @@ SET(scm_source_files scm.F90 ADD_EXECUTABLE(scm ${scm_source_files} ccpp_static_api.F90) if(OPENMP) - TARGET_LINK_LIBRARIES(scm PUBLIC OpenMP::OpenMP_Fortran) + target_link_libraries(scm OpenMP::OpenMP_Fortran) +endif() +target_link_libraries(scm NetCDF::NetCDF_Fortran) +if(TARGET bacio::bacio) + set(BACIO_TARGET bacio::bacio) +elseif(TARGET bacio::bacio_4) + set(BACIO_TARGET bacio::bacio_4) +else() + message(FATAL_ERROR + "bacio found but expected target not exported (bacio::bacio or bacio::bacio_4)") +endif() +target_link_libraries(scm ${BACIO_TARGET}) +if(ip_FOUND) + target_link_libraries(scm ip::ip_d) +else() + target_link_libraries(scm sp::sp_d) endif() -TARGET_LINK_LIBRARIES(scm NetCDF::NetCDF_Fortran) -TARGET_LINK_LIBRARIES(scm bacio::bacio_4) -TARGET_LINK_LIBRARIES(scm sp::sp_d) TARGET_LINK_LIBRARIES(scm w3emc::w3emc_d) TARGET_LINK_LIBRARIES(scm ccpp_framework) TARGET_LINK_LIBRARIES(scm ccpp_physics)