diff --git a/.gitmodules b/.gitmodules
index 925620f88..8ce51cd41 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -20,7 +20,7 @@
[submodule "ncar-physics"]
path = src/physics/ncar_ccpp
url = https://github.com/ESCOMP/atmospheric_physics
- fxtag = b47246dedfa7c377ac2bfb060e2d465beb931f2a
+ fxtag = atmos_phys0_21_000
fxrequired = AlwaysRequired
fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
[submodule "rrtmgp-data"]
@@ -32,7 +32,7 @@
[submodule "ccs_config"]
path = ccs_config
url = https://github.com/ESMCI/ccs_config_cesm.git
- fxtag = ccs_config_cesm1.0.65
+ fxtag = ccs_config_cesm1.0.72
fxrequired = ToplevelRequired
fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
[submodule "cdeps"]
diff --git a/ccs_config b/ccs_config
index 7fda4a54e..bd3990122 160000
--- a/ccs_config
+++ b/ccs_config
@@ -1 +1 @@
-Subproject commit 7fda4a54ec6261d2affb2351ca2b8f288308f0e8
+Subproject commit bd39901220166f8f8d7368cdf5aad8b704df3602
diff --git a/cime_config/cam_autogen.py b/cime_config/cam_autogen.py
index 695364b24..997e793ef 100644
--- a/cime_config/cam_autogen.py
+++ b/cime_config/cam_autogen.py
@@ -357,6 +357,68 @@ def _update_genccpp_dir(utility_files, genccpp_dir):
# end if
# end for
+###############################################################################
+def _set_rrtmgp_dependencies(dependency_files, gpu_flag):
+###############################################################################
+ """
+ Modify the list of files that physics schemes depend on
+ (as provided by the CCPP) so that the correct RRTMGP
+ dependencies exist for either CPUs or GPUs.
+
+ This function returns a modified list with either the
+ CPU or GPU RRTMGP dependencies, but not both.
+
+ >>> in_files = ['/some/path/file.F90','/some/rte-kernels/rrtmgp.F90', \
+ '/some/rte-kernels/accel/rrtmgp.F90', \
+ '/some/rte-kernels/mo_rte_solver_kernels.F90', \
+ '/some/rte-kernels/accel/mo_rte_solver_kernels.F90', \
+ '/some/other/path/file.F90', \
+ '/other/rrtmgp-kernels/mo_gas_optics_rrtmgp_kernels.F90', \
+ '/other/rrtmgp-kernels/accel/mo_gas_optics_rrtmgp_kernels.F90', ]
+ >>> _set_rrtmgp_dependencies(in_files, False)
+ ['/some/path/file.F90', '/some/rte-kernels/rrtmgp.F90', \
+'/some/rte-kernels/accel/rrtmgp.F90', '/some/rte-kernels/mo_rte_solver_kernels.F90', \
+'/some/other/path/file.F90', '/other/rrtmgp-kernels/mo_gas_optics_rrtmgp_kernels.F90']
+
+ >>> _set_rrtmgp_dependencies(in_files, True)
+ ['/some/path/file.F90', '/some/rte-kernels/rrtmgp.F90', \
+'/some/rte-kernels/accel/rrtmgp.F90', \
+'/some/rte-kernels/accel/mo_rte_solver_kernels.F90', '/some/other/path/file.F90', \
+'/other/rrtmgp-kernels/accel/mo_gas_optics_rrtmgp_kernels.F90']
+ """
+
+ # Create new list of CCPP-dependent files:
+ new_dependency_files = []
+
+ # List of directory strings that indicate
+ # RRTMGP dependencies:
+ rrtmgp_subdirs = ['rte-kernels', 'rrtmgp-kernels']
+
+ # List of RRTMGP files that differ between CPUs and GPUs:
+ rrtmgp_files = ['mo_rte_solver_kernels.F90',
+ 'mo_optical_props_kernels.F90',
+ 'mo_gas_optics_rrtmgp_kernels.F90']
+
+
+ for file_path in dependency_files:
+ if any(subdir in file_path for subdir in rrtmgp_subdirs):
+ if any(rfile in file_path for rfile in rrtmgp_files):
+ if (gpu_flag and 'accel' in file_path):
+ #If GPU-enabled and is "accelerated", include it:
+ new_dependency_files.append(file_path)
+ elif (not gpu_flag and 'accel' not in file_path):
+ #If CPU-only and not "accelerated", include it:
+ new_dependency_files.append(file_path)
+ else:
+ #Not a hardware-dependent file, so include it:
+ new_dependency_files.append(file_path)
+ else:
+ #Not an RRTMGP file, so include it:
+ new_dependency_files.append(file_path)
+
+ # Return newly-modified dependency files:
+ return new_dependency_files
+
###############################################################################
def generate_registry(data_search, build_cache, atm_root, bldroot,
source_mods_dir, dycore, gen_fort_indent):
@@ -420,7 +482,8 @@ def generate_registry(data_search, build_cache, atm_root, bldroot,
###############################################################################
def generate_physics_suites(build_cache, preproc_defs, host_name,
phys_suites_str, atm_root, bldroot,
- reg_dir, reg_files, source_mods_dir, force):
+ reg_dir, reg_files, source_mods_dir,
+ gpu_flag, force):
###############################################################################
"""
Generate the source for the configured physics suites,
@@ -664,8 +727,6 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
build_cache.update_ccpp(sdfs, scheme_files, host_files, xml_files,
scheme_nl_meta_files, nl_groups, create_nl_file,
preproc_cache_str, kind_types)
- ##XXgoldyXX: v Temporary fix: Copy CCPP Framework source code into
- ##XXgoldyXX: v generated code directory
request = DatatableReport("utility_files")
ufiles_str = datatable_report(cap_output_file, request, ";")
utility_files = ufiles_str.split(';')
@@ -674,8 +735,15 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
dep_str = datatable_report(cap_output_file, request, ";")
if len(dep_str) > 0:
dependency_files = dep_str.split(';')
+ # If using RRTMGP in the physics suite, then modify
+ # the provided dependency files list to use the correct
+ # CPU or GPU RRTMGP dependencies:
+ if any("rrtmgp_" in scheme_name for scheme_name in scheme_names):
+ dependency_files = _set_rrtmgp_dependencies(dependency_files,
+ gpu_flag)
+
+ # Copy dependencies files into CCPP build directory
_update_genccpp_dir(dependency_files, genccpp_dir)
- ##XXgoldyXX: ^ Temporary fix:
# End if
return [physics_blddir, genccpp_dir], do_gen_ccpp, cap_output_file, \
@@ -695,7 +763,7 @@ def generate_init_routines(build_cache, bldroot, force_ccpp, force_init,
and/or script).
"""
- #Add new directory to build path:
+ # Add new directory to build path:
init_dir = os.path.join(bldroot, "phys_init")
# Use this for cache check
gen_init_file = os.path.join(_REG_GEN_DIR, "write_init_files.py")
@@ -706,11 +774,11 @@ def generate_init_routines(build_cache, bldroot, force_ccpp, force_init,
if force_ccpp or force_init:
do_gen_init = True
else:
- #If not, then check cache to see if actual
- #"write_init_files.py" was modified:
+ # If not, then check cache to see if actual
+ # "write_init_files.py" was modified:
do_gen_init = build_cache.init_write_mismatch(gen_init_file)
else:
- #If no directory exists, then one will need
+ # If no directory exists, then one will need
# to create new routines:
os.mkdir(init_dir)
do_gen_init = True
diff --git a/cime_config/cam_config.py b/cime_config/cam_config.py
index 496f09b17..2224ded85 100644
--- a/cime_config/cam_config.py
+++ b/cime_config/cam_config.py
@@ -186,8 +186,9 @@ def __init__(self, case, case_log):
# Save case variables needed for code auto-generation:
self.__atm_root = case.get_value("COMP_ROOT_DIR_ATM")
self.__caseroot = case.get_value("CASEROOT")
- self.__bldroot = os.path.join(exeroot, "atm", "obj")
+ self.__bldroot = os.path.join(exeroot, "atm", "obj")
self.__atm_name = case.get_value("COMP_ATM")
+ self.__gpu_flag = case.get_value("OPENACC_GPU_OFFLOAD") #Returns a Boolean
# Save CPP definitions as a list:
self.__cppdefs = [x for x in case.get_value("CAM_CPPDEFS").split() if x]
@@ -879,7 +880,7 @@ def generate_cam_src(self, gen_fort_indent):
self.__atm_name, phys_suites,
self.__atm_root, self.__bldroot,
reg_dir, reg_files, source_mods_dir,
- force_ccpp)
+ self.__gpu_flag, force_ccpp)
phys_dirs, force_init, _, nml_fils, capgen_db, scheme_names = retvals
# Add namelist definition files to dictionary:
diff --git a/cime_config/testdefs/testlist_cam.xml b/cime_config/testdefs/testlist_cam.xml
index 81542fff0..93a495046 100644
--- a/cime_config/testdefs/testlist_cam.xml
+++ b/cime_config/testdefs/testlist_cam.xml
@@ -115,6 +115,15 @@
+
+
+
+
+
+
+
+
+
diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/shell_commands b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/shell_commands
new file mode 100644
index 000000000..c2a7ea1a7
--- /dev/null
+++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/shell_commands
@@ -0,0 +1,13 @@
+./xmlchange NTASKS=128
+./xmlchange NTHRDS=1
+./xmlchange ROOTPE='0'
+./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange TIMER_DETAIL='6'
+./xmlchange TIMER_LEVEL='999'
+./xmlchange GPU_TYPE=a100
+./xmlchange OPENACC_GPU_OFFLOAD=TRUE
+./xmlchange OVERSUBSCRIBE_GPU=TRUE
+./xmlchange NGPUS_PER_NODE=4
+./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites rrtmgp"
+./xmlchange RUN_STARTDATE=1979-01-01
diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/user_nl_cam b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/user_nl_cam
new file mode 100644
index 000000000..000dfd9de
--- /dev/null
+++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/user_nl_cam
@@ -0,0 +1,25 @@
+! these are CPU FHISTC_LTso snapshots
+ncdata = '/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_fhistc_ltso_rrtmgp_derecho_gnu_before_c20251013.nc'
+ncdata_check = '/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_fhistc_ltso_rrtmgp_derecho_gnu_after_c20251013.nc'
+
+! tolerances for testing (currently have high tolerance due to CPU snapshots)
+ncdata_check_err = .true.
+min_difference = 1e-08
+
+! vertical levels in snapshot
+pver = 58
+
+! Do radiation on every timestep we're testing
+irad_always=3
+
+! diagnostic output
+hist_output_frequency;h1: 1*nsteps
+hist_precision;h1: REAL64
+hist_add_inst_fields;h1: HR
+! Cloud output
+hist_add_inst_fields;h1: TOT_CLD_VISTAU,TOT_ICLD_VISTAU,ICE_ICLD_VISTAU,LIQ_ICLD_VISTAU
+! Longwave diagnostic output
+hist_add_inst_fields;h1: QRL,QRLC,FLNT,FLNTC,FLUT,FLUTC,LWCF,FLN200,FLN200C,FLNR,FLNS,FLNSC,FLDS,FLDSC,FUL,FDL,FULC,FDLC
+! Shortwave diagnostic fields
+hist_add_inst_fields;h1: SOLIN,QRS,QRSC,FSNT,FSNTC,FSNTOA,FSNTOAC,SWCF,FSUTOA,FSN200,FSN200C,FSNR,SOLL,SOLS,SOLLD,SOLSD
+hist_add_inst_fields;h1: FSNS,FSNSC,FSDS,FSDSC,FUS,FDS,FUSC,FDSC
diff --git a/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/user_nl_cpl b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/user_nl_cpl
new file mode 100644
index 000000000..083110452
--- /dev/null
+++ b/cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/user_nl_cpl
@@ -0,0 +1,5 @@
+! Set fixed orbital parameters
+orb_mode='fixed_parameters'
+orb_eccen = 0.
+orb_obliq = 0.
+orb_mvelp = 0.
diff --git a/src/core_utils/string_core_utils.F90 b/src/core_utils/string_core_utils.F90
index 053a9bf09..04c3cced8 100644
--- a/src/core_utils/string_core_utils.F90
+++ b/src/core_utils/string_core_utils.F90
@@ -124,7 +124,7 @@ pure function stringify(value, separator)
character(:), allocatable :: buffer, delimiter, format
character(:), allocatable :: value_c(:)
- integer :: i, n, offset
+ integer :: i, n, offset, fmt_len
if (present(separator)) then
delimiter = separator
@@ -168,43 +168,50 @@ pure function stringify(value, separator)
deallocate(value_c)
type is (integer(int32))
- allocate(character(11 * n + len(delimiter) * (n - 1)) :: buffer)
- allocate(character(17 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+ allocate(character(len=11 * n + len(delimiter) * (n - 1)) :: buffer)
+ fmt_len = 17 + len(delimiter) + floor(log10(real(n))) + 1
+ allocate(character(fmt_len) :: format)
write(format, '(a, i0, 3a)') '(ss, ', n, '(i0, :, "', delimiter, '"))'
write(buffer, format) value
type is (integer(int64))
- allocate(character(20 * n + len(delimiter) * (n - 1)) :: buffer)
- allocate(character(17 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+ allocate(character(len=20 * n + len(delimiter) * (n - 1)) :: buffer)
+ fmt_len = 17 + len(delimiter) + floor(log10(real(n))) + 1
+ allocate(character(len=fmt_len) :: format)
write(format, '(a, i0, 3a)') '(ss, ', n, '(i0, :, "', delimiter, '"))'
write(buffer, format) value
type is (logical)
- allocate(character(1 * n + len(delimiter) * (n - 1)) :: buffer)
- allocate(character(13 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+ allocate(character(len=1 * n + len(delimiter) * (n - 1)) :: buffer)
+ fmt_len = 13 + len(delimiter) + floor(log10(real(n))) + 1
+ allocate(character(len=fmt_len) :: format)
write(format, '(a, i0, 3a)') '(', n, '(l1, :, "', delimiter, '"))'
write(buffer, format) value
type is (real(real32))
- allocate(character(13 * n + len(delimiter) * (n - 1)) :: buffer)
+ allocate(character(len=13 * n + len(delimiter) * (n - 1)) :: buffer)
if (maxval(abs(value)) < 1.0e5_real32) then
- allocate(character(20 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+ fmt_len = 20 + len(delimiter) + floor(log10(real(n))) + 1
+ allocate(character(len=fmt_len) :: format)
write(format, '(a, i0, 3a)') '(ss, ', n, '(f13.6, :, "', delimiter, '"))'
else
- allocate(character(23 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+ fmt_len = 23 + len(delimiter) + floor(log10(real(n))) + 1
+ allocate(character(len=fmt_len) :: format)
write(format, '(a, i0, 3a)') '(ss, ', n, '(es13.6e2, :, "', delimiter, '"))'
end if
write(buffer, format) value
type is (real(real64))
- allocate(character(13 * n + len(delimiter) * (n - 1)) :: buffer)
+ allocate(character(len=13 * n + len(delimiter) * (n - 1)) :: buffer)
if (maxval(abs(value)) < 1.0e5_real64) then
- allocate(character(20 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+ fmt_len = 20 + len(delimiter) + floor(log10(real(n))) + 1
+ allocate(character(len=fmt_len) :: format)
write(format, '(a, i0, 3a)') '(ss, ', n, '(f13.6, :, "', delimiter, '"))'
else
- allocate(character(23 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+ fmt_len = 23 + len(delimiter) + floor(log10(real(n))) + 1
+ allocate(character(len=fmt_len) :: format)
write(format, '(a, i0, 3a)') '(ss, ', n, '(es13.6e2, :, "', delimiter, '"))'
end if
diff --git a/src/data/registry.xml b/src/data/registry.xml
index 98da28a49..5992cd32c 100644
--- a/src/data/registry.xml
+++ b/src/data/registry.xml
@@ -491,12 +491,12 @@
+ units="days" type="real" kind="kind_phys">
fractional calendar day at end of current timestep
+ units="days" type="real" kind="kind_phys">
fractional calendar day at end of next timestep
QRL pbuf_QRL
2
diff --git a/src/history/cam_history_support.F90 b/src/history/cam_history_support.F90
index 0e4eec70b..c5a26d148 100644
--- a/src/history/cam_history_support.F90
+++ b/src/history/cam_history_support.F90
@@ -585,6 +585,7 @@ subroutine add_hist_coord_r8(name, vlen, long_name, units, values, &
integer, pointer :: local_int_values(:)
real(r8), pointer :: local_bounds(:,:)
type(formula_terms_t) :: local_formula_terms
+ real(r8), pointer :: r_ptr(:)
nullify(local_int_values)
nullify(local_bounds)
@@ -626,8 +627,9 @@ subroutine add_hist_coord_r8(name, vlen, long_name, units, values, &
end if
! First, check to see if it is OK to add this coord
+ r_ptr => values
i = check_hist_coord(name, vlen, long_name, units, local_bounds, &
- local_int_values, values, local_bounds_name, local_positive, &
+ local_int_values, r_ptr, local_bounds_name, local_positive, &
local_standard_name, local_formula_terms)
! Register the name if necessary
diff --git a/src/physics/ncar_ccpp b/src/physics/ncar_ccpp
index b47246ded..b3abf21d9 160000
--- a/src/physics/ncar_ccpp
+++ b/src/physics/ncar_ccpp
@@ -1 +1 @@
-Subproject commit b47246dedfa7c377ac2bfb060e2d465beb931f2a
+Subproject commit b3abf21d92b101a899dec719c436e0cc046f1251
diff --git a/test/unit/python/test_cam_autogen.py b/test/unit/python/test_cam_autogen.py
index 48fd32236..1b46b2dee 100644
--- a/test/unit/python/test_cam_autogen.py
+++ b/test/unit/python/test_cam_autogen.py
@@ -586,7 +586,7 @@ def test_generate_physics_suites(self):
gen_results = generate_physics_suites(self.test_cache, "UNSET", "cam", "simple",
_CAM_ROOT_DIR, self.test_bldroot,
self.test_reg_dir, [],
- self.test_src_mods_dir, False)
+ self.test_src_mods_dir, False, False)
#Due to the presence of a "dict_values" dictview object which needs
#to be treated in a special way, the tuples will need to be iterated
@@ -628,7 +628,7 @@ def test_missing_sdf_generate_physics_suites(self):
_ = generate_physics_suites(self.test_cache, "UNSET", "cam", "missing",
_CAM_ROOT_DIR, self.test_bldroot,
self.test_reg_dir, [],
- self.test_src_mods_dir, False)
+ self.test_src_mods_dir, False, False)
#End with
#Check that error message matches what's expected:
@@ -656,7 +656,7 @@ def test_missing_scheme_generate_physics_suites(self):
_ = generate_physics_suites(self.test_cache, "UNSET", "cam", "simple",
_CAM_ROOT_DIR, self.test_bldroot,
self.test_reg_dir, [],
- self.test_src_mods_dir, False)
+ self.test_src_mods_dir, False, False)
#End with
#Check that error message matches what's expected:
diff --git a/test/unit/python/test_cam_config.py b/test/unit/python/test_cam_config.py
index 687a197c9..0ed135446 100644
--- a/test/unit/python/test_cam_config.py
+++ b/test/unit/python/test_cam_config.py
@@ -72,7 +72,8 @@ def __init__(self):
"CAM_CPPDEFS" : "UNSET",
"NTHRDS_ATM" : 1,
"RUN_STARTDATE" : "101",
- "DEBUG" : False
+ "DEBUG" : False,
+ "OPENACC_GPU_OFFLOAD": False
}
def get_value(self, key):