sima0_11_001: Bring in gpuized RRTMGP

.gitmodules

@@ -20,7 +20,7 @@
 [submodule "ncar-physics"]
         path          = src/physics/ncar_ccpp
 	url           = https://github.com/ESCOMP/atmospheric_physics
-	fxtag         = b47246dedfa7c377ac2bfb060e2d465beb931f2a
+	fxtag         = atmos_phys0_21_000
         fxrequired    = AlwaysRequired
         fxDONOTUSEurl = https://github.com/ESCOMP/atmospheric_physics
 [submodule "rrtmgp-data"]
@@ -32,7 +32,7 @@
 [submodule "ccs_config"]
         path          = ccs_config
         url           = https://github.com/ESMCI/ccs_config_cesm.git
-        fxtag         = ccs_config_cesm1.0.65
+        fxtag         = ccs_config_cesm1.0.72
         fxrequired    = ToplevelRequired
         fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
 [submodule "cdeps"]

cime_config/cam_autogen.py

@@ -357,6 +357,68 @@ def _update_genccpp_dir(utility_files, genccpp_dir):
         # end if
     # end for
 
+###############################################################################
+def _set_rrtmgp_dependencies(dependency_files, gpu_flag):
+###############################################################################
+    """
+    Modify the list of files that physics schemes depend on
+    (as provided by the CCPP) so that the correct RRTMGP
+    dependencies exist for either CPUs or GPUs.
+
+    This function returns a modified list with either the
+    CPU or GPU RRTMGP dependencies, but not both.
+
+    >>> in_files = ['/some/path/file.F90','/some/rte-kernels/rrtmgp.F90', \
+                    '/some/rte-kernels/accel/rrtmgp.F90', \
+                    '/some/rte-kernels/mo_rte_solver_kernels.F90', \
+                    '/some/rte-kernels/accel/mo_rte_solver_kernels.F90', \
+                    '/some/other/path/file.F90', \
+                    '/other/rrtmgp-kernels/mo_gas_optics_rrtmgp_kernels.F90', \
+                    '/other/rrtmgp-kernels/accel/mo_gas_optics_rrtmgp_kernels.F90', ]
+    >>> _set_rrtmgp_dependencies(in_files, False)
+    ['/some/path/file.F90', '/some/rte-kernels/rrtmgp.F90', \
+'/some/rte-kernels/accel/rrtmgp.F90', '/some/rte-kernels/mo_rte_solver_kernels.F90', \
+'/some/other/path/file.F90', '/other/rrtmgp-kernels/mo_gas_optics_rrtmgp_kernels.F90']
+
+    >>> _set_rrtmgp_dependencies(in_files, True)
+    ['/some/path/file.F90', '/some/rte-kernels/rrtmgp.F90', \
+'/some/rte-kernels/accel/rrtmgp.F90', \
+'/some/rte-kernels/accel/mo_rte_solver_kernels.F90', '/some/other/path/file.F90', \
+'/other/rrtmgp-kernels/accel/mo_gas_optics_rrtmgp_kernels.F90']
+    """
+
+    # Create new list of CCPP-dependent files:
+    new_dependency_files = []
+
+    # List of directory strings that indicate
+    # RRTMGP dependencies:
+    rrtmgp_subdirs = ['rte-kernels', 'rrtmgp-kernels']
+
+    # List of RRTMGP files that differ between CPUs and GPUs:
+    rrtmgp_files = ['mo_rte_solver_kernels.F90',
+                    'mo_optical_props_kernels.F90',
+                    'mo_gas_optics_rrtmgp_kernels.F90']
+
+
+    for file_path in dependency_files:
+        if any(subdir in file_path for subdir in rrtmgp_subdirs):
+            if any(rfile in file_path for rfile in rrtmgp_files):
+                if (gpu_flag and 'accel' in file_path):
+                    #If GPU-enabled and is "accelerated", include it:
+                    new_dependency_files.append(file_path)
+                elif (not gpu_flag and 'accel' not in file_path):
+                    #If CPU-only and not "accelerated", include it:
+                    new_dependency_files.append(file_path)
+            else:
+                #Not a hardware-dependent file, so include it:
+                new_dependency_files.append(file_path)
+        else:
+            #Not an RRTMGP file, so include it:
+            new_dependency_files.append(file_path)
+
+    # Return newly-modified dependency files:
+    return new_dependency_files
+
 ###############################################################################
 def generate_registry(data_search, build_cache, atm_root, bldroot,
                       source_mods_dir, dycore, gen_fort_indent):
@@ -420,7 +482,8 @@ def generate_registry(data_search, build_cache, atm_root, bldroot,
 ###############################################################################
 def generate_physics_suites(build_cache, preproc_defs, host_name,
                             phys_suites_str, atm_root, bldroot,
-                            reg_dir, reg_files, source_mods_dir, force):
+                            reg_dir, reg_files, source_mods_dir,
+                            gpu_flag, force):
 ###############################################################################
     """
     Generate the source for the configured physics suites,
@@ -664,8 +727,6 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
         build_cache.update_ccpp(sdfs, scheme_files, host_files, xml_files,
                                 scheme_nl_meta_files, nl_groups, create_nl_file,
                                 preproc_cache_str, kind_types)
-        ##XXgoldyXX: v Temporary fix: Copy CCPP Framework source code into
-        ##XXgoldyXX: v   generated code directory
         request = DatatableReport("utility_files")
         ufiles_str = datatable_report(cap_output_file, request, ";")
         utility_files = ufiles_str.split(';')
@@ -674,8 +735,15 @@ def generate_physics_suites(build_cache, preproc_defs, host_name,
         dep_str = datatable_report(cap_output_file, request, ";")
         if len(dep_str) > 0:
             dependency_files = dep_str.split(';')
+            # If using RRTMGP in the physics suite, then modify
+            # the provided dependency files list to use the correct
+            # CPU or GPU RRTMGP dependencies:
+            if any("rrtmgp_" in scheme_name for scheme_name in scheme_names):
+                dependency_files = _set_rrtmgp_dependencies(dependency_files,
+                                                            gpu_flag)
+
+            # Copy dependencies files into CCPP build directory
             _update_genccpp_dir(dependency_files, genccpp_dir)
-        ##XXgoldyXX: ^ Temporary fix:
     # End if
 
     return [physics_blddir, genccpp_dir], do_gen_ccpp, cap_output_file,       \
@@ -695,7 +763,7 @@ def generate_init_routines(build_cache, bldroot, force_ccpp, force_init,
     and/or script).
     """
 
-    #Add new directory to build path:
+    # Add new directory to build path:
     init_dir = os.path.join(bldroot, "phys_init")
     # Use this for cache check
     gen_init_file = os.path.join(_REG_GEN_DIR, "write_init_files.py")
@@ -706,11 +774,11 @@ def generate_init_routines(build_cache, bldroot, force_ccpp, force_init,
         if force_ccpp or force_init:
             do_gen_init = True
         else:
-            #If not, then check cache to see if actual
-            #"write_init_files.py" was modified:
+            # If not, then check cache to see if actual
+            # "write_init_files.py" was modified:
             do_gen_init = build_cache.init_write_mismatch(gen_init_file)
     else:
-        #If no directory exists, then one will need
+        # If no directory exists, then one will need
         # to create new routines:
         os.mkdir(init_dir)
         do_gen_init = True

cime_config/cam_config.py

@@ -186,8 +186,9 @@ def __init__(self, case, case_log):
         # Save case variables needed for code auto-generation:
         self.__atm_root = case.get_value("COMP_ROOT_DIR_ATM")
         self.__caseroot = case.get_value("CASEROOT")
-        self.__bldroot = os.path.join(exeroot, "atm", "obj")
+        self.__bldroot  = os.path.join(exeroot, "atm", "obj")
         self.__atm_name = case.get_value("COMP_ATM")
+        self.__gpu_flag = case.get_value("OPENACC_GPU_OFFLOAD") #Returns a Boolean
 
         # Save CPP definitions as a list:
         self.__cppdefs = [x for x in case.get_value("CAM_CPPDEFS").split() if x]
@@ -879,7 +880,7 @@ def generate_cam_src(self, gen_fort_indent):
                                           self.__atm_name, phys_suites,
                                           self.__atm_root, self.__bldroot,
                                           reg_dir, reg_files, source_mods_dir,
-                                          force_ccpp)
+                                          self.__gpu_flag, force_ccpp)
         phys_dirs, force_init, _, nml_fils, capgen_db, scheme_names = retvals
 
         # Add namelist definition files to dictionary:

cime_config/testdefs/testlist_cam.xml

@@ -115,6 +115,15 @@
       <option name="comment">Snapshot test for RRTMGP</option>
     </options>
   </test>
+  <test compset="FPHYStest" grid="ne3pg3_ne3pg3_mg37" name="SMS_Ln2" testmods="cam/outfrq_rrtmgp_derecho_gpu">
+    <machines>
+      <machine name="derecho" compiler="nvhpc" category="aux_sima"/>
+    </machines>
+    <options>
+      <option name="wallclock">00:10:00</option>
+      <option name="comment">GPU snapshot test for RRTMGP</option>
+    </options>
+  </test>
 
   <!-- Derecho dycore tests -->
   <test compset="FKESSLER" grid="mpasa480_mpasa480" name="SMS_Ln9" testmods="cam/outfrq_kessler_mpas_derecho">

cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/shell_commands

@@ -0,0 +1,13 @@
+./xmlchange NTASKS=128
+./xmlchange NTHRDS=1
+./xmlchange ROOTPE='0'
+./xmlchange ROF_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange GLC_NCPL=`./xmlquery --value ATM_NCPL`
+./xmlchange TIMER_DETAIL='6'
+./xmlchange TIMER_LEVEL='999'
+./xmlchange GPU_TYPE=a100
+./xmlchange OPENACC_GPU_OFFLOAD=TRUE
+./xmlchange OVERSUBSCRIBE_GPU=TRUE
+./xmlchange NGPUS_PER_NODE=4
+./xmlchange CAM_CONFIG_OPTS="--dyn none --physics-suites rrtmgp"
+./xmlchange RUN_STARTDATE=1979-01-01

cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/user_nl_cam

@@ -0,0 +1,25 @@
+! these are CPU FHISTC_LTso snapshots
+ncdata = '/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_fhistc_ltso_rrtmgp_derecho_gnu_before_c20251013.nc'
+ncdata_check = '/glade/campaign/cesm/community/amwg/sima_baselines/cam_sima_test_snapshots/cam_ne3pg3_fhistc_ltso_rrtmgp_derecho_gnu_after_c20251013.nc'
+
+! tolerances for testing (currently have high tolerance due to CPU snapshots)
+ncdata_check_err = .true.
+min_difference = 1e-08
+
+! vertical levels in snapshot
+pver = 58
+
+! Do radiation on every timestep we're testing
+irad_always=3
+
+! diagnostic output
+hist_output_frequency;h1: 1*nsteps
+hist_precision;h1: REAL64
+hist_add_inst_fields;h1: HR
+! Cloud output
+hist_add_inst_fields;h1: TOT_CLD_VISTAU,TOT_ICLD_VISTAU,ICE_ICLD_VISTAU,LIQ_ICLD_VISTAU
+! Longwave diagnostic output
+hist_add_inst_fields;h1: QRL,QRLC,FLNT,FLNTC,FLUT,FLUTC,LWCF,FLN200,FLN200C,FLNR,FLNS,FLNSC,FLDS,FLDSC,FUL,FDL,FULC,FDLC
+! Shortwave diagnostic fields
+hist_add_inst_fields;h1: SOLIN,QRS,QRSC,FSNT,FSNTC,FSNTOA,FSNTOAC,SWCF,FSUTOA,FSN200,FSN200C,FSNR,SOLL,SOLS,SOLLD,SOLSD
+hist_add_inst_fields;h1: FSNS,FSNSC,FSDS,FSDSC,FUS,FDS,FUSC,FDSC

cime_config/testdefs/testmods_dirs/cam/outfrq_rrtmgp_derecho_gpu/user_nl_cpl

@@ -0,0 +1,5 @@
+! Set fixed orbital parameters
+orb_mode='fixed_parameters'
+orb_eccen = 0.
+orb_obliq = 0.
+orb_mvelp = 0.

src/core_utils/string_core_utils.F90

@@ -124,7 +124,7 @@ pure function stringify(value, separator)
 
         character(:), allocatable :: buffer, delimiter, format
         character(:), allocatable :: value_c(:)
-        integer :: i, n, offset
+        integer :: i, n, offset, fmt_len
 
         if (present(separator)) then
             delimiter = separator
@@ -168,43 +168,50 @@ pure function stringify(value, separator)
 
                 deallocate(value_c)
             type is (integer(int32))
-                allocate(character(11 * n + len(delimiter) * (n - 1)) :: buffer)
-                allocate(character(17 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+                allocate(character(len=11 * n + len(delimiter) * (n - 1)) :: buffer)
+                fmt_len = 17 + len(delimiter) + floor(log10(real(n))) + 1
+                allocate(character(fmt_len) :: format)
 
                 write(format, '(a, i0, 3a)') '(ss, ', n, '(i0, :, "', delimiter, '"))'
                 write(buffer, format) value
             type is (integer(int64))
-                allocate(character(20 * n + len(delimiter) * (n - 1)) :: buffer)
-                allocate(character(17 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+                allocate(character(len=20 * n + len(delimiter) * (n - 1)) :: buffer)
+                fmt_len = 17 + len(delimiter) + floor(log10(real(n))) + 1
+                allocate(character(len=fmt_len) :: format)
 
                 write(format, '(a, i0, 3a)') '(ss, ', n, '(i0, :, "', delimiter, '"))'
                 write(buffer, format) value
             type is (logical)
-                allocate(character(1 * n + len(delimiter) * (n - 1)) :: buffer)
-                allocate(character(13 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+                allocate(character(len=1 * n + len(delimiter) * (n - 1)) :: buffer)
+                fmt_len = 13 + len(delimiter) + floor(log10(real(n))) + 1
+                allocate(character(len=fmt_len) :: format)
 
                 write(format, '(a, i0, 3a)') '(', n, '(l1, :, "', delimiter, '"))'
                 write(buffer, format) value
             type is (real(real32))
-                allocate(character(13 * n + len(delimiter) * (n - 1)) :: buffer)
+                allocate(character(len=13 * n + len(delimiter) * (n - 1)) :: buffer)
 
                 if (maxval(abs(value)) < 1.0e5_real32) then
-                    allocate(character(20 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+                    fmt_len = 20 + len(delimiter) + floor(log10(real(n))) + 1
+                    allocate(character(len=fmt_len) :: format)
                     write(format, '(a, i0, 3a)') '(ss, ', n, '(f13.6, :, "', delimiter, '"))'
                 else
-                    allocate(character(23 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+                    fmt_len = 23 + len(delimiter) + floor(log10(real(n))) + 1
+                    allocate(character(len=fmt_len) :: format)
                     write(format, '(a, i0, 3a)') '(ss, ', n, '(es13.6e2, :, "', delimiter, '"))'
                 end if
 
                 write(buffer, format) value
             type is (real(real64))
-                allocate(character(13 * n + len(delimiter) * (n - 1)) :: buffer)
+                allocate(character(len=13 * n + len(delimiter) * (n - 1)) :: buffer)
 
                 if (maxval(abs(value)) < 1.0e5_real64) then
-                    allocate(character(20 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+                    fmt_len = 20 + len(delimiter) + floor(log10(real(n))) + 1
+                    allocate(character(len=fmt_len) :: format)
                     write(format, '(a, i0, 3a)') '(ss, ', n, '(f13.6, :, "', delimiter, '"))'
                 else
-                    allocate(character(23 + len(delimiter) + floor(log10(real(n))) + 1) :: format)
+                    fmt_len = 23 + len(delimiter) + floor(log10(real(n))) + 1
+                    allocate(character(len=fmt_len) :: format)
                     write(format, '(a, i0, 3a)') '(ss, ', n, '(es13.6e2, :, "', delimiter, '"))'
                 end if
 

src/data/registry.xml

@@ -491,12 +491,12 @@
     <!-- Date and time variables -->
     <variable local_name="calday"
               standard_name="fractional_calendar_days_on_end_of_current_timestep"
-              units="1" type="real" kind="kind_phys">
+              units="days" type="real" kind="kind_phys">
       <long_name>fractional calendar day at end of current timestep</long_name>
     </variable>
     <variable local_name="next_calday"
               standard_name="fractional_calendar_days_on_end_of_next_timestep"
-              units="1" type="real" kind="kind_phys">
+              units="days" type="real" kind="kind_phys">
       <long_name>fractional calendar day at end of next timestep</long_name>
     </variable>
     <variable local_name="nextsw_cday"
@@ -1826,7 +1826,7 @@
       <ic_file_input_names>QRL pbuf_QRL</ic_file_input_names>
     </variable>
     <variable local_name="two"
-              standard_name="ccpp_constant_two"
+              standard_name="constant_dimension_two"
               units="count" type="integer"
 	      allocatable="parameter">
       <initial_value>2</initial_value>

src/history/cam_history_support.F90

@@ -585,6 +585,7 @@ subroutine add_hist_coord_r8(name, vlen, long_name, units, values,         &
     integer,        pointer                         :: local_int_values(:)
     real(r8),       pointer                         :: local_bounds(:,:)
     type(formula_terms_t)                           :: local_formula_terms
+    real(r8),       pointer                         :: r_ptr(:)
 
     nullify(local_int_values)
     nullify(local_bounds)
@@ -626,8 +627,9 @@ subroutine add_hist_coord_r8(name, vlen, long_name, units, values,         &
     end if
 
     ! First, check to see if it is OK to add this coord
+    r_ptr => values
     i = check_hist_coord(name, vlen, long_name, units, local_bounds,     &
-            local_int_values, values, local_bounds_name, local_positive,           &
+            local_int_values, r_ptr, local_bounds_name, local_positive,           &
             local_standard_name, local_formula_terms)
 
     ! Register the name if necessary

test/unit/python/test_cam_autogen.py

@@ -586,7 +586,7 @@ def test_generate_physics_suites(self):
         gen_results = generate_physics_suites(self.test_cache, "UNSET", "cam", "simple",
                                               _CAM_ROOT_DIR, self.test_bldroot,
                                               self.test_reg_dir, [],
-                                              self.test_src_mods_dir, False)
+                                              self.test_src_mods_dir, False, False)
 
         #Due to the presence of a "dict_values" dictview object which needs
         #to be treated in a special way, the tuples will need to be iterated
@@ -628,7 +628,7 @@ def test_missing_sdf_generate_physics_suites(self):
             _ = generate_physics_suites(self.test_cache, "UNSET", "cam", "missing",
                                         _CAM_ROOT_DIR, self.test_bldroot,
                                         self.test_reg_dir, [],
-                                        self.test_src_mods_dir, False)
+                                        self.test_src_mods_dir, False, False)
         #End with
 
         #Check that error message matches what's expected:
@@ -656,7 +656,7 @@ def test_missing_scheme_generate_physics_suites(self):
             _ = generate_physics_suites(self.test_cache, "UNSET", "cam", "simple",
                                         _CAM_ROOT_DIR, self.test_bldroot,
                                         self.test_reg_dir, [],
-                                        self.test_src_mods_dir, False)
+                                        self.test_src_mods_dir, False, False)
         #End with
 
         #Check that error message matches what's expected:

test/unit/python/test_cam_config.py

@@ -72,7 +72,8 @@ def __init__(self):
             "CAM_CPPDEFS" : "UNSET",
             "NTHRDS_ATM" : 1,
             "RUN_STARTDATE" : "101",
-            "DEBUG" : False
+            "DEBUG" : False,
+            "OPENACC_GPU_OFFLOAD": False
             }
 
     def get_value(self, key):