cam6_3_004: Add MPAS-A dycore

.gitignore

@@ -11,6 +11,7 @@ src/physics/pumas
 src/dynamics/fv3/atmos_cubed_sphere
 libraries/FMS
 src/atmos_phys
+src/dynamics/mpas/dycore
 
 # Ignore compiled python
 buildnmlc

Externals.cfg

@@ -9,7 +9,7 @@ local_path = components/cice
 required = True
 
 [cime]
-tag = cime5.8.32
+tag = cime5.8.34
 protocol = git
 repo_url = https://github.com/ESMCI/cime
 local_path = cime

Externals_CAM.cfg

@@ -54,6 +54,14 @@ repo_url = https://github.com/ESCOMP/FV3_CESM.git
 local_path = src/dynamics/fv3/atmos_cubed_sphere
 required = True
 
+[mpas]
+local_path = src/dynamics/mpas/dycore
+protocol = git
+repo_url = https://github.com/MPAS-Dev/MPAS-Model.git
+sparse = ../.mpas_sparse_checkout
+hash = d059bdf
+required = True
+
 [externals_description]
 schema_version = 1.0.0
 

bld/build-namelist

@@ -592,7 +592,7 @@ if ( ($chem ne 'none') or ( $prog_species ) ){
 if ($chem) {
     # drydep_srf_file is only needed for prognostic MAM when the grid is unstructured.
     # structured grids can do interpolation on the fly.
-    if ($chem =~ /_mam/ and ($dyn =~ /se/ or $dyn =~ /fv3/)) {
+    if ($chem =~ /_mam/ and ($dyn =~ /se|fv3|mpas/)) {
         add_default($nl, 'drydep_srf_file');
     }
 }
@@ -3637,6 +3637,78 @@ if ($dyn =~ /se/) {
     }
 }
 
+# MPAS-Atmosphere dycore
+# ======================
+
+if ($dyn =~ /mpas/) {
+
+    ####################################################
+    # namelist group: dyn_mpas                         #
+    ####################################################
+
+    add_default($nl, 'mpas_time_integration');
+    add_default($nl, 'mpas_time_integration_order');
+    add_default($nl, 'mpas_dt');
+    add_default($nl, 'mpas_split_dynamics_transport');
+    add_default($nl, 'mpas_number_of_sub_steps');
+    add_default($nl, 'mpas_dynamics_split_steps');
+    add_default($nl, 'mpas_h_mom_eddy_visc2');
+    add_default($nl, 'mpas_h_mom_eddy_visc4');
+    add_default($nl, 'mpas_v_mom_eddy_visc2');
+    add_default($nl, 'mpas_h_theta_eddy_visc2');
+    add_default($nl, 'mpas_h_theta_eddy_visc4');
+    add_default($nl, 'mpas_v_theta_eddy_visc2');
+    add_default($nl, 'mpas_horiz_mixing');
+    add_default($nl, 'mpas_len_disp');
+    add_default($nl, 'mpas_visc4_2dsmag');
+    add_default($nl, 'mpas_del4u_div_factor');
+    add_default($nl, 'mpas_w_adv_order');
+    add_default($nl, 'mpas_theta_adv_order');
+    add_default($nl, 'mpas_scalar_adv_order');
+    add_default($nl, 'mpas_u_vadv_order');
+    add_default($nl, 'mpas_w_vadv_order');
+    add_default($nl, 'mpas_theta_vadv_order');
+    add_default($nl, 'mpas_scalar_vadv_order');
+    add_default($nl, 'mpas_scalar_advection');
+    add_default($nl, 'mpas_positive_definite');
+    add_default($nl, 'mpas_monotonic');
+    add_default($nl, 'mpas_coef_3rd_order');
+    add_default($nl, 'mpas_smagorinsky_coef');
+    add_default($nl, 'mpas_mix_full');
+    add_default($nl, 'mpas_epssm');
+    add_default($nl, 'mpas_smdiv');
+    add_default($nl, 'mpas_apvm_upwinding');
+    add_default($nl, 'mpas_h_ScaleWithMesh');
+    add_default($nl, 'mpas_zd');
+    add_default($nl, 'mpas_xnutr');
+    add_default($nl, 'mpas_do_restart');
+    add_default($nl, 'mpas_print_global_minmax_vel');
+    add_default($nl, 'mpas_print_detailed_minmax_vel');
+    add_default($nl, 'mpas_print_global_minmax_sca');
+
+    # mpas_block_decomp_file_prefix should only be set when more than one task is used.
+    # Otherwise the file is not used and should not be added to the input dataset file.
+    if ($opts{'ntasks'} > 1) {
+        add_default($nl, 'mpas_block_decomp_file_prefix');
+    }
+
+    # invoke MPAS utility to generate streams files
+    my $mpas_libdir = $cfg->get('mpas_libdir');
+
+    # Check that the executable file streams_gen is present.  streams_gen is built
+    # at the same time as the MPAS library.  This allows build-namelist
+    # to be called before the MPAS library is built (via CESM scripts).
+    if ( -x "$mpas_libdir/streams_gen") {
+
+        my $cmnd = "$mpas_libdir/streams_gen $mpas_libdir/Registry_processed.xml ".
+               "streams.atmosphere stream_list.atmosphere. listed ";
+
+        system($cmnd) == 0 or die
+            "$ProgName - ERROR: generating MPAS streams files via command:\n".
+            "$cmnd";
+    }
+}
+
 # Defaults for history output
 add_default($nl, 'history_amwg');
 add_default($nl, 'history_vdiag');
@@ -4183,6 +4255,14 @@ sub check_input_files {
 
                 if ($input_pathname_type eq 'abs') {
                     if ($inputdata_rootdir) {
+
+                        # Special cases:
+                        # The file referenced by mpas_block_decomp_file_prefix needs to have the
+                        # task count appended before being added to the inputdata file.
+                        if ($var eq 'mpas_block_decomp_file_prefix') {
+                            $pathname = "$pathname$opts{'ntasks'}";
+                        }
+
                         print $fh "$var = $pathname\n";
                     }
                     else {

bld/config_files/definition.xml

@@ -26,8 +26,8 @@ Switch specifies whether CAM is being built by the CCSM sequential scripts.  0 =
 <entry id="cpl" valid_values="mct,nuopc" value="mct" >
 Coupling framework: mct or nuopc.  Default: mct.
 </entry>
-<entry id="dyn" valid_values="eul,fv,fv3,se" value="">
-Dynamics package: eul, fv, fv3, or se.
+<entry id="dyn" valid_values="eul,fv,fv3,se,mpas" value="">
+Dynamics package: eul, fv, fv3, se, or mpas.
 </entry>
 <entry id="waccm_phys" valid_values="0,1" value="0">
 Switch to turn on waccm physics: 0 => no, 1 => yes.
@@ -292,6 +292,9 @@ Directory containing COSP library.
 <entry id="fv3core_libdir" value="">
 Directory containing FV3CORE library.
 </entry>
+<entry id="mpas_libdir" value="">
+Directory containing MPAS library.
+</entry>
 <entry id="target_os" value="">
 OS for which CAM is being built.  The default value is the name contained
 in Perl's $OSNAME variable.  This parameter allows the user to override

bld/configure

@@ -78,7 +78,7 @@ OPTIONS
      -cppdefs <string>  A string of user specified CPP defines.  Appended to
                         Makefile defaults.  E.g. -cppdefs '-DVAR1 -DVAR2'
      -cpl               Coupling framework [mct | nuopc].  Default: mct.
-     -dyn <name>        Dynamical core option: [eul | fv | se | fv3].  Default: fv.
+     -dyn <name>        Dynamical core option: [eul | fv | se | fv3 | mpas].  Default: fv.
      -edit_chem_mech    Invokes CAMCHEM_EDITOR to allow the user to edit the chemistry mechanism file
      -hgrid <name>      Specify horizontal grid.  Use nlatxnlon for spectral grids;
                         dlatxdlon for fv grids (dlat and dlon are the grid cell size
@@ -175,6 +175,7 @@ OPTIONS
      -ldflags <string>  A string of user specified load options.  Appended to
                         Makefile defaults.
      -linker <name>     User specified linker.  Overrides Makefile default of \$(FC).
+     -mpas_libdir <dir> Directory containing MPAS library.
      -mct_libdir <dir>  Directory containing MCT library.  Default: build the library from source
                         in a subdirectory of \$cam_bld.
      -mpi_inc <dir>     Directory containing MPI include files.
@@ -274,6 +275,7 @@ GetOptions(
     "max_n_rad_cnst=s"          => \$opts{'max_n_rad_cnst'},
     "mct_libdir=s"              => \$opts{'mct_libdir'},
     "microphys=s"               => \$opts{'microphys'},
+    "mpas_libdir=s"             => \$opts{'mpas_libdir'},
     "mpi_inc=s"                 => \$opts{'mpi_inc'},
     "mpi_lib=s"                 => \$opts{'mpi_lib'},
     "nadv=s"                    => \$opts{'nadv'},
@@ -694,9 +696,9 @@ if ($print>=2) { print "WACCM physics: $waccm_phys$eol"; }
 
 
 # WACCM physics only runs with FV, SE and FV3 dycores
-if ( ($waccm_phys) and ($dyn_pkg ne 'fv') and ($dyn_pkg ne 'se') and ($dyn_pkg ne 'fv3') ) {
+if ( ($waccm_phys) and ($dyn_pkg eq 'eul') ) {
     die <<"EOF";
-** ERROR: WACCM physics only runs with FV, Spectral Element and FV3 as the dycore.
+** ERROR: WACCM physics does not run with the Eulerian spectral dycore.
 EOF
 }
 
@@ -1211,6 +1213,9 @@ elsif ($dyn_pkg eq 'eul') {
 elsif ($dyn_pkg eq 'se') {
     $hgrid = 'ne16np4';
 }
+elsif ($dyn_pkg eq 'mpas') {
+    $hgrid = 'mpasa480';
+}
 $cfg_ref->set('hgrid', $hgrid);
 
 # User override.
@@ -1979,6 +1984,37 @@ if ($dyn_pkg eq 'fv3') {
     write_fv3_makefile("$cam_dir/src/dynamics/fv3/Makefile.in.fv3",  "$fv3core_libdir/Makefile");
 }
 
+#-----------------------------------------------------------------------------------------------
+# MPAS library.
+if ($dyn_pkg eq 'mpas') {
+
+    # Set the directory used to build MPAS.  Add location and library name
+    # to the user specified load flags.
+    my $mpas_libdir = '';
+    if (defined $opts{'mpas_libdir'}) {
+      $mpas_libdir = $opts{'mpas_libdir'};
+    } else
+    {
+        $mpas_libdir = "$cam_bld/mpas";
+    }
+
+    $cfg_ref->set('mpas_libdir', "$mpas_libdir");
+
+    my $ldflags = $cfg_ref->get('ldflags');
+    $ldflags .= " -L$mpas_libdir -lmpas ";
+    $cfg_ref->set('ldflags', $ldflags);
+
+    # create the build directory for mpas
+    my $bld_dir = $mpas_libdir;
+    unless (-d $bld_dir or mkdirp($bld_dir)) {
+        die "** Could not create the mpas build directory: $bld_dir\n";
+    }
+
+    # Create the MPAS Makefile from a template and copy it into the mpas bld directory
+    write_mpas_makefile("$cfgdir/../src/dynamics/mpas/Makefile", "$mpas_libdir/Makefile");
+    if ($print) { print "creating $mpas_libdir/Makefile\n"; }
+}
+
 #-----------------------------------------------------------------------------------------------
 # Write configuration files ####################################################################
 #-----------------------------------------------------------------------------------------------
@@ -2259,6 +2295,34 @@ EOF
 }
 
 
+#-------------------------------------------------------------------------------
+
+sub write_mpas_makefile
+{
+
+    my ($file_in, $file_out) = @_;
+    my  $fh_in = new IO::File;
+    my  $fh_out = new IO::File;
+
+    $fh_out->open(">$file_out") or die "** can't open file: $file_out\n";
+
+    print $fh_out  <<"EOF";
+
+MPAS_SRC_ROOT := $cam_root/src/dynamics/mpas
+
+EOF
+
+    # Copy the "template" makefile to the new makefile.
+    $fh_in->open("<$file_in") or die "** can't open file: $file_in\n";
+    while (<$fh_in>) {
+        print $fh_out $_;
+    }
+
+    $fh_out->close;
+    $fh_in->close;
+}
+
+
 #-------------------------------------------------------------------------------
 
 sub write_config_h
@@ -2291,7 +2355,7 @@ sub set_horiz_grid
 
     my ($hgrid_file, $cfg_ref) = @_;
 
-    # Get dycore/grid from the package's configuration
+    # Get dycore/grid from the configuration object
     my $dyn_pkg = $cfg_ref->get('dyn');
     my $hgrid   = $cfg_ref->get('hgrid');
 
@@ -2321,9 +2385,9 @@ sub set_horiz_grid
         $hgrid =~ m/C(\d+)/;
         $cfg_ref->set('hgrid', $hgrid);
     }
-    else {
+    elsif ($dyn_pkg =~ m/^eul$|^fv/) {
 
-        # For non-SE dycores the parameters are read from an input file,
+        # For EUL and FV dycores the parameters are read from an input file,
         # and if no dycore/grid matches are found then issue error message.
 
         my $xml = XML::Lite->new( $hgrid_file );