cam6_3_004: Add MPAS-A dycore

.gitignore

@@ -11,6 +11,11 @@ src/physics/pumas
 src/dynamics/fv3/atmos_cubed_sphere
 libraries/FMS
 src/atmos_phys
+src/dynamics/mpas/core_atmosphere
+src/dynamics/mpas/external
+src/dynamics/mpas/framework
+src/dynamics/mpas/operators
+src/dynamics/mpas/tools
 
 # Ignore compiled python
 buildnmlc

Externals.cfg

@@ -9,9 +9,9 @@ local_path = components/cice
 required = True
 
 [cime]
-tag = cime5.8.28
+tag = mpas.009-cime5.8.28
 protocol = git
-repo_url = https://github.com/ESMCI/cime
+repo_url = https://github.com/brian-eaton/cime
 local_path = cime
 required = True
 

Externals_CAM.cfg

@@ -54,6 +54,41 @@ repo_url = https://github.com/ESCOMP/FV3_CESM.git
 local_path = src/dynamics/fv3/atmos_cubed_sphere
 required = True
 
+[mpas_core]
+local_path = src/dynamics/mpas/core_atmosphere
+protocol = svn
+repo_url = https://github.com/mgduda/MPAS-Model/tags/
+tag = v6.1-cam.021/src/core_atmosphere
+required = True
+
+[mpas_external]
+local_path = src/dynamics/mpas/external
+protocol = svn
+repo_url = https://github.com/mgduda/MPAS-Model/tags/
+tag = v6.1-cam.021/src/external
+required = True
+
+[mpas_framework]
+local_path = src/dynamics/mpas/framework
+protocol = svn
+repo_url = https://github.com/mgduda/MPAS-Model/tags/
+tag = v6.1-cam.021/src/framework
+required = True
+
+[mpas_operators]
+local_path = src/dynamics/mpas/operators
+protocol = svn
+repo_url = https://github.com/mgduda/MPAS-Model/tags/
+tag = v6.1-cam.021/src/operators
+required = True
+
+[mpas_tools]
+local_path = src/dynamics/mpas/tools
+protocol = svn
+repo_url = https://github.com/mgduda/MPAS-Model/tags/
+tag = v6.1-cam.021/src/tools
+required = True
+
 [externals_description]
 schema_version = 1.0.0
 

bld/build-namelist

@@ -3637,6 +3637,87 @@ if ($dyn =~ /se/) {
     }
 }
 
+# MPAS-Atmosphere dycore
+# ======================
+
+if ($dyn =~ /mpas/) {
+
+    ####################################################
+    # namelist group: dyn_mpas                         #
+    ####################################################
+
+    add_default($nl, 'mpas_time_integration');
+    add_default($nl, 'mpas_time_integration_order');
+    add_default($nl, 'mpas_dt');
+    add_default($nl, 'mpas_split_dynamics_transport');
+    add_default($nl, 'mpas_number_of_sub_steps');
+    add_default($nl, 'mpas_dynamics_split_steps');
+    add_default($nl, 'mpas_h_mom_eddy_visc2');
+    add_default($nl, 'mpas_h_mom_eddy_visc4');
+    add_default($nl, 'mpas_v_mom_eddy_visc2');
+    add_default($nl, 'mpas_h_theta_eddy_visc2');
+    add_default($nl, 'mpas_h_theta_eddy_visc4');
+    add_default($nl, 'mpas_v_theta_eddy_visc2');
+    add_default($nl, 'mpas_horiz_mixing');
+    add_default($nl, 'mpas_len_disp');
+    add_default($nl, 'mpas_visc4_2dsmag');
+    add_default($nl, 'mpas_del4u_div_factor');
+    add_default($nl, 'mpas_w_adv_order');
+    add_default($nl, 'mpas_theta_adv_order');
+    add_default($nl, 'mpas_scalar_adv_order');
+    add_default($nl, 'mpas_u_vadv_order');
+    add_default($nl, 'mpas_w_vadv_order');
+    add_default($nl, 'mpas_theta_vadv_order');
+    add_default($nl, 'mpas_scalar_vadv_order');
+    add_default($nl, 'mpas_scalar_advection');
+    add_default($nl, 'mpas_positive_definite');
+    add_default($nl, 'mpas_monotonic');
+    add_default($nl, 'mpas_coef_3rd_order');
+    add_default($nl, 'mpas_smagorinsky_coef');
+    add_default($nl, 'mpas_mix_full');
+    add_default($nl, 'mpas_epssm');
+    add_default($nl, 'mpas_smdiv');
+    add_default($nl, 'mpas_apvm_upwinding');
+    add_default($nl, 'mpas_h_ScaleWithMesh');
+    add_default($nl, 'mpas_num_halos');
+    add_default($nl, 'mpas_zd');
+    add_default($nl, 'mpas_xnutr');
+    add_default($nl, 'mpas_number_of_blocks');
+    add_default($nl, 'mpas_explicit_proc_decomp');
+    add_default($nl, 'mpas_proc_decomp_file_prefix');
+    add_default($nl, 'mpas_do_restart');
+    add_default($nl, 'mpas_print_global_minmax_vel');
+    add_default($nl, 'mpas_print_detailed_minmax_vel');
+    add_default($nl, 'mpas_print_global_minmax_sca');
+
+    # if user has not set mpas_block_decomp_file_prefix, then get the default value which
+    # will be a relative pathname, and prepend the $inputdata_rootdir
+    my $decomp_prefix = $nl->get_value('mpas_block_decomp_file_prefix');
+    if (!defined $decomp_prefix) {
+        add_default($nl, 'mpas_block_decomp_file_prefix');
+        $decomp_prefix = $nl->get_value('mpas_block_decomp_file_prefix');
+        $decomp_prefix =~ s/['"]//g;                          # strip the quotes
+        $decomp_prefix = "$inputdata_rootdir/$decomp_prefix"; # prepend the inputdata directory
+        # set the value in the namelist
+        $nl->set_variable_value('decomposition', 'mpas_block_decomp_file_prefix', quote_string($decomp_prefix));
+    }
+
+    # invoke MPAS utility to generate streams files
+    my $mpas_libdir = $cfg->get('mpas_libdir');
+
+    # Check that the executable file streams_gen is present.  This allows build-namelist
+    # to be called before the MPAS library is built (via CESM scripts).
+    if ( -x "$mpas_libdir/streams_gen") {
+
+        my $cmnd = "$mpas_libdir/streams_gen $mpas_libdir/Registry_processed.xml ".
+               "streams.atmosphere stream_list.atmosphere. listed ";
+
+        system($cmnd) == 0 or die
+            "$ProgName - ERROR: generating MPAS streams files via command:\n".
+            "$cmnd";
+    }
+}
+
 # Defaults for history output
 add_default($nl, 'history_amwg');
 add_default($nl, 'history_vdiag');

bld/config_files/definition.xml

@@ -26,8 +26,8 @@ Switch specifies whether CAM is being built by the CCSM sequential scripts.  0 =
 <entry id="cpl" valid_values="mct,nuopc" value="mct" >
 Coupling framework: mct or nuopc.  Default: mct.
 </entry>
-<entry id="dyn" valid_values="eul,fv,fv3,se" value="">
-Dynamics package: eul, fv, fv3, or se.
+<entry id="dyn" valid_values="eul,fv,fv3,se,mpas" value="">
+Dynamics package: eul, fv, fv3, se, or mpas.
 </entry>
 <entry id="waccm_phys" valid_values="0,1" value="0">
 Switch to turn on waccm physics: 0 => no, 1 => yes.
@@ -292,6 +292,9 @@ Directory containing COSP library.
 <entry id="fv3core_libdir" value="">
 Directory containing FV3CORE library.
 </entry>
+<entry id="mpas_libdir" value="">
+Directory containing MPAS library.
+</entry>
 <entry id="target_os" value="">
 OS for which CAM is being built.  The default value is the name contained
 in Perl's $OSNAME variable.  This parameter allows the user to override

bld/configure

@@ -175,6 +175,7 @@ OPTIONS
      -ldflags <string>  A string of user specified load options.  Appended to
                         Makefile defaults.
      -linker <name>     User specified linker.  Overrides Makefile default of \$(FC).
+     -mpas_libdir <dir> Directory containing MPAS library.
      -mct_libdir <dir>  Directory containing MCT library.  Default: build the library from source
                         in a subdirectory of \$cam_bld.
      -mpi_inc <dir>     Directory containing MPI include files.
@@ -274,6 +275,7 @@ GetOptions(
     "max_n_rad_cnst=s"          => \$opts{'max_n_rad_cnst'},
     "mct_libdir=s"              => \$opts{'mct_libdir'},
     "microphys=s"               => \$opts{'microphys'},
+    "mpas_libdir=s"             => \$opts{'mpas_libdir'},
     "mpi_inc=s"                 => \$opts{'mpi_inc'},
     "mpi_lib=s"                 => \$opts{'mpi_lib'},
     "nadv=s"                    => \$opts{'nadv'},
@@ -1211,6 +1213,9 @@ elsif ($dyn_pkg eq 'eul') {
 elsif ($dyn_pkg eq 'se') {
     $hgrid = 'ne16np4';
 }
+elsif ($dyn_pkg eq 'mpas') {
+    $hgrid = 'mpasa480';
+}
 $cfg_ref->set('hgrid', $hgrid);
 
 # User override.
@@ -1979,6 +1984,37 @@ if ($dyn_pkg eq 'fv3') {
     write_fv3_makefile("$cam_dir/src/dynamics/fv3/Makefile.in.fv3",  "$fv3core_libdir/Makefile");
 }
 
+#-----------------------------------------------------------------------------------------------
+# MPAS library.
+if ($dyn_pkg eq 'mpas') {
+
+    # Set the directory used to build MPAS.  Add location and library name
+    # to the user specified load flags.
+    my $mpas_libdir = '';
+    if (defined $opts{'mpas_libdir'}) {
+      $mpas_libdir = $opts{'mpas_libdir'};
+    } else
+    {
+        $mpas_libdir = "$cam_bld/mpas";
+    }
+
+    $cfg_ref->set('mpas_libdir', "$mpas_libdir");
+
+    my $ldflags = $cfg_ref->get('ldflags');
+    $ldflags .= " -L$mpas_libdir -lmpas ";
+    $cfg_ref->set('ldflags', $ldflags);
+
+    # create the build directory for mpas
+    my $bld_dir = $mpas_libdir;
+    unless (-d $bld_dir or mkdirp($bld_dir)) {
+        die "** Could not create the mpas build directory: $bld_dir\n";
+    }
+
+    # Create the MPAS Makefile from a template and copy it into the mpas bld directory
+    write_mpas_makefile("$cfgdir/../src/dynamics/mpas/Makefile", "$mpas_libdir/Makefile");
+    if ($print) { print "creating $mpas_libdir/Makefile\n"; }
+}
+
 #-----------------------------------------------------------------------------------------------
 # Write configuration files ####################################################################
 #-----------------------------------------------------------------------------------------------
@@ -2259,6 +2295,34 @@ EOF
 }
 
 
+#-------------------------------------------------------------------------------
+
+sub write_mpas_makefile
+{
+
+    my ($file_in, $file_out) = @_;
+    my  $fh_in = new IO::File;
+    my  $fh_out = new IO::File;
+
+    $fh_out->open(">$file_out") or die "** can't open file: $file_out\n";
+
+    print $fh_out  <<"EOF";
+
+MPAS_SRC_ROOT := $cam_root/src/dynamics/mpas
+
+EOF
+
+    # Copy the "template" makefile to the new makefile.
+    $fh_in->open("<$file_in") or die "** can't open file: $file_in\n";
+    while (<$fh_in>) {
+        print $fh_out $_;
+    }
+
+    $fh_out->close;
+    $fh_in->close;
+}
+
+
 #-------------------------------------------------------------------------------
 
 sub write_config_h
@@ -2291,7 +2355,7 @@ sub set_horiz_grid
 
     my ($hgrid_file, $cfg_ref) = @_;
 
-    # Get dycore/grid from the package's configuration
+    # Get dycore/grid from the configuration object
     my $dyn_pkg = $cfg_ref->get('dyn');
     my $hgrid   = $cfg_ref->get('hgrid');
 
@@ -2321,9 +2385,9 @@ sub set_horiz_grid
         $hgrid =~ m/C(\d+)/;
         $cfg_ref->set('hgrid', $hgrid);
     }
-    else {
+    elsif ($dyn_pkg =~ m/^eul$|^fv/) {
 
-        # For non-SE dycores the parameters are read from an input file,
+        # For EUL and FV dycores the parameters are read from an input file,
         # and if no dycore/grid matches are found then issue error message.
 
         my $xml = XML::Lite->new( $hgrid_file );

bld/namelist_files/namelist_defaults_cam.xml

@@ -33,12 +33,25 @@
 <dtime dyn="se"    hgrid="ne0np4.ARCTIC.ne30x4">450</dtime>
 <dtime dyn="se"    hgrid="ne0np4.ARCTICGRIS.ne30x8">225</dtime>
 
-<!-- Initial conditions -->
-<!-- Vertical coordinates only, good for analytic initial conditions -->
+<dtime dyn="mpas"                  >1800</dtime>
+
+<!-- Initial files -->
+
+<!-- Vertical coordinates only -->
 <ncdata nlev="26" analytic_ic="1" >atm/cam/inic/cam_vcoords_L26_c180105.nc</ncdata>
 <ncdata nlev="30" analytic_ic="1" >atm/cam/inic/cam_vcoords_L30_c180105.nc</ncdata>
 <ncdata nlev="32" analytic_ic="1" >atm/cam/inic/cam_vcoords_L32_c180105.nc</ncdata>
 
+<!-- Vertical coordinates and grid only -->
+<ncdata hgrid="mpasa480" nlev="32" analytic_ic="1" >atm/cam/inic/mpas/mpasa480_L32_v6.1.grid.nc</ncdata>
+<ncdata hgrid="mpasa120" nlev="32" analytic_ic="1" >atm/cam/inic/mpas/mpasa120_L32_v6.1.grid.nc</ncdata>
+<ncdata hgrid="mpasa60"  nlev="32" analytic_ic="1" >atm/cam/inic/mpas/mpasa60_L32_v6.1.grid.nc</ncdata>
+<ncdata hgrid="mpasa30"  nlev="32" analytic_ic="1" >atm/cam/inic/mpas/mpasa30_L32_v6.1.grid.nc</ncdata>
+<ncdata hgrid="mpasa15"  nlev="32" analytic_ic="1" >atm/cam/inic/mpas/mpasa15_L32_v6.1.grid.nc</ncdata>
+<ncdata hgrid="mpasa12"  nlev="32" analytic_ic="1" >atm/cam/inic/mpas/mpasa12_L32_v6.1.grid.nc</ncdata>
+<ncdata hgrid="mpasa15-3" nlev="32" analytic_ic="1" >atm/cam/inic/mpas/mpasa15-3_L32_v6.1.grid.nc</ncdata>
+
+<!-- Files with initial conditions -->
 <ncdata dyn="fv"  hgrid="0.23x0.31" nlev="26" ic_ymd="101"      >atm/cam/inic/fv/cami_0000-01-01_0.23x0.31_L26_c100513.nc</ncdata>
 <ncdata dyn="fv"  hgrid="0.23x0.31" nlev="26" ic_ymd="901"      >atm/cam/inic/fv/cami_0000-09-01_0.23x0.31_L26_c061106.nc</ncdata>
 <ncdata dyn="fv"  hgrid="0.47x0.63" nlev="26" ic_ymd="101"      >atm/cam/inic/fv/cami_1980-01-01_0.47x0.63_L26_c071226.nc</ncdata>
@@ -227,6 +240,8 @@
 
 <ncdata dyn="se" hgrid="ne0np4CONUS.ne30x8"       nlev="70" ic_ymd="101">atm/waccm/ic/FW2000_CONUS_30x8_L70_01-01-0001_c200602.nc</ncdata>
 
+<ncdata hgrid="mpasa480" nlev="26" >atm/cam/inic/mpas/mpasa480_L26_v6.1_bwave.nc</ncdata>
+
 <!-- Topography -->
 <bnd_topo hgrid="256x512" >atm/cam/topo/topo-from-cami_0000-01-01_256x512_L26_c030918.nc</bnd_topo>
 <bnd_topo hgrid="128x256" >atm/cam/topo/USGS-gtopo30_128x256_c050520.nc</bnd_topo>
@@ -2657,6 +2672,67 @@
 <se_tracer_num_threads> 0 </se_tracer_num_threads>
 <se_vert_num_threads> 0 </se_vert_num_threads>
 
+
+<!-- ================================================================== -->
+<!-- Defaults for MPAS                                                  -->
+<!-- ================================================================== -->
+<mpas_time_integration>SRK3</mpas_time_integration>
+<mpas_time_integration_order>2</mpas_time_integration_order>
+<mpas_dt                 > 1800.0 </mpas_dt>
+<mpas_dt hgrid="mpasa120">  600.0 </mpas_dt>
+
+<mpas_split_dynamics_transport>true</mpas_split_dynamics_transport>
+<mpas_number_of_sub_steps>2</mpas_number_of_sub_steps>
+<mpas_dynamics_split_steps>3</mpas_dynamics_split_steps>
+<mpas_h_mom_eddy_visc2>0.0</mpas_h_mom_eddy_visc2>
+<mpas_h_mom_eddy_visc4>0.0</mpas_h_mom_eddy_visc4>
+<mpas_v_mom_eddy_visc2>0.0</mpas_v_mom_eddy_visc2>
+<mpas_h_theta_eddy_visc2>0.0</mpas_h_theta_eddy_visc2>
+<mpas_h_theta_eddy_visc4>0.0</mpas_h_theta_eddy_visc4>
+<mpas_v_theta_eddy_visc2>0.0</mpas_v_theta_eddy_visc2>
+<mpas_horiz_mixing>2d_smagorinsky</mpas_horiz_mixing>
+
+<mpas_len_disp hgrid="mpasa480">480000.0</mpas_len_disp>
+<mpas_len_disp hgrid="mpasa120">120000.0</mpas_len_disp>
+
+<mpas_visc4_2dsmag>0.05</mpas_visc4_2dsmag>
+<mpas_del4u_div_factor>10.0</mpas_del4u_div_factor>
+<mpas_w_adv_order>3</mpas_w_adv_order>
+<mpas_theta_adv_order>3</mpas_theta_adv_order>
+<mpas_scalar_adv_order>3</mpas_scalar_adv_order>
+<mpas_u_vadv_order>3</mpas_u_vadv_order>
+<mpas_w_vadv_order>3</mpas_w_vadv_order>
+<mpas_theta_vadv_order>3</mpas_theta_vadv_order>
+<mpas_scalar_vadv_order>3</mpas_scalar_vadv_order>
+<mpas_scalar_advection>true</mpas_scalar_advection>
+<mpas_positive_definite>false</mpas_positive_definite>
+<mpas_monotonic>true</mpas_monotonic>
+<mpas_coef_3rd_order>0.25</mpas_coef_3rd_order>
+<mpas_smagorinsky_coef>0.125</mpas_smagorinsky_coef>
+<mpas_mix_full>true</mpas_mix_full>
+<mpas_epssm>0.1</mpas_epssm>
+<mpas_smdiv>0.1</mpas_smdiv>
+<mpas_apvm_upwinding>0.5</mpas_apvm_upwinding>
+<mpas_h_ScaleWithMesh>true</mpas_h_ScaleWithMesh>
+<mpas_num_halos>2</mpas_num_halos>
+<mpas_zd>22000.0</mpas_zd>
+<mpas_xnutr>0.2</mpas_xnutr>
+<mpas_number_of_blocks>0</mpas_number_of_blocks>
+<mpas_explicit_proc_decomp>false</mpas_explicit_proc_decomp>
+<mpas_proc_decomp_file_prefix>graph.info.part.</mpas_proc_decomp_file_prefix>
+<mpas_do_restart>false</mpas_do_restart>
+<mpas_print_global_minmax_vel>true</mpas_print_global_minmax_vel>
+<mpas_print_detailed_minmax_vel>false</mpas_print_detailed_minmax_vel>
+<mpas_print_global_minmax_sca>false</mpas_print_global_minmax_sca>
+
+<mpas_block_decomp_file_prefix hgrid="mpasa480">atm/cam/inic/mpas/mpasa480.graph.info.part.</mpas_block_decomp_file_prefix>
+<mpas_block_decomp_file_prefix hgrid="mpasa120">atm/cam/inic/mpas/mpasa120.graph.info.part.</mpas_block_decomp_file_prefix>
+<mpas_block_decomp_file_prefix hgrid="mpasa60">atm/cam/inic/mpas/mpasa60.graph.info.part.</mpas_block_decomp_file_prefix>
+<mpas_block_decomp_file_prefix hgrid="mpasa30">atm/cam/inic/mpas/mpasa30.graph.info.part.</mpas_block_decomp_file_prefix>
+<mpas_block_decomp_file_prefix hgrid="mpasa15">atm/cam/inic/mpas/mpasa15.graph.info.part.</mpas_block_decomp_file_prefix>
+<mpas_block_decomp_file_prefix hgrid="mpasa12">atm/cam/inic/mpas/mpasa12.graph.info.part.</mpas_block_decomp_file_prefix>
+<mpas_block_decomp_file_prefix hgrid="mpasa15-3">atm/cam/inic/mpas/mpasa15-3.graph.info.part.</mpas_block_decomp_file_prefix>
+
 <!-- ================================================================== -->
 <!-- Chemistry Rates Diagnostics                                        -->
 <!-- ================================================================== -->