cam6_3_016: Refactor coupling with WACCMX ionosphere

bld/build-namelist

@@ -2754,20 +2754,15 @@ if ($waccmx) {
   if (($ionos eq 'none') and ($wmx_opt =~ /ionosphere/) ) {
     die "CAM Namelist ERROR: WACCMX must be configured with an active ionosphere wmx_opt is set to ionosphere\n";
   }
-  if (($ionos ne 'none') and ($wmx_opt =~ /neutral/) ) {
+  if (($ionos eq 'wxie') and ($wmx_opt =~ /neutral/) ) {
     die "CAM Namelist ERROR: WACCMX cannot be configured with an active ionosphere wmx_opt is set to neutral\n";
   }
 
-  if (($ionos eq 'wxie') and ($wmx_opt =~ /ionosphere/) and ($dyn eq 'fv')) {
+  if (($ionos eq 'wxie') and ($wmx_opt =~ /ionosphere/)) {
     # turn on electro-dynamo generated ion drift velocities
     add_default($nl, 'ionos_xport_active', 'val'=>'.true.');
     add_default($nl, 'ionos_edyn_active', 'val'=>'.true.');
     add_default($nl, 'empirical_ion_velocities', 'val'=>'.false.');
-  } elsif (($ionos eq 'wxi') and ($wmx_opt =~ /ionosphere/) and ($dyn eq 'fv')) {
-    # turn off electro-dynamo generated ion drift velocities
-    add_default($nl, 'ionos_xport_active', 'val'=>'.true.');
-    add_default($nl, 'ionos_edyn_active', 'val'=>'.false.');
-    add_default($nl, 'empirical_ion_velocities', 'val'=>'.true.');
   } elsif (($ionos eq 'none') and ($wmx_opt =~ /neutral/)) {
     add_default($nl, 'ionos_xport_active', 'val'=>'.false.');
     add_default($nl, 'ionos_edyn_active', 'val'=>'.false.');
@@ -2778,6 +2773,10 @@ if ($waccmx) {
   if  ($dyn eq 'fv') {
     add_default($nl, 'fv_high_altitude', 'val'=>'.true.');
   }
+  if ($ionos ne 'none') {
+    add_default($nl, 'cam_physics_mesh');
+    add_default($nl, 'oplus_grid');
+  }
   add_default($nl,'dadadj_niter');
   add_default($nl,'ionos_epotential_model');
   if ($nl->get_value('ionos_epotential_model') =~ 'weimer') {

bld/config_files/definition.xml

@@ -49,9 +49,12 @@ Switch to turn on analytic initial conditions for the dynamics state:
 <entry id="waccmx" valid_values="0,1" value="0">
 Option to turn on waccmx thermosphere/ionosphere extension: 0 => no, 1 => yes
 </entry>
-<entry id="ionosphere" valid_values="none,wxi,wxie" value="none">
+<entry id="ionosphere" valid_values="none,wxie" value="none">
 Ionosphere model used in WACCMX.
 </entry>
+<entry id="mgrid" valid_values="97x80,193x160,385x320,769x640" value="97x80">
+Magnetic grid specifier for the ionosphere: 97x80, 193x160, 385x320, 769x640.
+</entry>
 <entry id="phys" valid_values="cam3,cam4,cam5,cam6,held_suarez,adiabatic,kessler,tj2016,spcam_sam1mom,spcam_m2005" value="">
 Physics package: cam3, cam4, cam5, cam6, held_suarez, adiabatic, kessler, tj2016, spcam_sam1mom, spcam_m2005.
 </entry>

bld/configure

@@ -84,7 +84,8 @@ OPTIONS
                         dlatxdlon for fv grids (dlat and dlon are the grid cell size
                         in degrees for latitude and longitude respectively); nexnp for
                         se grids.
-     -ionosphere        Ionophere module used in WACCMX [ none | wxi | wxie ].
+     -ionosphere        Ionophere module used in WACCMX [ none | wxie ].
+     -mgrid             Magnetic grid resolution for the ionosphere [ 97x80 | 193x160 | 385x320| 769x640 ].
      -macrophys <name>  Specify the macrophysics option [rk | park | clubb_sgs].
      -max_n_rad_cnst <n> Maximum number of constituents that are either radiatively
                         active, or in any single diagnostic list for the radiation.
@@ -268,6 +269,7 @@ GetOptions(
     "h|help"                    => \$opts{'help'},
     "hgrid=s"                   => \$opts{'hgrid'},
     "ionosphere=s"              => \$opts{'ionosphere'},
+    "mgrid=s"                   => \$opts{'mgrid'},
     "lapack_libdir=s"           => \$opts{'lapack_libdir'},
     "ldflags=s"                 => \$opts{'ldflags'},
     "linker=s"                  => \$opts{'linker'},
@@ -1273,15 +1275,10 @@ my $nlev = 0;
 
 # Defaults
 if ($waccmx) {
-    if ($ionos =~ /wxi/) {
-      if ($phys_pkg eq 'cam6') {
+    if ($phys_pkg eq 'cam6') {
         $nlev = 130;
-      } else {
+    } else {
         $nlev = 126;
-      }
-    }
-    else {
-        $nlev = 81;
     }
 }
 elsif ($chem_pkg =~ /waccm_/) {
@@ -1810,6 +1807,7 @@ if ($camiop eq 'ON') { $cfg_cppdefs .= " -DBFB_CAM_SCAM_IOP"; }
 # Resolution parameters for rectangular lat/lon grids
 my $nlon = $cfg_ref->get('nlon');
 my $nlat = $cfg_ref->get('nlat');
+my $csne = $cfg_ref->get('csne');
 $cfg_cppdefs .= " -DPLON=$nlon -DPLAT=$nlat";
 
 # Parameters for multiple instances
@@ -1881,14 +1879,39 @@ if ($cfg_ref->get('analytic_ic')) {
 #WACCM-X extended thermosphere/ionosphere model
 if ($waccmx) {
     $cfg_cppdefs .= ' -DWACCMX_PHYS';
-    if (($dyn_pkg ne 'fv') and ($ionos ne 'none'))  {
-        die "ERROR:  Ionosphere is only available for FV dycore \n";
-    }
-    if ($ionos =~ /wxi/) {
-       $cfg_cppdefs .= ' -DWACCMX_IONOS';
-    }
     if ($ionos =~ /wxie/) {
-       $cfg_cppdefs .= ' -DWACCMX_EDYN_ESMF';
+        my $mgrid;
+        if (defined $opts{'mgrid'}) {
+            $mgrid = $opts{'mgrid'};
+        } elsif ($nlat > 600 or $csne > 100) {
+            $mgrid = '769x640'; # ~0.25-deg magnetic lat
+        } elsif ($nlat > 300 or $csne > 50) {
+            $mgrid = '385x320'; # ~0.5-deg magnetic lat
+        } elsif ($nlat > 150 or $csne > 20) {
+            $mgrid = '193x160'; # ~1-deg magnetic lat
+        } else {
+            $mgrid = '97x80'; # default to ~2-deg magnetic lat
+        }
+
+        my ($nmlat, $nmlon) = split /x/, $mgrid;
+        my $nmlev = 5 + log($nmlon/80)/log(2);
+        my $ngrid = ($nmlon/10)*2;
+
+        if ($print>=2) {
+            print "***************************************************** \n";
+            print " Ionosphere magnetic grid parameters:\n";
+            print "   mgrid = $mgrid \n";
+            print "   nmlat = $nmlat \n";
+            print "   nmlon = $nmlon \n";
+            print "   nmlev = $nmlev \n";
+            print "   ngrid = $ngrid \n";
+            print "***************************************************** \n";
+        }
+
+        $cfg_ref->set('mgrid',$mgrid);
+
+        $cfg_cppdefs .= " -DEDYN_NLAT=$nmlat -DEDYN_NLON=$nmlon -DEDYN_NLEV=$nmlev -DEDYN_NGRID=$ngrid";
+        $cfg_cppdefs .= ' -DWACCMX_IONOS';
     }
 }
 
@@ -2136,7 +2159,7 @@ sub write_filepath
 
     if ($waccmx) {
         print $fh "$camsrcdir/src/physics/waccmx\n";
-        if ($ionos =~ /wxi/) {
+        if ($ionos =~ /wxie/) {
             print $fh "$camsrcdir/src/ionosphere/waccmx\n";
         }
     }

bld/namelist_files/namelist_defaults_cam.xml

@@ -23,6 +23,7 @@
 <dtime dyn="fv3"   hgrid="C384"    >450</dtime>
 
 <dtime dyn="se"                    >1800</dtime>
+<dtime dyn="se"    waccmx="1"      >300 </dtime>
 <dtime dyn="se"    hgrid="ne5np4"  >1800</dtime>
 <dtime dyn="se"    hgrid="ne16np4" >1800</dtime>
 <dtime dyn="se"    hgrid="ne30np4" >1800</dtime>
@@ -131,12 +132,20 @@
 <ncdata dyn="fv"  hgrid="4x5"       nlev="81"  waccmx="1"  ic_ymd="20000101" >atm/waccm/ic/WAX3548T08CO_2003top_f2000.waccm_0017bottom_4x5_L81_c160630.nc</ncdata>
 <ncdata dyn="fv"  hgrid="10x15"     nlev="81"  waccmx="1"  ic_ymd="20000101" >atm/waccm/ic/WAX3548T08CO_2003top_f2000.waccm_0017bottom_10x15_L81_c141027.nc</ncdata>
 <ncdata dyn="fv"  hgrid="4x5"       nlev="126" waccmx="1"  ic_ymd="101" aquaplanet="1">atm/waccm/ic/waccmx_aqua_4x5_L126_c170705.nc</ncdata>
+<ncdata dyn="fv"  hgrid="1.9x2.5"   nlev="126" waccmx="1">atm/waccm/ic/f.c54137.FX2000climo.f19_f19.ZGTest.001.cam.i.0002-01-01-00000_c170817.nc</ncdata>
 <ncdata dyn="fv"  hgrid="0.9x1.25"  nlev="126" waccmx="1">atm/waccm/ic/fx2000_0.9x1.25_126lev_0002-01-01-00000_c181221.nc</ncdata>
 <ncdata dyn="fv"  hgrid="1.9x2.5"   nlev="130" waccmx="1">atm/waccm/ic/wcmx-cam6-phys_1.9x2.5_130lev_2000_c181115.nc</ncdata>
 <ncdata dyn="fv"  hgrid="0.9x1.25"  nlev="130" waccmx="1">atm/waccm/ic/wcmx-cam6-phys_0.9x1.25_130lev_2000_c190122.nc</ncdata>
 <ncdata dyn="fv"  hgrid="0.47x0.63" nlev="130" waccmx="1">atm/waccm/ic/FC6X2000_f05_spinup01.cam.i.0002-01-01-00000_c190711.nc</ncdata>
 <ncdata dyn="fv"  hgrid="4x5"       nlev="130" waccmx="1"  ic_ymd="101" aquaplanet="1">atm/waccm/ic/waccmx_mam4_aqua_4x5_L130_c180803.nc</ncdata>
 <ncdata dyn="fv"  hgrid="1.9x2.5"   nlev="130" waccmx="1"  ic_ymd="101" aquaplanet="1">atm/waccm/ic/waccmx_mam4_aqua_1.9x2.5_L130_c180803.nc</ncdata>
+<ncdata dyn="se"  hgrid="ne5np4"    nlev="126" waccmx="1"               >atm/waccm/ic/waccmx_aqua_ne5np4_126L_c210304.nc</ncdata>
+<ncdata dyn="se"  hgrid="ne16np4"   nlev="126" waccmx="1"               >atm/waccm/ic/waccmx_ne16np4_126L_c200108.nc</ncdata>
+<ncdata dyn="se"  hgrid="ne16np4"   nlev="130" waccmx="1"               >atm/waccm/ic/fx2000_phys-ionos-cpl_ne16_spinup03.cam.i.0002-01-01-00000_c201005.nc</ncdata>
+<ncdata dyn="se"  hgrid="ne16np4"   nlev="126" waccmx="1" aquaplanet="1">atm/waccm/ic/waccmx_aqua_ne16np4_126L_c191108.nc</ncdata>
+<ncdata dyn="se"  hgrid="ne16np4"   nlev="126" waccmx="1" aquaplanet="1" ionosphere="none">atm/waccm/ic/waccmx4_neutral_aquap_ne16np4_126lev_c200827.nc</ncdata>
+<ncdata dyn="se"  hgrid="ne30np4"   nlev="130" waccmx="1"               >atm/waccm/ic/fx2000_phys-ionos-cpl_ne30_spinup01.cam.i.0002-01-01-00000_c201014.nc</ncdata>
+<ncdata dyn="se"  hgrid="ne30np4" npg="3" nlev="130" waccmx="1"         >atm/waccm/ic/fx2000_phys-ionos-cpl_ne30pg3_spinup01.cam.i.0002-01-01-00000_c201014.nc</ncdata>
 
 <ncdata dyn="fv3"   hgrid="C24"  nlev="32"  aquaplanet="1"  ic_ymd="101" >atm/cam/inic/fv3/aqua_0006-01-01_C24_L32_c200625.nc</ncdata>
 <ncdata dyn="fv3"   hgrid="C48"  nlev="32"  aquaplanet="1"  ic_ymd="101" >atm/cam/inic/fv3/aqua_0006-01-01_C48_L32_c200625.nc</ncdata>
@@ -702,10 +711,26 @@
 <wei05_coefs_file>atm/waccm/efld/wei05sc_c080415.nc</wei05_coefs_file>
 <ionos_xport_nsplit waccmx="1">5</ionos_xport_nsplit>
 <ionos_xport_nsplit hgrid="0.9x1.25" waccmx="1">30</ionos_xport_nsplit>
+<ionos_xport_nsplit hgrid="ne30np4"  waccmx="1">30</ionos_xport_nsplit>
 <ionos_xport_nsplit hgrid="0.47x0.63" waccmx="1">90</ionos_xport_nsplit>
-<oplus_ring_polar_filter>.false.</oplus_ring_polar_filter>
-<oplus_ring_polar_filter hgrid="0.9x1.25" waccmx="1">.true.</oplus_ring_polar_filter>
-<oplus_ring_polar_filter hgrid="0.47x0.63" waccmx="1">.true.</oplus_ring_polar_filter>
+<oplus_ring_polar_filter>.true.</oplus_ring_polar_filter>
+
+<!-- Oplus transport grid configuration -->
+<oplus_grid> 144,96 </oplus_grid>
+<oplus_grid hgrid="ne16np4"> 144,96 </oplus_grid>
+<oplus_grid hgrid="ne30np4"> 288,192 </oplus_grid>
+<oplus_grid hgrid="0.9x1.25"> 288,192 </oplus_grid>
+<oplus_grid hgrid="0.47x0.63"> 576,384 </oplus_grid>
+
+<!-- ESMF mesh files for physics decomposition -->
+<cam_physics_mesh waccmx_opt="ionosphere" hgrid="0.47x0.63">atm/cam/coords/fv0.47x0.63_esmf_191122.nc</cam_physics_mesh>
+<cam_physics_mesh waccmx_opt="ionosphere" hgrid="0.9x1.25">atm/cam/coords/fv0.9x1.25_esmf_191121.nc</cam_physics_mesh>
+<cam_physics_mesh waccmx_opt="ionosphere" hgrid="1.9x2.5">atm/cam/coords/fv1.9x2.5_esmf_200428.nc</cam_physics_mesh>
+<cam_physics_mesh waccmx_opt="ionosphere" hgrid="4x5">atm/cam/coords/fv4x5_esmf_191122.nc</cam_physics_mesh>
+<cam_physics_mesh waccmx_opt="ionosphere" hgrid="ne5np4">atm/cam/coords/ne5np4_esmf_20191204.nc</cam_physics_mesh>
+<cam_physics_mesh waccmx_opt="ionosphere" hgrid="ne30np4">atm/cam/coords/ne30np4_esmf_191122.nc</cam_physics_mesh>
+<cam_physics_mesh waccmx_opt="ionosphere" hgrid="ne16np4">atm/cam/coords/ne16np4_esmf_191122.nc</cam_physics_mesh>
+<cam_physics_mesh waccmx_opt="ionosphere" hgrid="ne30np4" npg="3">atm/cam/coords/ne30pg3_esmf_20200428.nc</cam_physics_mesh>
 
 <!-- For scaling lightning sources of NOx -->
 <lght_no_prd_factor                                              >1.00D0</lght_no_prd_factor>
@@ -2589,6 +2614,7 @@
 <se_hypervis_subcycle  hgrid="ne0np4CONUS.ne30x8" waccm_phys="1" > 1 </se_hypervis_subcycle>
 
 <se_hypervis_subcycle_sponge                                          > 1  </se_hypervis_subcycle_sponge>
+<se_hypervis_subcycle_sponge waccmx="1"                               > 2  </se_hypervis_subcycle_sponge>
 <se_hypervis_subcycle_sponge waccm_phys="1"                           > 1  </se_hypervis_subcycle_sponge>
 <se_hypervis_subcycle_sponge hgrid="ne120np4"                         > 4  </se_hypervis_subcycle_sponge>
 <se_hypervis_subcycle_sponge se_refined_mesh="1"                      > 4  </se_hypervis_subcycle_sponge>
@@ -2606,15 +2632,18 @@
 
 <se_nu>-1</se_nu>
 <se_nu se_refined_mesh="1" hypervis_type="scalar" > 1.0e13 </se_nu>
+<se_nu waccmx="1">5.e15</se_nu>
 
 <se_nu_div> -1 </se_nu_div>
 <se_nu_div se_refined_mesh="1" hypervis_type="scalar" > 1.5625e13 </se_nu_div>
+<se_nu_div waccmx="1"> 10.e15 </se_nu_div>
 
 <se_nu_p>-1 </se_nu_p>
 <se_nu_p se_refined_mesh="1" hypervis_type="scalar" > 1.5625e13 </se_nu_p>
 
 <se_nu_top                    > 5.0e5 </se_nu_top>
 <se_nu_top se_refined_mesh="1"> 2.0e5  </se_nu_top>
+<se_nu_top waccmx="1"         > 1.0e6 </se_nu_top>
 <!-- WACCM uses molecular/thermal diffusion in sponge   -->
 <se_nu_top waccm_phys="1"     > 0.0   </se_nu_top>
 
@@ -2627,7 +2656,7 @@
 <se_rayk0     waccm_phys="1"     >  2    </se_rayk0>
 
 <se_molecular_diff               >  0.0 </se_molecular_diff>
-<se_molecular_diff waccm_phys="1">100.0 </se_molecular_diff>
+<se_molecular_diff waccm_phys="1" waccmx="0">100.0 </se_molecular_diff>
 
 <se_qsplit > 1 </se_qsplit>
 
@@ -2641,6 +2670,7 @@
 <se_nsplit hgrid="ne16np4" >  1 </se_nsplit>
 
 <se_nsplit hgrid="ne5np4"   waccm_phys="1"            >  3  </se_nsplit>
+<se_nsplit hgrid="ne16np4"  waccm_phys="1" waccmx="1" >  5  </se_nsplit>
 <se_nsplit hgrid="ne16np4"  waccm_phys="1"            >  3  </se_nsplit>
 <se_nsplit hgrid="ne30np4"  waccm_phys="1"            >  5  </se_nsplit>
 <se_nsplit hgrid="ne30np4"  waccm_phys="1"  nlev="110">  4  </se_nsplit>

bld/namelist_files/namelist_definition.xml

@@ -4333,6 +4333,15 @@ cache file to be consistent with how CAM was built.
 Default: set by build-namelist
 </entry>
 
+<entry id="cam_physics_mesh" type="char*256" input_pathname="abs"
+       category="build" group="phys_ctl_nl" valid_values="" >
+Full pathname to CAM physics grid ESMF mesh file.
+N.B. this variable may not be set by the user.
+It is set by build-namelist via information in the configure
+cache file to be consistent with how CAM was built.
+Default: set by build-namelist
+</entry>
+
 <entry id="waccmx_opt" type="char*16" category="waccmx"
        group="phys_ctl_nl" valid_values="ionosphere,neutral,off" >
 Runtime options of upper thermosphere WACCM-X.  'ionosphere' for
@@ -4367,6 +4376,19 @@ Switch to apply ring polar filter within in ionosphere O+ transport.
 Default: FALSE
 </entry>
 
+<entry id="oplus_grid" type="integer(2)"
+       category="waccmx" group="ionosphere_nl" valid_values="" >
+Oplus transport grid size, entered as num_longitudes, num_latitudes.
+Default: set by build-namelist
+</entry>
+
+<entry id="ionos_npes" type="integer"
+       category="waccmx" group="ionosphere_nl" valid_values="" >
+Number of MPI processes on which to run the WACCM-X ionosphere
+electro-dynamo and O+ ion transport modules.
+Default: 0 (use all atmosphere tasks)
+</entry>
+
 <entry id="ionos_xport_active" type="logical"  category="waccmx"
        group="ionosphere_nl" valid_values="">
 Switch to to turn on/off O+ transport in ionosphere.
@@ -4393,12 +4415,6 @@ Electric potential model used in the waccmx ionosphere.
 Default: set by build-namelist
 </entry>
 
-<entry id="ionos_epotential_amie" type="logical"  category="waccmx"
-       group="ionosphere_nl" valid_values="">
-Give the user the ability to input prescribed high-latitude electric potential.
-Default: FALSE
-</entry>
-
 <entry id="epot_crit_colats" type="real*2"  category="waccmx"
        group="ionosphere_nl" valid_values="">
 Co-latitudes (degrees) of the critical angles where the ionosphere
@@ -4414,15 +4430,33 @@ high latitude electric potential model.
 Default: set by build-namelist.
 </entry>
 
-<entry id="amienh_file" type="char*256" input_pathname="abs" category="waccmx"
+<entry id="ionos_epotential_amie" type="logical"  category="waccmx"
+       group="ionosphere_nl" valid_values="">
+Give the user the ability to input prescribed high-latitude electric potential.
+Default: FALSE
+</entry>
+
+<entry id="amienh_files" type="char*256(20)" input_pathname="abs" category="waccmx"
+       group="ionosphere_nl" valid_values="" >
+List of full pathnames of AMIE electic potential inputs for northern hemisphere.
+Default: NONE.
+</entry>
+
+<entry id="amiesh_files" type="char*256(20)" input_pathname="abs" category="waccmx"
        group="ionosphere_nl" valid_values="" >
-Full pathname of AMIE inputs for northern hemisphere.
+List of full pathnames of AMIE electic potential inputs for southern hemisphere.
 Default: NONE.
 </entry>
 
-<entry id="amiesh_file" type="char*256" input_pathname="abs" category="waccmx"
+<entry id="ionos_epotential_ltr" type="logical"  category="waccmx"
+       group="ionosphere_nl" valid_values="">
+Give the user the ability to input LTR high-latitude electric potential.
+Default: FALSE
+</entry>
+
+<entry id="ltr_files" type="char*256(20)" input_pathname="abs" category="waccmx"
        group="ionosphere_nl" valid_values="" >
-Full pathname of AMIE inputs for southern hemisphere.
+List of full pathnames of LTR electic potential inputs for both hemispheres.
 Default: NONE.
 </entry>
 

bld/namelist_files/use_cases/sd_waccmx_ma_cam6.xml

@@ -15,7 +15,7 @@
 <epp_all_varname>'epp_ion_rates'</epp_all_varname>
 
 <!--Species IC -->
-<ncdata dyn="fv"  hgrid="0.9x1.25">atm/waccm/ic/f.c61016.FC6XSD.f09_f09_mg17.test01.cam.i.2000-01-21-00000_c190325.nc</ncdata>
+<ncdata dyn="fv"  hgrid="0.9x1.25">atm/waccm/ic/f.c62004.FC6XSD.f09_f09_mg17.cam.i.1980-01-01-00000_c191211.nc</ncdata>
 <ncdata dyn="fv"  hgrid="1.9x2.5" >atm/waccm/ic/FC6XSD_f19_f19_mg17_L145_1981-01-01-00000_c190617.nc</ncdata>
 
 <met_rlx_bot>50.</met_rlx_bot>