Lfric2lfric multi cpu

applications/lfric2lfric/example/configuration.nml

@@ -73,13 +73,13 @@ coord_order             = 1,
 /
 
 &partitioning
-  mesh_type             = 'destination',
+  mesh_target           = 'destination',
   partitioner           = 'cubedsphere',
   panel_decomposition   = 'auto',
 /
 
 &partitioning
-  mesh_type             = 'source',
+  mesh_target           = 'source',
   partitioner           = 'cubedsphere',
   panel_decomposition   = 'auto',
 /

applications/lfric2lfric/rose-meta/lfric-lfric2lfric/HEAD/rose-meta.conf

@@ -278,15 +278,17 @@ type=character
 
 [namelist:partitioning]
 duplicate=true
-!instance_key_member=mesh_type
+!instance_key_member=mesh_target
 
-[namelist:partitioning=mesh_type]
+[namelist:partitioning=mesh_target]
 compulsory=true
 description=The purpose of the mesh
-!enumeration=true
-help=The mesh can be used to perform a LFRic forecast (Dynamics),
-    =or to describe the source and the destination grids in
-    =the lfric2lfric regridding program.
+help=The designation of the mesh to which this
+    =partitioning profile is to be applied.
+    =
+    =Recognised designations are:
+    =  * 'source'
+    =  * 'destination'
 ns=namelist/lfric2lfric/configuration
-value-titles=Dynamics, Source, Destination
-values='dynamics', 'source', 'destination'
+!string_length=default
+type=character

applications/lfric2lfric/rose-meta/lfric-lfric2lfric/versions.py

@@ -31,3 +31,24 @@ def upgrade(self, config, meta_config=None):
         # Add settings
         return config, self.reports
 """
+
+class vn31_t192(MacroUpgrade):
+    """Upgrade macro for ticket #192 by Juan M. Castillo."""
+
+    BEFORE_TAG = "vn3.1"
+    AFTER_TAG = "vn3.1_t192"
+
+    def upgrade(self, config, meta_config=None):
+        # Commands From: rose-meta/lfric-lfric2lfric
+        self.rename_setting(
+            config,
+            ["namelist:partitioning(source)", "mesh_type"],
+            ["namelist:partitioning(source)", "mesh_target"],
+        )
+        self.rename_setting(
+            config,
+            ["namelist:partitioning(destination)", "mesh_type"],
+            ["namelist:partitioning(destination)", "mesh_target"],
+        )
+
+        return config, self.reports

applications/lfric2lfric/source/algorithm/lfric2lfric_oasis_regrid_mod.X90

@@ -123,7 +123,7 @@ contains
                                                  field_src, l, k))
 
           ! send the layer via OASIS
-          ! Extract the version that was actaully placed in the collection
+          ! Extract the version that was actually placed in the collection
           coupling_ptr => get_coupling_from_collection(modeldb%values, "coupling" )
           local_index => coupling_ptr%get_local_index()
           call coupler_exchange_2d%initialise(src_field, local_index)
@@ -132,7 +132,7 @@ contains
           call coupler_exchange_2d%clear()
 
           ! receive the layer via OASIS
-          ! Extract the version that was actaully placed in the collection
+          ! Extract the version that was actually placed in the collection
           coupling_ptr => get_coupling_from_collection(modeldb%values, "coupling_dst" )
           local_index => coupling_ptr%get_local_index()
           call coupler_exchange_2d%initialise(dst_field, local_index)

applications/lfric2lfric/source/initialisation/lfric2lfric_init_mesh.f90

@@ -158,6 +158,17 @@ subroutine init_mesh( config, configuration,   &
     end if
   end do
 
+  if (.not. associated(src_partitioning_nml)) then
+    write( log_scratch_space, '(A)' )                                     &
+         'Source mesh partitioning namelist (partitioning:source) not found.'
+    call log_event(log_scratch_space, log_level_error)
+  end if
+  if (.not. associated(dst_partitioning_nml)) then
+    write( log_scratch_space, '(A)' )                                          &
+         'Destination mesh partitioning namelist (partitioning:destination) not found.'
+    call log_event(log_scratch_space, log_level_error)
+  end if
+
   generate_inner_halos(src) = src_partitioning_nml%generate_inner_halos()
   generate_inner_halos(dst) = dst_partitioning_nml%generate_inner_halos()
 

rose-stem/app/lfric2lfric/opt/rose-app-multi-cpu.conf

@@ -0,0 +1,4 @@
+[namelist:partitioning(source)]
+panel_decomposition='custom'
+panel_xproc=2
+panel_yproc=1

rose-stem/app/lfric2lfric/opt/rose-app-suite_controlled.conf

@@ -29,11 +29,6 @@ panel_decomposition='${PANEL_DECOMP}'
 panel_xproc=${XPROC}
 panel_yproc=${YPROC}
 
-[namelist:partitioning(source)]
-panel_decomposition='${PANEL_DECOMP}'
-panel_xproc=${XPROC}
-panel_yproc=${YPROC}
-
 [namelist:time]
 timestep_end='${RESTART_STOP}'
 timestep_start='${RESTART_START}'

rose-stem/app/lfric2lfric/rose-app.conf

@@ -835,15 +835,15 @@ wavelength=0
 
 [namelist:partitioning(destination)]
 generate_inner_halos=.false.
-mesh_type='destination'
+mesh_target='destination'
 panel_decomposition='auto'
 !!panel_xproc=0
 !!panel_yproc=0
 partitioner='cubedsphere'
 
 [namelist:partitioning(source)]
 generate_inner_halos=.false.
-mesh_type='source'
+mesh_target='source'
 panel_decomposition='auto'
 !!panel_xproc=0
 !!panel_yproc=0

rose-stem/site/common/lfric2lfric/tasks_lfric2lfric.cylc

@@ -5,7 +5,7 @@
 {# ########################################################################### #}
 {% do LOG.debug("Entered site/common/lfric2lfric/tasks_solver.cylc") %}
 
-{% if task_ns.conf_name == "ral_seuk-C32_lam_MG" %}
+{% if task_ns.conf_name == "ral_seuk-C32_lam_MG_1cpu" %}
 
     {% do task_dict.update({
         "opt_confs": ["ral_seuk"],
@@ -18,7 +18,34 @@
         "src_type": "",
     }) %}
 
-{% elif task_ns.conf_name == "oasis_ral_seuk-C32_lam_MG" %}
+{% elif task_ns.conf_name == "ral_seuk-C32_lam_MG_4cpu" %}
+
+    {% do task_dict.update({
+        "opt_confs": ["ral_seuk"],
+        "resolution": "seuk_MG",
+        "dst_mesh": "",
+        "dst_name": "",
+        "dst_type": "",
+        "src_mesh": "",
+        "src_name": "",
+        "src_type": "",
+        "mpi_parts": 4,
+    }) %}
+
+{% elif task_ns.conf_name == "oasis_ral_seuk-C32_lam_MG_1cpu" %}
+
+    {% do task_dict.update({
+        "opt_confs": ["ral_seuk","oasis"],
+        "resolution": "seuk_MG",
+        "dst_mesh": "seuk_MG",
+        "dst_name": "multigrid_l1",
+        "dst_type": "regional",
+        "src_mesh": "seuk_MG",
+        "src_name": "dynamics",
+        "src_type": "regional",
+    }) %}
+
+{% elif task_ns.conf_name == "oasis_ral_seuk-C32_lam_MG_4cpu" %}
 
     {% do task_dict.update({
         "opt_confs": ["ral_seuk","oasis"],
@@ -29,6 +56,7 @@
         "src_mesh": "seuk_MG",
         "src_name": "dynamics",
         "src_type": "regional",
+        "mpi_parts": 4,
     }) %}
 
 {% elif task_ns.conf_name == "ral3-seuk" %}
@@ -70,7 +98,20 @@
         "src_type": "",
     }) %}
 
-{% elif task_ns.conf_name == "clim_gal9-C24_C12" %}
+{% elif task_ns.conf_name == "clim_gal9-C24_C12_1cpu" %}
+
+    {% do task_dict.update({
+        "opt_confs": ["clim_gal9"],
+        "resolution": "C24_C12",
+        "dst_mesh": "",
+        "dst_name": "",
+        "dst_type": "",
+        "src_mesh": "",
+        "src_name": "",
+        "src_type": "",
+    }) %}
+
+{% elif task_ns.conf_name == "clim_gal9-C24_C12_6cpu" %}
 
     {% do task_dict.update({
         "opt_confs": ["clim_gal9"],
@@ -81,9 +122,10 @@
         "src_mesh": "",
         "src_name": "",
         "src_type": "",
+        "mpi_parts": 6,
     }) %}
 
-{% elif task_ns.conf_name == "oasis_clim_gal9-C24_C12" %}
+{% elif task_ns.conf_name == "oasis_clim_gal9-C24_C12_1cpu" %}
 
     {% do task_dict.update({
         "opt_confs": ["clim_gal9","oasis"],
@@ -96,7 +138,21 @@
         "src_type": "global",
     }) %}
 
-{% elif task_ns.conf_name == "oasis_clim_gal9_C12-ral_seuk_C16_lam" %}
+{% elif task_ns.conf_name == "oasis_clim_gal9-C24_C12_6cpu" %}
+
+    {% do task_dict.update({
+        "opt_confs": ["clim_gal9","oasis"],
+        "resolution": "C24_C12",
+        "dst_mesh": "C24_C12",
+        "dst_name": "C24",
+        "dst_type": "global",
+        "src_mesh": "C24_C12",
+        "src_name": "C12",
+        "src_type": "global",
+        "mpi_parts": 6,
+    }) %}
+
+{% elif task_ns.conf_name == "oasis_clim_gal9_C12-ral_seuk_C16_lam_1cpu" %}
 
     {% do task_dict.update({
         "opt_confs": ["clim_gal9_ral_seuk","oasis"],
@@ -109,7 +165,21 @@
         "src_type": "global",
     }) %}
 
-{% elif task_ns.conf_name == "oasis_clim_gal9_C12-ral_seuk_C16_lam-lbc" %}
+{% elif task_ns.conf_name == "oasis_clim_gal9_C12-ral_seuk_C16_lam_4cpu" %}
+
+    {% do task_dict.update({
+        "opt_confs": ["clim_gal9_ral_seuk","oasis", "multi-cpu"],
+        "resolution": "C12_C16_lam",
+        "dst_mesh": "seuk_MG",
+        "dst_name": "dynamics",
+        "dst_type": "regional",
+        "src_mesh": "C24_C12",
+        "src_name": "C12",
+        "src_type": "global",
+        "mpi_parts": 4,
+    }) %}
+
+{% elif task_ns.conf_name == "oasis_clim_gal9_C12-ral_seuk_C16_lam-lbc_1cpu" %}
 
   {# dst_name will need to change to "dynamics-lbc" when lbc meshes can be read #}
     {% do task_dict.update({
@@ -123,6 +193,21 @@
         "src_type": "global",
     }) %}
 
+{% elif task_ns.conf_name == "oasis_clim_gal9_C12-ral_seuk_C16_lam-lbc_4cpu" %}
+
+  {# dst_name will need to change to "dynamics-lbc" when lbc meshes can be read #}
+    {% do task_dict.update({
+        "opt_confs": ["clim_gal9_ral_seuk","oasis","lbc", "multi-cpu"],
+        "resolution": "C12_C16_lam",
+        "dst_mesh": "seuk_MG",
+        "dst_name": "dynamics-lbc",
+        "dst_type": "regional",
+        "src_mesh": "C24_C12",
+        "src_name": "C12",
+        "src_type": "global",
+        "mpi_parts": 4,
+    }) %}
+
 {% elif task_ns.conf_name == "canned" %}
 
     {% do task_dict.update({

rose-stem/site/meto/groups/groups_lfric2lfric.cylc

@@ -8,13 +8,19 @@
 {# Azspice Groups #}
 {% do site_groups.update({
     "lfric2lfric_azspice_developer": [
-        "lfric2lfric_ral_seuk-C32_lam_MG_azspice_gnu_fast-debug-64bit",
-        "lfric2lfric_oasis_ral_seuk-C32_lam_MG_azspice_gnu_fast-debug-64bit",
+        "lfric2lfric_ral_seuk-C32_lam_MG_1cpu_azspice_gnu_fast-debug-64bit",
+        "lfric2lfric_ral_seuk-C32_lam_MG_4cpu_azspice_gnu_fast-debug-64bit",
+        "lfric2lfric_oasis_ral_seuk-C32_lam_MG_1cpu_azspice_gnu_fast-debug-64bit",
+        "lfric2lfric_oasis_ral_seuk-C32_lam_MG_4cpu_azspice_gnu_fast-debug-64bit",
         "lfric2lfric_ral3-seuk_azspice_gnu_fast-debug-64bit",
-        "lfric2lfric_clim_gal9-C24_C12_azspice_gnu_fast-debug-64bit",
-        "lfric2lfric_oasis_clim_gal9-C24_C12_azspice_gnu_fast-debug-64bit",
-        "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam_azspice_gnu_fast-debug-64bit",
-        "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam-lbc_azspice_gnu_fast-debug-64bit",
+        "lfric2lfric_clim_gal9-C24_C12_1cpu_azspice_gnu_fast-debug-64bit",
+        "lfric2lfric_clim_gal9-C24_C12_6cpu_azspice_gnu_fast-debug-64bit",
+        "lfric2lfric_oasis_clim_gal9-C24_C12_1cpu_azspice_gnu_fast-debug-64bit",
+        "lfric2lfric_oasis_clim_gal9-C24_C12_6cpu_azspice_gnu_fast-debug-64bit",
+        "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam_1cpu_azspice_gnu_fast-debug-64bit",
+        "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam_4cpu_azspice_gnu_fast-debug-64bit",
+        "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam-lbc_1cpu_azspice_gnu_fast-debug-64bit",
+        "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam-lbc_4cpu_azspice_gnu_fast-debug-64bit",
         "lfric2lfric_azspice_canned",
     ],
     "lfric2lfric_azspice_extra": [
@@ -36,13 +42,19 @@
 {# EX1A Groups #}
 {% do site_groups.update({
     "lfric2lfric_ex1a_developer": [
-        "lfric2lfric_ral_seuk-C32_lam_MG_ex1a_cce_fast-debug-64bit",
-        "lfric2lfric_oasis_ral_seuk-C32_lam_MG_ex1a_cce_fast-debug-64bit",
+        "lfric2lfric_ral_seuk-C32_lam_MG_1cpu_ex1a_cce_fast-debug-64bit",
+        "lfric2lfric_ral_seuk-C32_lam_MG_4cpu_ex1a_cce_fast-debug-64bit",
+        "lfric2lfric_oasis_ral_seuk-C32_lam_MG_1cpu_ex1a_cce_fast-debug-64bit",
+        "lfric2lfric_oasis_ral_seuk-C32_lam_MG_4cpu_ex1a_cce_fast-debug-64bit",
         "lfric2lfric_ral3-seuk_ex1a_cce_fast-debug-64bit",
-        "lfric2lfric_clim_gal9-C24_C12_ex1a_cce_fast-debug-64bit",
-        "lfric2lfric_oasis_clim_gal9-C24_C12_ex1a_cce_fast-debug-64bit",
-        "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam_ex1a_cce_fast-debug-64bit",
-        "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam-lbc_ex1a_cce_fast-debug-64bit",
+        "lfric2lfric_clim_gal9-C24_C12_1cpu_ex1a_cce_fast-debug-64bit",
+        "lfric2lfric_clim_gal9-C24_C12_6cpu_ex1a_cce_fast-debug-64bit",
+        "lfric2lfric_oasis_clim_gal9-C24_C12_1cpu_ex1a_cce_fast-debug-64bit",
+        "lfric2lfric_oasis_clim_gal9-C24_C12_6cpu_ex1a_cce_fast-debug-64bit",
+        "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam_1cpu_ex1a_cce_fast-debug-64bit",
+        "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam_4cpu_ex1a_cce_fast-debug-64bit",
+        "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam-lbc_1cpu_ex1a_cce_fast-debug-64bit",
+        "lfric2lfric_oasis_clim_gal9_C12-ral_seuk_C16_lam-lbc_4cpu_ex1a_cce_fast-debug-64bit",
         "lfric2lfric_ex1a_canned",
     ],
     "lfric2lfric_ex1a_extra": [