Sprint/refactor workflow

configs/esm_software/esm_runscripts/defaults.yaml

@@ -9,12 +9,61 @@ general:
 
 # Defaults to be added to each model or component
 per_model_defaults:
-        file_movements:
-                default:
-                        all_directions: copy
-                bin:
-                        init_to_exp:    copy
-                        exp_to_run:     copy
-                        run_to_work:    copy
-                        work_to_run:    copy
+    file_movements:
+        default:
+            all_directions: copy
+        bin:
+            init_to_exp:    copy
+            exp_to_run:     copy
+            run_to_work:    copy
+            work_to_run:    copy
 
+workflow:
+    user_phases: None
+    default_cluster: sim_cluster
+    clusters:
+        sim_cluster:
+            first_task_in_queue: prepcompute
+            last_task_in_queue: tidy
+            next_run_triggered_by: tidy
+            phases:
+                - prepcompute
+                - compute
+                - tidy
+
+    phases:
+        prepcompute:
+            phase_type: SimulationSetup
+            called_from: tidy
+            cluster: sim_cluster
+            name: prepcompute
+            next_submit:
+                - compute
+            nproc: 1
+            order_in_cluster: sequential
+            run_after: tidy
+            run_before: compute
+        compute:
+            phase_type: compute
+            called_from: prepcompute
+            cluster: sim_cluster
+            name: compute
+            next_submit:
+                - tidy
+            #nproc: None
+            nproc: 1
+            order_in_cluster: sequential
+            run_after: prepcompute
+            run_before: tidy
+            #run_on_queue: ${computer.partitions.pp.name}
+        tidy:
+            phase_type: SimulationSetup
+            called_from: compute
+            cluster: sim_cluster
+            name: tidy
+            next_submit:
+                - prepcompute
+            nproc: 1
+            order_in_cluster: sequential
+            run_after: compute
+            run_before: prepcompute

src/esm_plugin_manager/esm_plugin_manager.py

@@ -123,16 +123,30 @@ def check_plugin_availability(plugins):
 
 
 def work_through_recipe(recipe, plugins, config):
+    """
+    Works through the esm_runscripts recipes and plugin recipes.
+
+    Parameters
+    ----------
+        recipe : dict            # What is in these two dictionaries? Where do the entries are comming from?
+        plugins : dict
+        config : dict
+
+    Returns
+    -------
+        config : dict
+    """
     if config.get("general", {}).get("debug_recipe", False):
         import pdb
 
         pdb.set_trace()
     recipes = recipe["recipe"]
+    # Loop over the recipe
     for index, workitem in enumerate(recipes, start=1):
         if config["general"].get("verbose", False):
             # diagnostic message of which recipe step is being executed
             message = (
-                f"::: Executing the step:  {workitem}    "
+                f"::: START Executing the step:  {workitem}    "
                 f"(step [{index}/{len(recipes)}] of the job:  "
                 f'{recipe["job_type"]})'
             )
@@ -162,6 +176,18 @@ def work_through_recipe(recipe, plugins, config):
                 thismodule = importlib.util.module_from_spec(spec)
                 spec.loader.exec_module(thismodule)
                 config = getattr(thismodule, workitem)(config)
+        if config["general"].get("verbose", False):
+            # diagnostic message of which recipe step is being executed
+            message = (
+                f"::: END Executing the step:  {workitem}    "
+                f"(step [{index}/{len(recipes)}] of the job:  "
+                f'{recipe["job_type"]})'
+            )
+
+            print()
+            print("=" * len(message))
+            print(message)
+            print("=" * len(message))
     return config