Refactor/workflow

configs/esm_software/esm_runscripts/defaults.yaml

@@ -9,12 +9,61 @@ general:
 
 # Defaults to be added to each model or component
 per_model_defaults:
-        file_movements:
-                default:
-                        all_directions: copy
-                bin:
-                        init_to_exp:    copy
-                        exp_to_run:     copy
-                        run_to_work:    copy
-                        work_to_run:    copy
+    file_movements:
+        default:
+            all_directions: copy
+        bin:
+            init_to_exp:    copy
+            exp_to_run:     copy
+            run_to_work:    copy
+            work_to_run:    copy
 
+workflow:
+    user_phases: None
+    default_cluster: sim_cluster
+    clusters:
+        sim_cluster:
+            first_task_in_queue: prepcompute
+            last_task_in_queue: tidy
+            next_run_triggered_by: tidy
+            phases:
+                - prepcompute
+                - compute
+                - tidy
+
+    phases:
+        prepcompute:
+            phase_type: SimulationSetup
+            called_from: tidy
+            cluster: sim_cluster
+            name: prepcompute
+            next_submit:
+                - compute
+            nproc: 1
+            order_in_cluster: sequential
+            run_after: tidy
+            run_before: compute
+        compute:
+            phase_type: compute
+            called_from: prepcompute
+            cluster: sim_cluster
+            name: compute
+            next_submit:
+                - tidy
+            #nproc: None
+            nproc: 1
+            order_in_cluster: sequential
+            run_after: prepcompute
+            run_before: tidy
+            #run_on_queue: ${computer.partitions.pp.name}
+        tidy:
+            phase_type: SimulationSetup
+            called_from: compute
+            cluster: sim_cluster
+            name: tidy
+            next_submit:
+                - prepcompute
+            nproc: 1
+            order_in_cluster: sequential
+            run_after: compute
+            run_before: prepcompute

configs/esm_software/esm_runscripts/esm_plugins.yaml

@@ -1,6 +1,6 @@
 # Mappings of functions/methods to parent ESM python libraries:
 # tells ESM-Tools in which library and file/sublibrary can find the functions
-# of the recipies
+# of the recipes
 # (prescribed in configs/esm_software/esm_runscripts/esm_runscripts.yaml).
 #
 # Core (not an external plugin)
@@ -25,8 +25,8 @@ core:
                         - "initialize_batch_system"
                         - "initialize_coupler"
                         - "set_logfile"
-                        #- "add_vcs_info"
-                        #- "check_vcs_info_against_last_run"
+                        - "add_vcs_info"
+                        - "check_vcs_info_against_last_run"
                         - "check_config_for_warnings_errors"
 
                 prepexp:
@@ -111,3 +111,6 @@ core:
                 workflow:
                         - "assemble_workflow"
 
+                postprocess:
+                        - "convert_to_zarr"
+

configs/esm_software/esm_runscripts/esm_runscripts.yaml

@@ -1,4 +1,4 @@
-# Default recipies
+# Default recipes
 # ESM-Tools uses config/esm_software/esm_runscripts/esm_plugins.yaml to understand
 # where to look for each of this function/methods (the steps within each recipy, e.g.
 # "_read_date_file").
@@ -49,7 +49,7 @@ choose_job_type:
                         - "copy_tools_to_thisrun"
                         # The next step will call esm_runscripts again from the experiment folder,
                         # if the current folder is not the experiment folder already.
-                        # If esm_runscripts will be excuted, the following step will be skipped, since
+                        # If esm_runscripts will be executed, the following step will be skipped, since
                         # there is a sys.exit() after the esm_runscripts call.
                         - "call_esm_runscripts_from_prepexp"
                         # The following step will be skipped, if not in experiment folder.

runscripts/fesom2/fesom2.6-albedo-reference.yaml

@@ -0,0 +1,25 @@
+general:
+    user: !ENV ${USER}
+    account: "computing.computing"
+    setup_name: fesom
+    compute_time: "08:00:00"
+    initial_date: "1958-01-01"
+    final_date: "1958-03-31"
+    project_base: !ENV ${PROJECT_BASE}
+    base_dir: "${general.project_base}/experiments"
+    nyear: 0
+    nmonth: 1
+    use_venv: False
+
+fesom:
+    version: 2.6
+    model_dir: "${general.project_base}/model-codes/fesom-2.6/"
+    mesh_dir: /albedo/pool/FESOM2/core2/
+    restart_rate: 1
+    restart_first: 1
+    restart_unit: "m"
+    resolution: "CORE2"
+    lresume: false
+    time_step: 1800
+    # FIXME(@pgierz, @hpc-team): We have FESOM and FESOM2 in the albedo pool, this should be unified...
+    forcing_data_dir: "/albedo/pool/FESOM/forcing/"

src/esm_plugin_manager/esm_plugin_manager.py

@@ -8,12 +8,11 @@
 
 import esm_parser
 import esm_profile
-from esm_parser import yaml_file_to_dict
 
 
 def read_recipe(recipe, additional_dict, needs_parse=True):
     if needs_parse:
-        recipe = yaml_file_to_dict(recipe)
+        recipe = esm_parser.yaml_file_to_dict(recipe)
     recipe.update(additional_dict)
     esm_parser.basic_choose_blocks(recipe, recipe)
     esm_parser.recursive_run_function(
@@ -26,7 +25,7 @@ def read_recipe(recipe, additional_dict, needs_parse=True):
 def read_plugin_information(plugins_bare, recipe, needs_parse=True):
     # pluginfile = esm_plugins.yaml
     if needs_parse:
-        plugins_bare = yaml_file_to_dict(plugins_bare)
+        plugins_bare = esm_parser.yaml_file_to_dict(plugins_bare)
     extra_info = ["location", "git-url"]
     plugins = {}
     for workitem in recipe["recipe"]:
@@ -62,6 +61,7 @@ def read_plugin_information(plugins_bare, recipe, needs_parse=True):
             if found:
                 break
 
+    breakpoint()
     attach_installed_plugins_to_all(plugins)
     return plugins
 
@@ -127,17 +127,31 @@ def check_plugin_availability(plugins):
 
 
 def work_through_recipe(recipe, plugins, config):
+    """
+    Works through the esm_runscripts recipes and plugin recipes.
+
+    Parameters
+    ----------
+        recipe : dict            # What is in these two dictionaries? Where do the entries are comming from?
+        plugins : dict
+        config : dict
+
+    Returns
+    -------
+        config : dict
+    """
     if config.get("general", {}).get("debug_recipe", False):
         import pdb
 
         pdb.set_trace()
     recipes = recipe["recipe"]
     recipe_name = recipe["job_type"]
+    # Loop over the recipe
     for index, workitem in enumerate(recipes, start=1):
         if config["general"].get("verbose", False):
             # diagnostic message of which recipe step is being executed
             message = (
-                f"::: Executing the step:  {workitem}    "
+                f"::: START Executing the step:  {workitem}    "
                 f"(step [{index}/{len(recipes)}] of the job:  "
                 f'{recipe["job_type"]})'
             )
@@ -187,6 +201,19 @@ def work_through_recipe(recipe, plugins, config):
                     config = timed_workitem_callable(config)
                 else:
                     config = getattr(thismodule, workitem)(config)
+                config = getattr(thismodule, workitem)(config)
+        if config["general"].get("verbose", False):
+            # diagnostic message of which recipe step is being executed
+            message = (
+                f"::: END Executing the step:  {workitem}    "
+                f"(step [{index}/{len(recipes)}] of the job:  "
+                f'{recipe["job_type"]})'
+            )
+
+            logger.info()
+            logger.info("=" * len(message))
+            logger.info(message)
+            logger.info("=" * len(message))
     return config