wip: removes nested workflow plugin finder

configs/esm_software/esm_runscripts/esm_plugins.yaml

@@ -25,8 +25,8 @@ core:
                         - "initialize_batch_system"
                         - "initialize_coupler"
                         - "set_logfile"
-                        #- "add_vcs_info"
-                        #- "check_vcs_info_against_last_run"
+                        - "add_vcs_info"
+                        - "check_vcs_info_against_last_run"
                         - "check_config_for_warnings_errors"
 
                 prepexp:
@@ -41,29 +41,22 @@ core:
                         - "compile_model"
                         - "_write_finalized_config"
                         - "_show_simulation_info"
-                        - "assemble_filelists"
                         - "wait_for_iterative_coupling"
                         - "copy_files_to_thisrun"
                         - "modify_namelists"
                         - "modify_files"
                         - "create_new_files"
                         - "create_empty_folders"
                         - "prepare_coupler_files"
-                        - "add_batch_hostfile"
                         - "copy_files_to_work"
 
                 tidy:
                         - "tidy_coupler"
                         - "copy_stuff_back_from_work"
                         - "copy_all_results_to_exp"
                         - "clean_run_dir"
-                        - "start_post_job"
-                        - "_increment_date_and_run_number"
-                        - "_write_date_file"
-                        - "signal_tidy_completion"
                         - "throw_away_some_infiles"
 
-
                 observe:
                         - "init_monitor_file"
                         - "wait_and_observe"
@@ -102,6 +95,8 @@ core:
 
                 resubmit:
                         - "submit"
+                        - "_increment_date_and_run_number"
+                        - "_write_date_file"
 
                 chunky_parts:
                         - "_update_run_in_chunk"
@@ -111,3 +106,5 @@ core:
                 workflow:
                         - "assemble_workflow"
 
+                postprocess:
+                        - "convert_to_zarr"

setup.py

@@ -1,8 +1,15 @@
 #!/usr/bin/env python
-
 """The setup script."""
+import sys
+
+try:
+    import yaml
+except ImportError:
+    print("PyYAML is not installed. Installing it now...", file=sys.stderr)
+    import subprocess
 
-from os import getenv
+    subprocess.check_call([sys.executable, "-m", "pip", "install", "PyYAML"])
+    import yaml  # Try again after installation
 
 from setuptools import find_packages, setup
 
@@ -41,12 +48,21 @@
     "h5netcdf>=0.8.1",
 ]
 
-setup_requirements = []
+setup_requirements = ["pyyaml==6.0.1"]
 
 test_requirements = [
     "pyfakefs==4.6.0",
 ]
 
+with open("configs/esm_software/esm_runscripts/esm_plugins.yaml", "r") as f:
+    plugin_name_path_list = []
+    core_plugins = yaml.safe_load(f)
+    for package in core_plugins["core"]:
+        for module in core_plugins["core"][package]:
+            for func in core_plugins["core"][package][module]:
+                plugin_name_path_list.append(f"{func}={package}.{module}:{func}")
+
+
 setup(
     author="The ESM Tools Team",
     author_email=[
@@ -85,6 +101,7 @@
             "esm_tools=esm_tools.cli:main",
             "esm_utilities=esm_utilities.cli:main",
         ],
+        "esm_tools.core_plugins": plugin_name_path_list,
     },
     install_requires=requirements,
     license="GNU General Public License v2",

src/esm_plugin_manager/cli.py

@@ -3,7 +3,7 @@
 
 Example::
 
-    $ esm_plugins
+    $ esm_plugins ls external
 
     Plugin 1
     --------
@@ -14,17 +14,33 @@
     Docstring of plugin 2
 
 """
+import sys
+
+import click
+
 from esm_plugin_manager import find_installed_plugins
 
 
+@click.group
 def main():
     """The main plugin CLI"""
-    discovered_plugins = find_installed_plugins()
-    print("The following plugins are installed and available:")
-    for plugin_name in discovered_plugins:
-        plugin_code = discovered_plugins[plugin_name]["callable"]
-        print(plugin_name)
-        doc = plugin_code.__doc__
-        if doc:
-            print("-" * len(plugin_name))
-            print(doc)
+
+
+@main.command()
+@click.argument("category")
+def ls(category):
+    """List plugins installed"""
+    if category in ["core", "external"]:
+        discovered_plugins = find_installed_plugins(category=category)
+        print(f"The following plugins are installed and available ({category}):")
+        for plugin_name in discovered_plugins:
+            plugin_code = discovered_plugins[plugin_name]["callable"]
+            print(plugin_name)
+            doc = plugin_code.__doc__
+            if doc:
+                print("-" * len(plugin_name))
+                print(doc)
+        sys.exit(0)
+    else:
+        print("Please use esm_plugins ls [external, core]!")
+        sys.exit(1)

src/esm_plugin_manager/esm_plugin_manager.py

@@ -4,6 +4,7 @@
 import subprocess
 import sys
 
+import pkg_resources
 from loguru import logger
 
 import esm_parser
@@ -24,64 +25,34 @@ def read_recipe(recipe, additional_dict, needs_parse=True):
 
 
 def read_plugin_information(plugins_bare, recipe, needs_parse=True):
-    # pluginfile = esm_plugins.yaml
-    if needs_parse:
-        plugins_bare = yaml_file_to_dict(plugins_bare)
-    extra_info = ["location", "git-url"]
-    plugins = {}
-    for workitem in recipe["recipe"]:
-        found = False
-        for module_type in ["core", "plugins"]:
-            if module_type in plugins_bare:
-                for module in plugins_bare[module_type]:
-                    for submodule in plugins_bare[module_type][module]:
-                        if submodule in extra_info:
-                            continue
-                        functionlist = plugins_bare[module_type][module][submodule]
-                        if workitem in functionlist:
-                            plugins[workitem] = {
-                                "module": module,
-                                "submodule": submodule,
-                                "type": module_type,
-                            }
-                            for extra in extra_info:
-                                if extra in plugins_bare[module_type][module]:
-                                    plugins[workitem].update(
-                                        {
-                                            extra: plugins_bare[module_type][module][
-                                                extra
-                                            ]
-                                        }
-                                    )
-                            found = True
-                            break
-                    if found:
-                        break
-                if found:
-                    break
-            if found:
-                break
-
-    attach_installed_plugins_to_all(plugins)
+    plugins = find_installed_plugins(category="core")
+    plugins.update(find_installed_plugins(category="external"))
     return plugins
 
 
-def find_installed_plugins():
-    import pkg_resources
-
-    discovered_plugins = {
-        entry_point.name: {
-            "plugin_name": entry_point.module_name.split(".")[0],
-            "callable": entry_point.load(),
-            "type": "installed",
+def find_installed_plugins(category="external"):
+    if category == "external":
+        discovered_plugins = {
+            entry_point.name: {
+                "plugin_name": entry_point.module_name.split(".")[0],
+                "callable": entry_point.load(),
+                "type": "installed",
+            }
+            for entry_point in pkg_resources.iter_entry_points("esm_tools.plugins")
         }
-        for entry_point in pkg_resources.iter_entry_points("esm_tools.plugins")
-    }
-    return discovered_plugins
-
-
-def attach_installed_plugins_to_all(plugins):
-    plugins.update(find_installed_plugins())
+        return discovered_plugins
+    if category == "core":
+        discovered_plugins = {
+            entry_point.name: {
+                "module": entry_point.module_name.split(".")[0],
+                "submodule": entry_point.module_name.split(".")[1],
+                "callable": entry_point.load(),
+                "type": "core",
+            }
+            for entry_point in pkg_resources.iter_entry_points("esm_tools.core_plugins")
+        }
+        return discovered_plugins
+    raise ValueError("Please provide a category of 'external' or 'core'")
 
 
 def check_plugin_availability(plugins):
@@ -123,7 +94,7 @@ def check_plugin_availability(plugins):
                 )
                 something_missing = True
     if something_missing:
-        sys.exit(-1)
+        sys.exit(1)
 
 
 def work_through_recipe(recipe, plugins, config):
@@ -146,26 +117,15 @@ def work_through_recipe(recipe, plugins, config):
             logger.info("=" * len(message))
             logger.info(message)
             logger.info("=" * len(message))
-        if plugins[workitem]["type"] == "core":
-            thismodule = __import__(plugins[workitem]["module"])
-            submodule = getattr(thismodule, plugins[workitem]["submodule"])
-            if config["general"].get("profile", False):
-                workitem_callable = getattr(submodule, workitem)
-                timed_workitem_callable = esm_profile.timing(
-                    workitem_callable, recipe_name
-                )
-                config = timed_workitem_callable(config)
-            else:
-                config = getattr(submodule, workitem)(config)
-        elif plugins[workitem]["type"] == "installed":
+        if plugins[workitem]["type"] in ["core", "installed"]:
+            workitem_callable = plugins[workitem]["callable"]
             if config["general"].get("profile", False):
-                workitem_callable = plugins[workitem]["callable"]
                 timed_workitem_callable = esm_profile.timing(
                     workitem_callable, recipe_name
                 )
                 config = timed_workitem_callable(config)
             else:
-                config = plugins[workitem]["callable"](config)
+                config = workitem_callable(config)
         else:
             if sys.version_info >= (3, 5):
                 import importlib.util