diff --git a/configs/esm_software/esm_runscripts/esm_plugins.yaml b/configs/esm_software/esm_runscripts/esm_plugins.yaml index 47158d88e..283276307 100644 --- a/configs/esm_software/esm_runscripts/esm_plugins.yaml +++ b/configs/esm_software/esm_runscripts/esm_plugins.yaml @@ -25,8 +25,8 @@ core: - "initialize_batch_system" - "initialize_coupler" - "set_logfile" - #- "add_vcs_info" - #- "check_vcs_info_against_last_run" + - "add_vcs_info" + - "check_vcs_info_against_last_run" - "check_config_for_warnings_errors" prepexp: @@ -41,7 +41,6 @@ core: - "compile_model" - "_write_finalized_config" - "_show_simulation_info" - - "assemble_filelists" - "wait_for_iterative_coupling" - "copy_files_to_thisrun" - "modify_namelists" @@ -49,7 +48,6 @@ core: - "create_new_files" - "create_empty_folders" - "prepare_coupler_files" - - "add_batch_hostfile" - "copy_files_to_work" tidy: @@ -57,13 +55,8 @@ core: - "copy_stuff_back_from_work" - "copy_all_results_to_exp" - "clean_run_dir" - - "start_post_job" - - "_increment_date_and_run_number" - - "_write_date_file" - - "signal_tidy_completion" - "throw_away_some_infiles" - observe: - "init_monitor_file" - "wait_and_observe" @@ -102,6 +95,8 @@ core: resubmit: - "submit" + - "_increment_date_and_run_number" + - "_write_date_file" chunky_parts: - "_update_run_in_chunk" @@ -111,3 +106,5 @@ core: workflow: - "assemble_workflow" + postprocess: + - "convert_to_zarr" diff --git a/setup.py b/setup.py index d714ae03c..f16d29047 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,15 @@ #!/usr/bin/env python - """The setup script.""" +import sys + +try: + import yaml +except ImportError: + print("PyYAML is not installed. Installing it now...", file=sys.stderr) + import subprocess -from os import getenv + subprocess.check_call([sys.executable, "-m", "pip", "install", "PyYAML"]) + import yaml # Try again after installation from setuptools import find_packages, setup @@ -41,12 +48,21 @@ "h5netcdf>=0.8.1", ] -setup_requirements = [] +setup_requirements = ["pyyaml==6.0.1"] test_requirements = [ "pyfakefs==4.6.0", ] +with open("configs/esm_software/esm_runscripts/esm_plugins.yaml", "r") as f: + plugin_name_path_list = [] + core_plugins = yaml.safe_load(f) + for package in core_plugins["core"]: + for module in core_plugins["core"][package]: + for func in core_plugins["core"][package][module]: + plugin_name_path_list.append(f"{func}={package}.{module}:{func}") + + setup( author="The ESM Tools Team", author_email=[ @@ -85,6 +101,7 @@ "esm_tools=esm_tools.cli:main", "esm_utilities=esm_utilities.cli:main", ], + "esm_tools.core_plugins": plugin_name_path_list, }, install_requires=requirements, license="GNU General Public License v2", diff --git a/src/esm_plugin_manager/cli.py b/src/esm_plugin_manager/cli.py index eaf0400b3..0e4d58f1f 100644 --- a/src/esm_plugin_manager/cli.py +++ b/src/esm_plugin_manager/cli.py @@ -3,7 +3,7 @@ Example:: - $ esm_plugins + $ esm_plugins ls external Plugin 1 -------- @@ -14,17 +14,33 @@ Docstring of plugin 2 """ +import sys + +import click + from esm_plugin_manager import find_installed_plugins +@click.group def main(): """The main plugin CLI""" - discovered_plugins = find_installed_plugins() - print("The following plugins are installed and available:") - for plugin_name in discovered_plugins: - plugin_code = discovered_plugins[plugin_name]["callable"] - print(plugin_name) - doc = plugin_code.__doc__ - if doc: - print("-" * len(plugin_name)) - print(doc) + + +@main.command() +@click.argument("category") +def ls(category): + """List plugins installed""" + if category in ["core", "external"]: + discovered_plugins = find_installed_plugins(category=category) + print(f"The following plugins are installed and available ({category}):") + for plugin_name in discovered_plugins: + plugin_code = discovered_plugins[plugin_name]["callable"] + print(plugin_name) + doc = plugin_code.__doc__ + if doc: + print("-" * len(plugin_name)) + print(doc) + sys.exit(0) + else: + print("Please use esm_plugins ls [external, core]!") + sys.exit(1) diff --git a/src/esm_plugin_manager/esm_plugin_manager.py b/src/esm_plugin_manager/esm_plugin_manager.py index 02d18ecaf..5b0e0bdfe 100644 --- a/src/esm_plugin_manager/esm_plugin_manager.py +++ b/src/esm_plugin_manager/esm_plugin_manager.py @@ -4,6 +4,7 @@ import subprocess import sys +import pkg_resources from loguru import logger import esm_parser @@ -24,64 +25,34 @@ def read_recipe(recipe, additional_dict, needs_parse=True): def read_plugin_information(plugins_bare, recipe, needs_parse=True): - # pluginfile = esm_plugins.yaml - if needs_parse: - plugins_bare = yaml_file_to_dict(plugins_bare) - extra_info = ["location", "git-url"] - plugins = {} - for workitem in recipe["recipe"]: - found = False - for module_type in ["core", "plugins"]: - if module_type in plugins_bare: - for module in plugins_bare[module_type]: - for submodule in plugins_bare[module_type][module]: - if submodule in extra_info: - continue - functionlist = plugins_bare[module_type][module][submodule] - if workitem in functionlist: - plugins[workitem] = { - "module": module, - "submodule": submodule, - "type": module_type, - } - for extra in extra_info: - if extra in plugins_bare[module_type][module]: - plugins[workitem].update( - { - extra: plugins_bare[module_type][module][ - extra - ] - } - ) - found = True - break - if found: - break - if found: - break - if found: - break - - attach_installed_plugins_to_all(plugins) + plugins = find_installed_plugins(category="core") + plugins.update(find_installed_plugins(category="external")) return plugins -def find_installed_plugins(): - import pkg_resources - - discovered_plugins = { - entry_point.name: { - "plugin_name": entry_point.module_name.split(".")[0], - "callable": entry_point.load(), - "type": "installed", +def find_installed_plugins(category="external"): + if category == "external": + discovered_plugins = { + entry_point.name: { + "plugin_name": entry_point.module_name.split(".")[0], + "callable": entry_point.load(), + "type": "installed", + } + for entry_point in pkg_resources.iter_entry_points("esm_tools.plugins") } - for entry_point in pkg_resources.iter_entry_points("esm_tools.plugins") - } - return discovered_plugins - - -def attach_installed_plugins_to_all(plugins): - plugins.update(find_installed_plugins()) + return discovered_plugins + if category == "core": + discovered_plugins = { + entry_point.name: { + "module": entry_point.module_name.split(".")[0], + "submodule": entry_point.module_name.split(".")[1], + "callable": entry_point.load(), + "type": "core", + } + for entry_point in pkg_resources.iter_entry_points("esm_tools.core_plugins") + } + return discovered_plugins + raise ValueError("Please provide a category of 'external' or 'core'") def check_plugin_availability(plugins): @@ -123,7 +94,7 @@ def check_plugin_availability(plugins): ) something_missing = True if something_missing: - sys.exit(-1) + sys.exit(1) def work_through_recipe(recipe, plugins, config): @@ -146,26 +117,15 @@ def work_through_recipe(recipe, plugins, config): logger.info("=" * len(message)) logger.info(message) logger.info("=" * len(message)) - if plugins[workitem]["type"] == "core": - thismodule = __import__(plugins[workitem]["module"]) - submodule = getattr(thismodule, plugins[workitem]["submodule"]) - if config["general"].get("profile", False): - workitem_callable = getattr(submodule, workitem) - timed_workitem_callable = esm_profile.timing( - workitem_callable, recipe_name - ) - config = timed_workitem_callable(config) - else: - config = getattr(submodule, workitem)(config) - elif plugins[workitem]["type"] == "installed": + if plugins[workitem]["type"] in ["core", "installed"]: + workitem_callable = plugins[workitem]["callable"] if config["general"].get("profile", False): - workitem_callable = plugins[workitem]["callable"] timed_workitem_callable = esm_profile.timing( workitem_callable, recipe_name ) config = timed_workitem_callable(config) else: - config = plugins[workitem]["callable"](config) + config = workitem_callable(config) else: if sys.version_info >= (3, 5): import importlib.util