diff --git a/wellies/__init__.py b/wellies/__init__.py index 53ecc1a..2119818 100644 --- a/wellies/__init__.py +++ b/wellies/__init__.py @@ -46,6 +46,7 @@ def wrapper(*args, **kwargs): from .data import StaticDataStore from .deployment import deploy_suite from .hosts import EcflowServer +from .hosts import DeployHost from .hosts import get_host from .log_archiving import ArchivedRepeatFamily from .tasks import EcfResourcesTask diff --git a/wellies/deployment.py b/wellies/deployment.py index 4c6ccd0..e01b9b8 100644 --- a/wellies/deployment.py +++ b/wellies/deployment.py @@ -215,3 +215,5 @@ def deploy_suite( logger.info(f"Definition file: {target_repo}/{name}.def") else: logger.info("No deploy option activated. Deployment aborted") + + return f"{target_repo}/{name}.def" \ No newline at end of file diff --git a/wellies/hosts.py b/wellies/hosts.py index 20b4dca..f97853e 100644 --- a/wellies/hosts.py +++ b/wellies/hosts.py @@ -16,9 +16,17 @@ class EcflowServer: hostname: str user: str deploy_dir: str + port: str = "3141" group: str = None # Optional suites group for the ecflow server +@dataclass +class DeployHost: + hostname: str + user: str + port: str = "22" + + def get_host( hostname: str, user: str, diff --git a/wellies/quickstart.py b/wellies/quickstart.py index a2b6ce6..50b7ed6 100755 --- a/wellies/quickstart.py +++ b/wellies/quickstart.py @@ -23,7 +23,7 @@ "user": "{USER}", "author": pw_user.pw_gecos, "output_root": "{HOME}/output", - "deploy_root": "{HOME}/pyflow", + "deploy_root": "{HOME}/pyflow" } @@ -102,6 +102,29 @@ def write_file(fpath: str, content: str) -> None: path.join(root_path, "profiles.yaml"), renderer.render("profiles.yaml_t", options), ) + write_file( + path.join(root_path, "pyproject.toml"), + renderer.render("pyproject.toml_t", options), + ) + write_file( + path.join(root_path, ".gitignore"), + renderer.render(".gitignore_t", options), + ) + + #create launch.json for VSCode + vscode_dir = path.join(root_path, ".vscode") + os.makedirs(vscode_dir, exist_ok=True) + options['lib_dir'] = options.get('lib_dir', f"{out_root.replace('{name}', project).replace('{HOME}', os.environ['HOME']).replace('{PERM}', os.environ['PERM']).replace('{HPCPERM}', os.environ['HPCPERM'])}/local") + options['src_dir'] = options.get('src_dir', f"{out_root.replace('{name}', project).replace('{HOME}', '${env:HOME}').replace('{PERM}', '${env:PERM}').replace('{HPCPERM}', '${env:HPCPERM}')}/src") + options['suite_env'] = options.get('suite_env', 'suite_env') + write_file( + path.join(vscode_dir, "launch.json"), + renderer.render("launch.json_t", options), + ) + write_file( + path.join(vscode_dir, "tasks.json"), + renderer.render("tasks.json_t", options), + ) # create suite folder containing config.py and nodes.py suite_dir = path.join(root_path, project) @@ -116,6 +139,38 @@ def write_file(fpath: str, content: str) -> None: ) write_file(path.join(suite_dir, "__init__.py"), "") # empty __init__.py + # create snippets folder containing ecf stubs + snippets_dir = path.join(root_path, "snippets") + os.makedirs(snippets_dir, exist_ok=True) + write_file( + path.join(snippets_dir, "dummy"), + renderer.render("dummy_t", options), + ) + write_file( + path.join(snippets_dir, "clean_init"), + renderer.render("clean_init_t", options), + ) + write_file( + path.join(snippets_dir, "self_destruct"), + renderer.render("self_destruct_t", options), + ) + + # create src folder containing source files called in ecf scritps + src_dir = path.join(root_path, "src") + os.makedirs(src_dir, exist_ok=True) + write_file( + path.join(src_dir, "dummy.py"), + renderer.render("dummy.py_t", options), + ) + + # create manuals folder containing the suite manual + manuals_dir = path.join(root_path, "manuals") + os.makedirs(manuals_dir, exist_ok=True) + write_file( + path.join(manuals_dir, "generic.man_t"), + renderer.render("generic.man_t_t", options), + ) + # create config folder containing yaml files config_dir = path.join(root_path, "configs") os.makedirs(config_dir, exist_ok=True) @@ -135,6 +190,10 @@ def write_file(fpath: str, content: str) -> None: path.join(config_dir, "data.yaml"), renderer.render("data.yaml_t", options), ) + write_file( + path.join(config_dir, "src.yaml"), + renderer.render("src.yaml_t", options), + ) # write test file test_dir = path.join(root_path, "tests") @@ -271,6 +330,7 @@ def main(argv: List[str] = sys.argv[1:]) -> int: r"\W", "_", options["project"] ) # can't have special characters in python module names options["profiles"] = '"profiles.yaml"' # default profiles file + try: if "interactive" in options: ask_user(options) diff --git a/wellies/templates/.gitignore_t b/wellies/templates/.gitignore_t new file mode 100644 index 0000000..0c464e7 --- /dev/null +++ b/wellies/templates/.gitignore_t @@ -0,0 +1,210 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock +#poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +#pdm.lock +#pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +#pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Cursor +# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to +# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data +# refer to https://docs.cursor.com/context/ignore-files +.cursorignore +.cursorindexingignore + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# generated man pages +*.man \ No newline at end of file diff --git a/wellies/templates/build.sh_t b/wellies/templates/build.sh_t index 8618bb8..9228e34 100755 --- a/wellies/templates/build.sh_t +++ b/wellies/templates/build.sh_t @@ -1,36 +1,78 @@ #!/bin/bash +set -TEexuva -o pipefail +vx + # Creates a local virtual environment -# setup() { -# python3 -m venv .venv --system-site-packages -# python3 -m pip install . -# } +setup() { + module load uv/new > /dev/null 2>&1 + uv sync +} + +# Extracts hostname, port, and suite name from configs/user.yaml +extract_config_values() { + local config_file="configs/user.yaml" + # Use yq to extract values safely, install it if needed + SUITE_NAME=$(.venv/bin/python -m yq -r '.name' "${config_file}") + DEPLOY_DIR=$(.venv/bin/python -m yq -r '.deploy_host.deploy_dir' "${config_file}" | sed -e "s#{HOME}#${HOME}#g;s#{PERM}#${PERM}#g;s#{name}#${SUITE_NAME}#g") + HOSTNAME=$(.venv/bin/python -m yq -r '.ecflow_server.hostname' "${config_file}" | sed -e "s#{USER}#${USER}#g") + HOSTNAME=$(echo "${HOSTNAME}" | sed -e "s#{USER}#${USER}#g") + PORT=$(.venv/bin/python -m yq -r '.ecflow_server.port' "${config_file}") + echo + echo "Deployment Directory: ${DEPLOY_DIR}" + echo "Suite Name: ${SUITE_NAME}" + echo "Hostname: ${HOSTNAME}" + echo "Port: ${PORT}" + echo +} # Main function to parse arguments main() { case "$1" in - # setup) - # rm -rf .venv - # setup - # ;; + setup) + rm -rf .venv + setup + ;; tests) python3 -m pytest tests/ -v "${@:2}" ;; + scripts) + ./deploy.py "user" + ;; + update) + ./deploy.py "user" + if [[ $? -eq 1 ]]; then + echo "Suite not deployed with profile 'user'. Exiting." + exit 1 + fi + # automatically replace the suite conservatively with tracksuite-replace + extract_config_values + tracksuite-replace --def-file "${DEPLOY_DIR}/${SUITE_NAME}.def" --host "${HOSTNAME}" --port "${PORT}" --skip-status --skip-attributes --skip-repeat "${SUITE_NAME}" + ;; + replace) + ./deploy.py "user" + if [[ $? -eq 1 ]]; then + echo "Suite not deployed with profile 'user'. Exiting." + exit 1 + fi + # automatically replace the suite completely with tracksuite-replace + extract_config_values + tracksuite-replace --def-file "${DEPLOY_DIR}/${SUITE_NAME}.def" --host "${HOSTNAME}" --port "${PORT}" --sync-variables "${SUITE_NAME}" + ;; *) ./deploy.py "$@" ;; esac } -module load wellies/${WELLIES_VERSION:-1.1.0} -module list - -# # if local environment doesn't exist create it -# if [ ! -d .venv ]; then -# setup -# fi +# if local environment doesn't exist create it +if ! command -v yq &> /dev/null || [ ! -d ".venv" ] || [ ! -f ".venv/bin/activate" ]; then + setup +fi # Activate the virtual environment -# source .venv/bin/activate +source ".venv/bin/activate" > /dev/null 2>&1 + +module load wellies/${WELLIES_VERSION:-1.2.0} > /dev/null 2>&1 +module list # Run the main function with arguments -main "$@" +main "$@" \ No newline at end of file diff --git a/wellies/templates/clean_init_t b/wellies/templates/clean_init_t new file mode 100644 index 0000000..c78efe3 --- /dev/null +++ b/wellies/templates/clean_init_t @@ -0,0 +1,9 @@ + +set -TEexuva -o pipefail +v + +# clean up wellies build dir as the last package to be installed and can be cleaned +# (WARNING: check no pip install -e is used in any other installer) +rm -rf $LIB_DIR/build $LIB_DIR/git $DATA_DIR/git $SRC_DIR/git + +cd $LIB_DIR/../.. +/usr/bin/tree -L 5 \ No newline at end of file diff --git a/wellies/templates/config.py_t b/wellies/templates/config.py_t index 32e3fe7..00a3dfa 100644 --- a/wellies/templates/config.py_t +++ b/wellies/templates/config.py_t @@ -25,7 +25,7 @@ class Config: self.__dict__.update(options) # Ecflow server options - self.ecflow_server = wl.EcflowServer(**options["ecflow_server"]) + self.ecflow_server = wl.EcflowServer(**options["deploy_host"]) # HPC server options self.host, submit_variables = wl.get_host(**options["host"]) @@ -35,15 +35,18 @@ class Config: # Tools tools = options.get("tools", {}) - self.tools = wl.ToolStore("$LIB_DIR", tools) + self.tools = wl.ToolStore("${LIB_DIR}", tools) # Static data static_data = options.get("static_data", {}) - self.static_data = wl.StaticDataStore("$DATA_DIR", static_data) + self.static_data = wl.StaticDataStore("${DATA_DIR}", static_data) + + # Src + self.src = wl.StaticDataStore("${SRC_DIR}", options.get("src", {})) # Suite variables self.suite_variables = { "SUITE": self.name, **options.get("ecflow_variables", {}), **submit_variables, - } + } \ No newline at end of file diff --git a/wellies/templates/data.yaml_t b/wellies/templates/data.yaml_t index 6455248..cbf12d8 100644 --- a/wellies/templates/data.yaml_t +++ b/wellies/templates/data.yaml_t @@ -2,19 +2,19 @@ # For a complete list of options, see: # https://pyflow-wellies.readthedocs.io/latest/config/data_config/ static_data: - git_sample: - type: git - source: git@github.com:ecmwf/earthkit.git - branch: develop - mars_sample: - type: mars - request: - "class": "od" - "type": "an" - "expver": "1" - "date": "20230930" - "time": "12" - "param": "t" - "levtype": "pressure level" - "levelist": [1000, 850, 700, 500] - "target": "t.grb" \ No newline at end of file + git_sample: + type: git + source: git@github.com:ecmwf/earthkit.git + branch: develop + mars_sample: + type: mars + request: + "class": "od" + "type": "an" + "expver": "1" + "date": "20230930" + "time": "12" + "param": "t" + "levtype": "pressure level" + "levelist": [1000, 850, 700, 500] + "target": "t.grb" \ No newline at end of file diff --git a/wellies/templates/deploy.py_t b/wellies/templates/deploy.py_t index 6e93977..415c954 100644 --- a/wellies/templates/deploy.py_t +++ b/wellies/templates/deploy.py_t @@ -61,4 +61,4 @@ if __name__ == "__main__": no_prompt=args.y, no_deploy=args.no_deploy, message=args.message, - ) + ) \ No newline at end of file diff --git a/wellies/templates/dummy.py_t b/wellies/templates/dummy.py_t new file mode 100644 index 0000000..841dd4a --- /dev/null +++ b/wellies/templates/dummy.py_t @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +A short Hello World script. +""" + +def main(): + print("Hello, world!") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wellies/templates/dummy_t b/wellies/templates/dummy_t new file mode 100644 index 0000000..753c39b --- /dev/null +++ b/wellies/templates/dummy_t @@ -0,0 +1,4 @@ + +set -TEexuva -o pipefail +v + +python ${SRC_DIR}/${SUITE}/dummy.py \ No newline at end of file diff --git a/wellies/templates/generic.man_t_t b/wellies/templates/generic.man_t_t new file mode 100644 index 0000000..8395df6 --- /dev/null +++ b/wellies/templates/generic.man_t_t @@ -0,0 +1,108 @@ +{% raw %} +DESCRIPTION + + ecFlow Suite: {{ suite }} + + The {{ suite }} suite orchestrates hourly ingestion of satellite radiance files, + applies quality control, calibrates the data, and generates level-2 + brightness-temperature products. Input files arrive in an S3 bucket and are + staged into /data/ {{ suite }}/YYYYMMDD_HH. + + The workflow is fully restart-safe: each task checks for pre-existing + artefacts and regenerates only what is missing. You can kill and rerun the + suite at any point without risking data corruption. + + WORKFLOW OVERVIEW + (stage_in --> qc --> calibration --> product_gen --> publish) + + Each stage records provenance metadata and updates a "heartbeat" label that + downstream monitoring tools read for live status. + +DEPENDENCY + + • *stage_in* must complete before *qc* starts; all downstream tasks wait on + *qc* success. + • If you rerun *calibration* for a cycle, *product_gen* and *publish* for the + same cycle are automatically queued for rerun. + • Never initiate a rerun while any *publish* task for the same cycle is still + active—duplicate uploads will result. + • External trigger: *stage_in* waits until ≥ 3 satellite NetCDF files with + pattern *.nc are detected in the inbound S3 path. + • The suite holds if the external service “sat-ingest-api” reports status + ≠ OK in the last 5 minutes. + +OPERATORS (1st-line) + + The primary documentation for operators is found in these ecFlow manual pages. + This documentation should include: + - Step-by-step instructions for common operational tasks. + - Troubleshooting guides for common issues. + - Contact information for 2nd-line support (link to on-call page). + +ANALYSTS (2nd-line) + + Analysts will find relevant documentation in both the ecFlow manual pages and + the repository README and code. This should include: + - Detailed descriptions of the suite''s functionality. + - Examples of typical analysis workflows. + - Troubleshooting guides for more-complex reruns and diagnosis of issues. + - Code snippets demonstrating how to interact with the suite programmatically. + +DEVELOPERS + + Developers should refer to the repository README and code comments for + comprehensive documentation. This should cover: + - Code architecture and design principles. + - Guidelines for contributing to the codebase. + - Detailed comments within the code explaining complex logic and algorithms. + +SCIENTISTS + + Scientists require a high-level description of the suite, which is documented + on a Confluence page. This should include: + - An overview of the suite''s purpose and capabilities. + - Descriptions of key scientific algorithms and models used. + - Links to relevant research papers and technical documentation. + +COMMON ERRORS + {% if task == 'dummy' %} + Missing satellite files + • Symptom: job log shows “No input files found for cycle …”. + • Fix: verify upstream acquisition service, adjust WAIT_FOR_MIN_FILES, and + rerun *stage_in*. + + Checksum mismatch in QC + • Symptom: stderr contains “MD5 mismatch for file …”. + • Fix: mark the file as corrupt in the DB, rerun *stage_in* so the file is + re-downloaded, then rerun *qc*. + + Calibration lookup table expired + • Symptom: *calibration* fails with “LUT date out of range”. + • Fix: update the LUT path via SUITE_LUT_PATH variable, rerun *calibration* + and all downstream tasks. + + Publish permission denied + • Symptom: *publish* stderr shows “403 Forbidden pushing to archive”. + • Fix: refresh the OAuth token in vault, rerun *publish*. + {% else %} + 🪦 --- unchartered territory --- 🪦 This is all in your expert analyst hands. + {% endif %} + +ADDITIONAL RESOURCES + + - Confluence overview page + https://confluence.ecmwf.int/display/FORECAST/High-Level+Suite+Overview + + - ecFlow operator manual + https://confluence.ecmwf.int/display/ECFLOW/Operator+Handbook + + - ECMWF user documentation portal + https://confluence.ecmwf.int/display/UDOC/User+Documentation+Home + + - Example JIRA tickets (integration issues) + https://jira.ecmwf.int/browse/SYS-12345 + https://jira.ecmwf.int/browse/SYS-67890 + + - GitHub repository (dummy example) + https://github.com/ecmwf/dummy-suite +{% endraw %} \ No newline at end of file diff --git a/wellies/templates/host.yaml_t b/wellies/templates/host.yaml_t index dd95c68..79228b0 100644 --- a/wellies/templates/host.yaml_t +++ b/wellies/templates/host.yaml_t @@ -5,9 +5,44 @@ host: workdir: "$TMPDIR" submit_arguments: defaults: - queue: nf + queue: "%EC_QUEUE_PREFIX:n%f" memory_per_cpu: 8Gb - job_name: "%FAMILY1:NOT_DEF%_%TASK%" + threads_per_core: 1 + job_name: "%FAMILY1:NOT_DEF%_%TASK%.%ECF_TRYNO%" tmpdir_size: 20Gb + priority: 70 + sthost: "%STHOST%" sequential: total_tasks: 1 + memory_per_cpu: 64Gb + tmpdir_size: 2Gb + largetmp: + tmpdir_size: 100Gb + parallel: + reservation: "%RESERVE_NAME:%" + queue: "%EC_QUEUE_PREFIX:n%p" + threads_per_core: "%EC_THREADS_PER_CORE:1%" + total_nodes: "%EC_NODES:1%" + cpus_per_task: "%EC_THREADS_PER_TASK:1%" + total_tasks: "%EC_TOTAL_TASKS:1%" + tasks_per_node: "%EC_TASKS_PER_NODE:1%" + enable_hyperthreading: "no" + slurm_keywords_mapping: + queue: '--qos=' + job_name: '--job-name=' + total_tasks: '--ntasks=' + total_nodes: '--nodes=' + cpus_per_task: '--cpus-per-task=' + tasks_per_node: '--ntasks-per-node=' + threads_per_core: '--threads-per-core=' + memory_per_cpu: '--mem-per-cpu=' + billing_account: '--account=' + working_dir: '--chdir=' + time: '--time=' + output: '--output=' + error: '--error=' + priority: '--priority=' + tmpdir_size: '--gres=ssdtmp:' + sthost: '--export=STHOST=' + hint: '--hint=' + distribution: '--distribution=' \ No newline at end of file diff --git a/wellies/templates/launch.json_t b/wellies/templates/launch.json_t new file mode 100644 index 0000000..7620e43 --- /dev/null +++ b/wellies/templates/launch.json_t @@ -0,0 +1,99 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "build setup", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/{{ project }}/__init__.py", + "console": "integratedTerminal", + "preLaunchTask": "build setup" + }, + { + "name": "build tests", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/{{ project }}/__init__.py", + "console": "integratedTerminal", + "postDebugTask": "build tests" + }, + { + "name": "build scripts", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/{{ project }}/__init__.py", + "console": "integratedTerminal", + "postDebugTask": "build scripts" + }, + { + "name": "build update", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/{{ project }}/__init__.py", + "console": "integratedTerminal", + "postDebugTask": "build update" + }, + { + "name": "build replace", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/{{ project }}/__init__.py", + "console": "integratedTerminal", + "postDebugTask": "build replace" + }, + { + "name": "debug deploy user", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/deploy.py", + "console": "integratedTerminal", + "args": ["user"], + "justMyCode": false + }, + { + "name": "debug deploy ", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/deploy.py", + "console": "externalTerminal", + "args": ["${input:profileArg}"], + "justMyCode": false + }, + { + "name": "dummy.py args", + "type": "debugpy", + "request": "launch", + "program": "src/dummy.py", + "console": "integratedTerminal", + "args": ["${input:args}"], + "justMyCode": false + }, + { + "name": "dummy.py args with suite_env", + "type": "debugpy", + "request": "launch", + "python": "{{ lib_dir }}/{{ suite_env }}/bin/python", + "program": "{{ src_dir }}/{{ project }}/dummy.py", + "console": "integratedTerminal", + "args": ["${input:args}"], + "justMyCode": false + } + ], + "inputs": [ + { + "id": "profileArg", + "type": "promptString", + "description": "Enter argument for source build.sh *", + "default": "user" + }, + { + "id": "args", + "type": "promptString", + "description": "Enter argument for dummy.py", + "default": "" + } + ] +} \ No newline at end of file diff --git a/wellies/templates/nodes.py_t b/wellies/templates/nodes.py_t index 7793411..47b871d 100644 --- a/wellies/templates/nodes.py_t +++ b/wellies/templates/nodes.py_t @@ -1,5 +1,64 @@ +import os import pyflow as pf import wellies as wl +from jinja2 import Template +from pathlib import Path + + +class GenericTask(pf.Task): #(wl.EcfResourcesTask): + def __init__(self, **kwargs): + name = kwargs.pop('name') + config = kwargs.pop('config') + # adapt default submit_arguments + submit_arguments = kwargs.pop('submit_arguments', 'sequential') + config.host.submit_arguments[submit_arguments]['job_name'] = name + # add default preamble + env = kwargs.pop('env', 'suite_env') + modules = kwargs.pop('modules', ['python3', 'ecmwf-toolbox']) + # add default script decoration + script = kwargs.pop('script', []) + script_args = kwargs.pop('script_args', {}) + if len(script) == 0: + script += [config.tools.load(env)] + script += [config.tools.load('ps4')] + snippet = Path(__file__).parent.parent / 'snippets' / f"{name}" + # script += [pf.FileScript(str(snippet))] + script += [pf.TemplateFileScript(str(snippet), **script_args)] + # add default variables + variables = kwargs.pop('variables', {}) + variables.update( + { + "DEBUG": 0, + "VERBOSE": 0 + } + ) + # generate man page + manual = kwargs.pop('manual', 'generic') + manual_template = Path(__file__).parent.parent / 'manuals' / f"{manual}.man_t" + template = Template(manual_template.read_text()) + task_manual = Path(__file__).parent.parent / 'manuals' / f"{name}.man" + content = template.render(suite=config.name, task=name) + task_manual.write_text(content) + man_page = "\n".join(pf.TemplateFileScript(str(task_manual)).generate()) + # pop default attributes + label = kwargs.pop('label', 'This is a generic task') + event = kwargs.pop('event', 'set') + inlimits = kwargs.pop('inlimits', []) + defstatus = kwargs.pop('defstatus', 'queued') + # init the pf.Task object + super().__init__( + name = name, + variables=variables, + submit_arguments=submit_arguments, + script=script, + manual=man_page, + modules=modules, + labels=pf.Label('info', label), + events=pf.Event(event), + inlimits=inlimits, + defstatus=getattr(pf.state, defstatus), + **kwargs + ) class InitFamily(pf.AnchorFamily): @@ -7,26 +66,79 @@ class InitFamily(pf.AnchorFamily): super().__init__(name='init', **kwargs) with self: # install environments and packages - wl.DeployToolsFamily( + tools_family = wl.DeployToolsFamily( config.tools, - submit_arguments="sequential", + submit_arguments='sequential', ) # setup static data (remote/local copy/link) - wl.DeployDataFamily( + data_family = wl.DeployDataFamily( config.static_data, - submit_arguments="sequential", + submit_arguments='sequential', ) + + # set up static source code used in the suite tasks + src_family = wl.DeployDataFamily( + config.src, + name='deploy_src', + submit_arguments='sequential' + ) + + with pf.Family(name='clean'): + # clean task + clean_task = GenericTask( + config=config, + name='clean_init' + ) + + # self destruct task + self_destruct = GenericTask( + config=config, + name='self_destruct', + label='Set the self_destruct_toggle and execute this task to obliterate this suite and all its files.', + event='self_destruct_toggle', + defstatus='complete', + modules=['ecflow'], + env=[] + ) + + # set triggers + tools_family >> data_family >> src_family >> clean_task >> self_destruct + self_destruct.triggers &= self_destruct.events class MainFamily(pf.AnchorFamily): def __init__(self, config, **kwargs): super().__init__(name='main', **kwargs) with self: - pf.Task( - name='dummy', + t_dummy_pyflow = pf.Task( + name='dummy_pyflow', submit_arguments="sequential", + script=[ + config.tools.load('suite_env'), + config.tools.load('ps4'), + pf.TemplateFileScript("snippets/dummy") + ] ) + + t_dummy_wellies = wl.EcfResourcesTask( + name='dummy_wellies', + submit_arguments="sequential", + script=[pf.TemplateFileScript("snippets/dummy")] + ) + + t_dummy_generic = GenericTask( + config=config, + name="dummy" + ) + + t_dummy_generic_other_man = GenericTask( + config=config, + name="dummy_other_man", + script=[pf.TemplateFileScript("snippets/dummy")] + ) + + t_dummy_pyflow >> t_dummy_wellies >> t_dummy_generic >> t_dummy_generic_other_man class Suite(pf.Suite): @@ -37,4 +149,4 @@ class Suite(pf.Suite): f_init = InitFamily(config=config, inlimits="work") f_main = MainFamily(config=config, inlimits="work") - f_main.triggers = f_init.complete + f_main.triggers = f_init.complete \ No newline at end of file diff --git a/wellies/templates/profiles.yaml_t b/wellies/templates/profiles.yaml_t index 445516a..3f920b0 100644 --- a/wellies/templates/profiles.yaml_t +++ b/wellies/templates/profiles.yaml_t @@ -4,3 +4,4 @@ user: - configs/host.yaml - configs/data.yaml - configs/tools.yaml + - configs/src.yaml \ No newline at end of file diff --git a/wellies/templates/pyproject.toml_t b/wellies/templates/pyproject.toml_t new file mode 100644 index 0000000..8aea0b3 --- /dev/null +++ b/wellies/templates/pyproject.toml_t @@ -0,0 +1,8 @@ +[project] +name = "{{ project }}" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "yq", + "pyflow-wellies" +] \ No newline at end of file diff --git a/wellies/templates/self_destruct_t b/wellies/templates/self_destruct_t new file mode 100644 index 0000000..fdd0fa5 --- /dev/null +++ b/wellies/templates/self_destruct_t @@ -0,0 +1,15 @@ + +set -TEexuva -o pipefail +v + +%end +export ECF_HOME=%ECF_HOME% +export ECF_HOST=%ECF_HOST% +export ECF_PORT=%ECF_PORT% +%nopp + +toggle_status=$(ecflow_client --host ${ECF_HOST} --port ${ECF_PORT} --query event /${SUITE}/init/clean/self_destruct:self_destruct_toggle) +if [[ "${toggle_status}" == "set" ]]; then + sbatch --mem=4G --ntasks=1 --ntasks-per-node=1 --output=/dev/null --error=/dev/null --wrap="sleep 15; ecflow_client --host ${ECF_HOST} --port ${ECF_PORT} --delete=force yes /${SUITE}; rm -rf ${OUTPUT_ROOT} ${ECF_HOME}/${SUITE} ${DEPLOY_DIR}" +else + echo "Self-destruct toggle is not set. Exiting without action." +fi \ No newline at end of file diff --git a/wellies/templates/src.yaml_t b/wellies/templates/src.yaml_t new file mode 100644 index 0000000..a1c45c0 --- /dev/null +++ b/wellies/templates/src.yaml_t @@ -0,0 +1,10 @@ +src: + {{ project }}: + type: git + # source: git@github.com:ecmwf/pyflow-wellies.git + source: git@github.com:vandome/pyflow-wellies.git + branch: feature/automatic_replace + files: + - wellies/templates/dummy.py_t + post_script: | + mv dummy.py_t dummy.py \ No newline at end of file diff --git a/wellies/templates/tasks.json_t b/wellies/templates/tasks.json_t new file mode 100644 index 0000000..51f9b74 --- /dev/null +++ b/wellies/templates/tasks.json_t @@ -0,0 +1,55 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "shell", + "label": "build setup", + "command": "${workspaceFolder}/build.sh", + "args": [ + "setup" + ], + "group": "build", + "isBackground": false + }, + { + "type": "shell", + "label": "build tests", + "command": "${workspaceFolder}/build.sh", + "args": [ + "tests" + ], + "group": "build", + "isBackground": false + }, + { + "type": "shell", + "label": "build scripts", + "command": "${workspaceFolder}/build.sh", + "args": [ + "scripts" + ], + "group": "build", + "isBackground": false + }, + { + "type": "shell", + "label": "build update", + "command": "${workspaceFolder}/build.sh", + "args": [ + "update" + ], + "group": "build", + "isBackground": false + }, + { + "type": "shell", + "label": "build replace", + "command": "${workspaceFolder}/build.sh", + "args": [ + "replace" + ], + "group": "build", + "isBackground": false + } + ] +} \ No newline at end of file diff --git a/wellies/templates/tools.yaml_t b/wellies/templates/tools.yaml_t index 67516af..02b06d6 100644 --- a/wellies/templates/tools.yaml_t +++ b/wellies/templates/tools.yaml_t @@ -14,18 +14,31 @@ tools: ecmwf-toolbox: version: $MODULES_VERSION depends: [python] + ecflow: + version: $MODULES_VERSION # ---------- Extra installables ----------------------------------------------- packages: + xarray: + type: custom + post_script: "python -m pip install xarray" earthkit: type: git source: git@github.com:ecmwf/earthkit-data.git branch: develop - post_script: "pip install . --no-deps" + post_script: "python -m pip install . --no-deps" # --------Virtual envs -------------------------------------------------- environments: suite_env: type: system_venv depends: [python, ecmwf-toolbox] - packages: [earthkit] + packages: [earthkit, xarray] + + # --------Environment variables ----------------------------------------- + env_variables: + ps4: + variable: PS4 + {% raw %} + value: '"[ \D{{%Y-%m-%dT%H:%M:%S}} | L$LINENO ] --> "' + {% endraw %} \ No newline at end of file diff --git a/wellies/templates/user.yaml_t b/wellies/templates/user.yaml_t index b286f9d..107b7da 100644 --- a/wellies/templates/user.yaml_t +++ b/wellies/templates/user.yaml_t @@ -10,12 +10,18 @@ labels: limits: work: 20 -# -------- Ecflow server ------------------------------------------------------ -ecflow_server: +# -------- Deploy host -------------------------------------------------------- +deploy_host: hostname: "localhost" user: "{{ user }}" deploy_dir: "{{ deploy_dir }}" +# -------- Ecflow server ------------------------------------------------------ +ecflow_server: + hostname: "ecfg-{USER}-1" + port: "3141" + user: "{{ user }}" + # -------- Global variables ------------------------------------ # this group can be redefined in other config files. The resulting option will be the concatenation of all # the variables. Be careful with the order of the variables and the order of the configuration files, @@ -24,6 +30,8 @@ ecflow_variables: OUTPUT_ROOT: "{{ output_root }}" LIB_DIR: "%OUTPUT_ROOT%/local" DATA_DIR: "%OUTPUT_ROOT%/data" + SRC_DIR: "%OUTPUT_ROOT%/src" + DEPLOY_DIR: "{deploy_dir}" # ---------- Specific optionals ----------------------------------------------- -# add your project's specific variables and options here +# add your project's specific variables and options here \ No newline at end of file