From 9762c69cb13405f936188175d22b571807f23fe4 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Thu, 29 Jan 2026 09:26:13 +1100 Subject: [PATCH 1/5] Remove separate entry points for payu-checkout, payu-clone, payu-branch --- pyproject.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c0356e54..44f55acc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,9 +58,6 @@ payu-run = "payu.subcommands.run_cmd:runscript" payu-collate = "payu.subcommands.collate_cmd:runscript" payu-profile = "payu.subcommands.profile_cmd:runscript" payu-sync = "payu.subcommands.sync_cmd:runscript" -payu-branch = "payu.subcommands.branch_cmd:runscript" -payu-clone = "payu.subcommands.clone_cmd:runscript" -payu-checkout = "payu.subcommands.checkout_cmd:runscript" [build-system] build-backend = "setuptools.build_meta" From c29674d2513f7a1351eb2fba0c35890c44d9ada8 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Fri, 30 Jan 2026 09:00:15 +1100 Subject: [PATCH 2/5] Use a wrappers for each runscript payu-cmd --- payu/cli.py | 40 +++++++++++++++++++++++++++++++-- payu/subcommands/collate_cmd.py | 9 +------- payu/subcommands/profile_cmd.py | 9 +------- payu/subcommands/run_cmd.py | 8 +------ payu/subcommands/sync_cmd.py | 8 +------ pyproject.toml | 8 +++---- 6 files changed, 46 insertions(+), 36 deletions(-) diff --git a/payu/cli.py b/payu/cli.py index 6d721cda..c43bdd74 100644 --- a/payu/cli.py +++ b/payu/cli.py @@ -35,8 +35,7 @@ lambda message, category, filename, lineno, line=None: ( formatwarning_orig(message, category, filename, lineno, line='') ) -) - +) def parse(): """Parse the command line inputs and execute the subcommand.""" @@ -53,6 +52,43 @@ def parse(): run_cmd(**args) +# Add wrappers for runscript commands +def parse_run(): + _parse_runscript("run") + + +def parse_collate(): + _parse_runscript("collate") + + +def parse_sync(): + _parse_runscript("sync") + + +def parse_profile(): + _parse_runscript("profile") + + +def _parse_runscript(cmd_name): + + # Attempt to import the requested runscript command module + try: + cmd = importlib.import_module(f'payu.subcommands.{cmd_name}_cmd') + except ImportError: + print(f'payu: error: Unknown runscript command payu-{cmd_name}') + sys.exit(1) + + # Construct the subcommand parser + parser = argparse.ArgumentParser(**cmd.parameters) + + for arg in cmd.arguments: + parser.add_argument(*arg['flags'], **arg['parameters']) + + args = parser.parse_args() + + cmd.runscript(args) + + def generate_parser(is_interactive=False): """Parse the command line inputs generate and return parser.""" diff --git a/payu/subcommands/collate_cmd.py b/payu/subcommands/collate_cmd.py index f481b8e9..3955e01a 100644 --- a/payu/subcommands/collate_cmd.py +++ b/payu/subcommands/collate_cmd.py @@ -89,14 +89,7 @@ def runcmd(model_type, config_path, init_run, lab_path, dir_path): cli.submit_job('payu-collate', pbs_config, pbs_vars) -def runscript(): - - parser = argparse.ArgumentParser() - for arg in arguments: - parser.add_argument(*arg['flags'], **arg['parameters']) - - run_args = parser.parse_args() - +def runscript(run_args): pbs_vars = cli.set_env_vars(init_run=run_args.init_run, lab_path=run_args.lab_path, dir_path=run_args.dir_path) diff --git a/payu/subcommands/profile_cmd.py b/payu/subcommands/profile_cmd.py index b39e380c..0b7afdf5 100644 --- a/payu/subcommands/profile_cmd.py +++ b/payu/subcommands/profile_cmd.py @@ -65,14 +65,7 @@ def runcmd(model_type, config_path, init_run, n_runs, lab_path): cli.submit_job('payu-profile', pbs_config, pbs_vars) -def runscript(): - - parser = argparse.ArgumentParser() - for arg in arguments: - parser.add_argument(*arg['flags'], **arg['parameters']) - - run_args = parser.parse_args() - +def runscript(run_args): pbs_vars = cli.set_env_vars(init_run=run_args.init_run, n_runs=run_args.n_runs) for var in pbs_vars: diff --git a/payu/subcommands/run_cmd.py b/payu/subcommands/run_cmd.py index b41666da..56b193a5 100644 --- a/payu/subcommands/run_cmd.py +++ b/payu/subcommands/run_cmd.py @@ -189,13 +189,7 @@ def runcmd(model_type, config_path, init_run, n_runs, lab_path, ) -def runscript(): - parser = argparse.ArgumentParser() - for arg in arguments: - parser.add_argument(*arg['flags'], **arg['parameters']) - - run_args = parser.parse_args() - +def runscript(run_args): lab = Laboratory(run_args.model_type, run_args.config_path, run_args.lab_path) diff --git a/payu/subcommands/sync_cmd.py b/payu/subcommands/sync_cmd.py index 253b3da5..f59720f7 100644 --- a/payu/subcommands/sync_cmd.py +++ b/payu/subcommands/sync_cmd.py @@ -61,13 +61,7 @@ def runcmd(model_type, config_path, lab_path, dir_path, sync_restarts, cli.submit_job('payu-sync', pbs_config, pbs_vars) -def runscript(): - parser = argparse.ArgumentParser() - for arg in arguments: - parser.add_argument(*arg['flags'], **arg['parameters']) - - run_args = parser.parse_args() - +def runscript(run_args): pbs_vars = cli.set_env_vars(lab_path=run_args.lab_path, dir_path=run_args.dir_path, sync_restarts=run_args.sync_restarts, diff --git a/pyproject.toml b/pyproject.toml index 44f55acc..1eba1653 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,10 +54,10 @@ mitgcm = ["mnctools>=0.2"] [project.scripts] payu = "payu.cli:parse" -payu-run = "payu.subcommands.run_cmd:runscript" -payu-collate = "payu.subcommands.collate_cmd:runscript" -payu-profile = "payu.subcommands.profile_cmd:runscript" -payu-sync = "payu.subcommands.sync_cmd:runscript" +payu-run = "payu.cli:parse_run" +payu-collate = "payu.cli:parse_collate" +payu-profile = "payu.cli:parse_profile" +payu-sync = "payu.cli:parse_sync" [build-system] build-backend = "setuptools.build_meta" From 1716f2005ee9610a8eb9414a2a8d11ea11d40522 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Tue, 31 Mar 2026 13:35:31 +1100 Subject: [PATCH 3/5] Add run command wrapper that sets up logging, captures warnings and exceptions --- payu/cli.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/payu/cli.py b/payu/cli.py index c43bdd74..faf92c9c 100644 --- a/payu/cli.py +++ b/payu/cli.py @@ -10,6 +10,7 @@ import argparse import sysconfig import importlib +import logging import os import pkgutil import shlex @@ -26,6 +27,8 @@ import payu.subcommands from payu.logger import setup_logger +logger = logging.getLogger(__name__) + # Default configuration DEFAULT_CONFIG = 'config.yaml' @@ -37,9 +40,30 @@ ) ) + +def _run_command(func, *args, **kwargs): + """Execute a payu command with error handling and logging. + + Sets up logging, captures warnings through the logging system, + and catches exceptions to provide clean error messages. + """ + setup_logger() + # Capture warnings through the logging system + logging.captureWarnings(True) + + try: + func(*args, **kwargs) + except SystemExit: + # TODO: We want to remove any sys.exit(1) calls in the code + # and replace with exceptions! + raise + except Exception as exc: + logger.error(str(exc)) + sys.exit(1) + + def parse(): """Parse the command line inputs and execute the subcommand.""" - setup_logger() parser = generate_parser(is_interactive = True) # Display help if no arguments are provided if len(sys.argv) == 1: @@ -49,7 +73,7 @@ def parse(): parser = generate_parser() args = vars(parser.parse_args()) run_cmd = args.pop('run_cmd') - run_cmd(**args) + _run_command(run_cmd, **args) # Add wrappers for runscript commands @@ -86,7 +110,7 @@ def _parse_runscript(cmd_name): args = parser.parse_args() - cmd.runscript(args) + _run_command(cmd.runscript, args) def generate_parser(is_interactive=False): From 78723c72c4b23282e726205d8e0abdba1bcd1673 Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Tue, 31 Mar 2026 13:40:17 +1100 Subject: [PATCH 4/5] Move warning formatting to wrapper function and only display warning message by default --- payu/cli.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/payu/cli.py b/payu/cli.py index faf92c9c..23fe63b1 100644 --- a/payu/cli.py +++ b/payu/cli.py @@ -32,14 +32,6 @@ # Default configuration DEFAULT_CONFIG = 'config.yaml' -# Force warnings.warn() to omit the source code line in the message -formatwarning_orig = warnings.formatwarning -warnings.formatwarning = ( - lambda message, category, filename, lineno, line=None: ( - formatwarning_orig(message, category, filename, lineno, line='') - ) -) - def _run_command(func, *args, **kwargs): """Execute a payu command with error handling and logging. @@ -51,6 +43,11 @@ def _run_command(func, *args, **kwargs): # Capture warnings through the logging system logging.captureWarnings(True) + # Only display the warning message + warnings.formatwarning = ( + lambda message, category, filename, lineno, line=None: str(message) + ) + try: func(*args, **kwargs) except SystemExit: From 9feb65ce1d1d0c9cf3c1b00387f527108f25161f Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Tue, 31 Mar 2026 16:38:59 +1100 Subject: [PATCH 5/5] Add --stacktrace and --log-level flags to every payu command --- payu/cli.py | 38 +++++++++++++++++++++++++++++--------- payu/logger.py | 4 ++-- payu/subcommands/args.py | 20 ++++++++++++++++++++ 3 files changed, 51 insertions(+), 11 deletions(-) diff --git a/payu/cli.py b/payu/cli.py index 23fe63b1..58381a5a 100644 --- a/payu/cli.py +++ b/payu/cli.py @@ -25,6 +25,7 @@ from payu.models import index as supported_models from payu.schedulers import index as scheduler_index, DEFAULT_SCHEDULER_CONFIG import payu.subcommands +import payu.subcommands.args as arg_templates from payu.logger import setup_logger logger = logging.getLogger(__name__) @@ -33,20 +34,21 @@ DEFAULT_CONFIG = 'config.yaml' -def _run_command(func, *args, **kwargs): +def _run_command(func, *args, stacktrace=False, log_level=None, **kwargs): """Execute a payu command with error handling and logging. Sets up logging, captures warnings through the logging system, and catches exceptions to provide clean error messages. """ - setup_logger() + setup_logger(log_level=log_level or logging.INFO) # Capture warnings through the logging system logging.captureWarnings(True) - # Only display the warning message - warnings.formatwarning = ( - lambda message, category, filename, lineno, line=None: str(message) - ) + if not stacktrace: + # Only display the warning message + warnings.formatwarning = ( + lambda message, category, filename, lineno, line=None: str(message) + ) try: func(*args, **kwargs) @@ -55,7 +57,10 @@ def _run_command(func, *args, **kwargs): # and replace with exceptions! raise except Exception as exc: - logger.error(str(exc)) + if stacktrace: + logger.exception(str(exc), exc_info=True) + else: + logger.error(str(exc)) sys.exit(1) @@ -70,7 +75,9 @@ def parse(): parser = generate_parser() args = vars(parser.parse_args()) run_cmd = args.pop('run_cmd') - _run_command(run_cmd, **args) + stacktrace = args.pop('stacktrace') + log_level = args.pop('log_level') + _run_command(run_cmd, stacktrace=stacktrace, log_level=log_level, **args) # Add wrappers for runscript commands @@ -102,12 +109,21 @@ def _parse_runscript(cmd_name): # Construct the subcommand parser parser = argparse.ArgumentParser(**cmd.parameters) + # Add global flags to each command + for arg in [arg_templates.stacktrace, arg_templates.log_level]: + parser.add_argument(*arg['flags'], **arg['parameters']) + for arg in cmd.arguments: parser.add_argument(*arg['flags'], **arg['parameters']) args = parser.parse_args() + stacktrace = args.stacktrace + log_level = args.log_level + # Remove them so they don't get passed to runscript + delattr(args, 'stacktrace') + delattr(args, 'log_level') - _run_command(cmd.runscript, args) + _run_command(cmd.runscript, args, stacktrace=stacktrace, log_level=log_level) def generate_parser(is_interactive=False): @@ -132,6 +148,10 @@ def generate_parser(is_interactive=False): cmd_parser = subparsers.add_parser(cmd.title, **cmd.parameters) cmd_parser.set_defaults(run_cmd=cmd.runcmd) + # Add global flags to each subcommand + for arg in [arg_templates.stacktrace, arg_templates.log_level]: + cmd_parser.add_argument(*arg['flags'], **arg['parameters']) + for arg in cmd.arguments: cmd_parser.add_argument(*arg['flags'], **arg['parameters']) diff --git a/payu/logger.py b/payu/logger.py index 9dbc3f87..f6de1b7e 100644 --- a/payu/logger.py +++ b/payu/logger.py @@ -22,14 +22,14 @@ def format(self, record): formatter = logging.Formatter(log_color + self.FORMAT + self.RESET) return formatter.format(record) -def setup_logger(): +def setup_logger(log_level=logging.INFO): """Configure the root logger""" # Color Formatter: Initialize Colorama for cross-platform compatibility and auto-reset init(autoreset=True) logger = logging.getLogger() - logger.setLevel(logging.INFO) + logger.setLevel(log_level) # Create a stream handler and set the custom formatter console_handler = logging.StreamHandler() diff --git a/payu/subcommands/args.py b/payu/subcommands/args.py index 35f9859e..f0c9d98b 100644 --- a/payu/subcommands/args.py +++ b/payu/subcommands/args.py @@ -342,4 +342,24 @@ 'action': 'store', 'help': 'Display information about a specific run number' } +} + +stacktrace = { + 'flags': ('--stacktrace',), + 'parameters': { + 'action': 'store_true', + 'default': False, + 'dest': 'stacktrace', + 'help': 'Show full stack traces on errors and warnings' + } +} + +log_level = { + 'flags': ('--log-level',), + 'parameters': { + 'dest': 'log_level', + 'choices': ['DEBUG', 'INFO', 'WARNING', 'ERROR'], + 'default': None, + 'help': 'Set logging verbosity level' + } } \ No newline at end of file