Current expt time for running jobs in payu status

payu/experiment.py

@@ -69,7 +69,7 @@ def wrapper(self, *args, **kwargs):
 
 
 class Experiment(object):
-    def __init__(self, lab, reproduce=False, force=False, metadata_off=False):
+    def __init__(self, lab, reproduce=False, force=False, metadata_off=False, config_path=None):
         self.init_timings()
         self.lab = lab
         # Check laboratory directories are writable
@@ -82,14 +82,14 @@ def __init__(self, lab, reproduce=False, force=False, metadata_off=False):
             self.force = force
 
         # Initialise experiment metadata - uuid and experiment name
-        self.metadata = Metadata(Path(lab.archive_path), disabled=metadata_off)
+        self.metadata = Metadata(Path(lab.archive_path), disabled=metadata_off, config_path=config_path)
         self.metadata.setup()
 
         # TODO: replace with dict, check versions via key-value pairs
         self.modules = set()
 
         # TODO: __init__ should not be a config dumping ground!
-        self.config = read_config()
+        self.config = read_config(config_path)
 
         # Payu experiment type
         self.debug = self.config.get('debug', False)
@@ -857,6 +857,9 @@ def get_model_restart_datetimes(self):
             pass
         return datetimes
 
+    def get_model_cur_expt_time(self):
+        return self.model.get_cur_expt_time()
+
     def archiving(self):
         """
         Determine whether to run archive step based on config.yaml settings.

payu/models/access_esm1p6.py

@@ -15,6 +15,7 @@
 import re
 import shutil
 import sys
+import warnings
 
 # Extensions
 import f90nml
@@ -333,3 +334,71 @@ def set_model_output_paths(self):
 
     def collate(self):
         pass
+
+    def read_start_date(self, nl_path):
+        """Read the start date from the namelist file"""
+        try:
+            namelist = f90nml.read(nl_path)
+            model_basis_time = namelist.get('nlstcall', {}).get('model_basis_time', None)
+            if model_basis_time is None:
+                warnings.warn(f"model_basis_time not found in {nl_path}")
+                return None
+            return datetime(*model_basis_time)
+        except Exception as e:
+            warnings.warn(f"Could not read file {nl_path}: {e}")
+            return None
+
+    def convert_timestep(self, log_path):
+        """ Convert the timestep to experiment runtime 
+        based on the information in log file"""
+        timestep = None
+        step_per_period = 0
+        secs_per_period = 0
+
+        with open(log_path, 'r') as f:
+            for line in f:
+                try:
+                    if 'STEPS_PER_PERIODim' in line:
+                        step_per_period = int(line.split('=')[-1].strip())
+                    if 'SECS_PER_PERIODim' in line:
+                        secs_per_period = int(line.split('=')[-1].strip())
+                        print(f"Found secs_per_period: {secs_per_period}")
+                    if 'Atm_Step: Timestep' in line:
+                        timestep = int(line.split()[-1])
+                except ValueError:
+                    continue
+
+        if timestep is not None and secs_per_period > 0 and step_per_period > 0:
+            secs_per_step = secs_per_period / step_per_period 
+            runtime_sec = timestep * secs_per_step
+            return runtime_sec
+
+        warnings.warn(
+                f"""Could not find all required entries in file {log_path}
+                to calculate run time"""
+            )
+        return None
+
+
+    def get_cur_expt_time(self):
+        """Get the current experiment time from file"""
+        nl_path = os.path.join(self.expt.work_path, 'atmosphere', 'namelists')
+        log_path = os.path.join(self.expt.work_path, 'atmosphere', 'atm.fort6.pe0')
+
+        if not os.path.exists(nl_path) or not os.path.exists(log_path):
+            warnings.warn(f"Could not find required files: {nl_path} or {log_path}")
+            return None
+
+        start_date = self.read_start_date(nl_path)
+        runtime_sec = self.convert_timestep(log_path)
+        if start_date is None or runtime_sec is None:
+            return None
+
+        # Both ESM1.6 and datetime package use Gregorian calendar, 
+        # so we can use datetime here for date calculations.
+        cur_expt_time = start_date + timedelta(seconds=runtime_sec)
+        return cur_expt_time.isoformat()
+
+
+
+

payu/models/accessom2.py

@@ -12,6 +12,8 @@
 
 import os
 import shutil
+import json
+import warnings
 
 from payu.models.model import Model
 
@@ -94,3 +96,25 @@ def get_restart_datetime(self, restart_path):
 
         return self.get_restart_datetime_using_submodel(restart_path,
                                                         model_types)
+
+    def get_cur_expt_time(self):
+        """Get the current experiment time from file work/atmosphere/log/matmxx.pe00000.log."""
+        try:
+            log_path = os.path.join(self.expt.work_path, 'atmosphere', 'log', 
+                                    'matmxx.pe00000.log')
+
+            # Read out the latest `cur_exp-datetime` from the log file
+            if os.path.exists(log_path):
+                with open(log_path, 'r') as f:
+                    for line in reversed(f.readlines()):
+                        if 'cur_exp-datetime' in line:
+                            cur_expt_time = json.loads(line)['cur_exp-datetime']
+                            return cur_expt_time
+
+            warnings.warn(f"Log file {log_path} does not exist or does not contain current model time.")
+            return None
+
+        except KeyError as e:
+            warnings.warn('Error getting current experiment time: {}'.format(e))
+            return None
+

payu/models/cesm_cmeps.py

@@ -422,6 +422,27 @@ def get_components(self):
                 "Access-OM3 comprises a data runoff model, but the runoff model in nuopc.runconfig is set "
                 f"to {self.components['rof']}."
             )
+
+    def get_cur_expt_time(self):
+        """Get the current experiment time from file work/log/med.log."""
+        try:
+            log_path = os.path.join(self.expt.work_path, 'log', 'med.log')
+
+            # Read out the latest `cur_exp-datetime` from the log file
+            if os.path.exists(log_path):
+                with open(log_path, 'r') as f:
+                    for line in reversed(f.readlines()):
+                        if line.startswith(" memory_write: model date"):
+                            cur_expt_time = line.split()[4]
+                            return cur_expt_time
+
+            warn(f"Log file {log_path} does not exist or does not contain current model time.")
+            return None
+
+        except KeyError as e:
+            warn('Error getting current experiment time: {}'.format(e))
+            return None
+
 
 
 class Runconfig:

payu/models/model.py

@@ -449,3 +449,31 @@ def get_restart_datetime_using_submodel(self, restart_path, model_types):
             f'{self.model_type} date-based restart pruning requires one of '
             'these sub-models to determine restart dates.'
         )
+
+    def get_cur_expt_time(self):
+        """For model not implemented experiment time calculate/read-out,
+        leaves a warning and returns None."""
+        print("Getting current experiment time is not yet implemented.")
+        return None
+
+    def get_cur_expt_time_using_submodel(self, model_types):
+        """
+        Use a specified submodel's get_cur_expt_time method
+
+        Parameters
+        ----------
+        model_types: List of submodels in order of priority. Use first
+        submodel in model_types that is present in the experiment.
+
+        Returns:
+        --------
+        Current experiment time in string (e.g., 1900-01-01T01:00:00)
+        """
+        for model_type in model_types:
+            for model in self.expt.models:
+                if model.model_type == model_type and hasattr(model, 'get_cur_expt_time'):
+                    cur_expt_time = model.get_cur_expt_time()
+                    if cur_expt_time is not None:
+                        return cur_expt_time
+
+        return None

payu/models/mom6.py

@@ -10,6 +10,7 @@
 
 # Standard library
 import os
+from datetime import datetime, timedelta
 
 # Extensions
 import f90nml
@@ -130,3 +131,44 @@ def archive(self):
         mom6_save_docs_files(self)
 
         super().archive()
+
+    def read_start_date(self, input_path):
+        """Read the start date from input.nml."""
+        try:
+            input_nml = f90nml.read(input_path)
+            start_date = input_nml.get('ocean_solo_nml', {}).get('date_init', None)
+            if start_date is None:
+                warn(f"date_init not found in {input_path}")
+                return None
+            return datetime(*start_date)
+        except Exception as e:
+            warn(f"Could not read file {input_path}: {e}")
+            return None
+
+    def read_timestep(self, stats_path):
+        """ Read the current timestep from ocean.stats."""
+        try:
+            with open(stats_path, 'r') as f:
+                for line in reversed(f.readlines()):
+                    timestep = float(line.split(',')[1])
+                    return timestep
+        except Exception as e:
+            warn(f"Could not read timestep from {stats_path}: {e}")
+            return None
+
+    def get_cur_expt_time(self):
+        """Get the current experiment time from log file."""
+        input_path = os.path.join(self.expt.work_path, 'input.nml')
+        stats_path = os.path.join(self.expt.work_path, 'ocean.stats')
+        if not os.path.exists(input_path) or not os.path.exists(stats_path):
+            warn(f"Could not find required files: {input_path} or {stats_path}")
+            return None
+
+        start_date = self.read_start_date(input_path)
+        timestep = self.read_timestep(stats_path)
+
+        if start_date is None or timestep is None:
+            return None
+
+        cur_expt_time = start_date + timedelta(days=timestep)
+        return cur_expt_time.isoformat()

payu/status.py

@@ -279,7 +279,7 @@ def print_line(label: str, key: Any, data: dict[str, Any]) -> None:
         print(f"  {f'{label}:':<{label_width}} {value}")
 
 
-def display_job_info(data: dict[str, Any]) -> None:
+def display_job_info(data: dict[str, Any], cur_expt_time) -> None:
     """
     Display the job information in a human-readable way
     """
@@ -299,10 +299,15 @@ def display_job_info(data: dict[str, Any]) -> None:
             #read out qtime and stime from the job file
             job_file = run_info.get("job_file")
             job_id = run_info.get("job_id")
-            all_job_info = read_job_file(Path(job_file)).get("scheduler_job_info", {})
-            job_info = all_job_info.get("Jobs", {}).get(job_id, {})
+            all_job_info = read_job_file(Path(job_file))
+            job_info = all_job_info.get("scheduler_job_info", {}).get("Jobs", {}).get(job_id, {})
             display_wait_time(job_info.get("qtime", None), job_info.get("stime", None))
 
+            if run_info.get("stage") == "model-run" and cur_expt_time is not None:
+                print(f"  {'Current Expt Time:':<18} {cur_expt_time}")
+            if run_info.get("stage") == "archive":
+                model_finish_time = all_job_info.get("model_finish_time", None)
+                print(f"  {'Model Finish Time:':<18} {model_finish_time}")
             exit_status = run_info.get("exit_status")
             if exit_status is not None:
                 status_str = "Success" if exit_status == 0 else "Failed"

payu/subcommands/status_cmd.py

@@ -9,6 +9,7 @@
 from payu.fsops import read_config
 from payu.metadata import MetadataWarning, Metadata
 from payu.laboratory import Laboratory
+from payu.experiment import Experiment
 import payu.subcommands.args as args
 from payu.status import (
     build_job_info,
@@ -31,21 +32,19 @@ def runcmd(lab_path, config_path, json_output,
     # Suppress output to os.devnull
     with redirect_stdout(open(os.devnull, 'w')):
         # Determine archive path
-        lab = Laboratory(lab_path)
+        lab = Laboratory(config_path=config_path, lab_path=lab_path)
         warnings.filterwarnings("error", category=MetadataWarning)
         try:
-            metadata = Metadata(Path(lab.archive_path),
-                                config_path=config_path)
-            metadata.setup()
+            expt = Experiment(lab, config_path=config_path)
+            expt.init_models()
+            cur_expt_time = expt.get_model_cur_expt_time()
         except MetadataWarning as e:
             raise RuntimeError(
                 "Metadata is not setup - can't determine archive path"
             )
-        archive_path = Path(lab.archive_path) / metadata.experiment_name
 
-        # Determine control path
-        config = read_config(config_path)
-        control_path = Path(config['control_path'])
+        archive_path = Path(expt.archive_path)
+        control_path = Path(expt.control_path)
 
     run_number = int(run_number) if run_number is not None else None
 
@@ -57,8 +56,7 @@ def runcmd(lab_path, config_path, json_output,
     )
     if update_jobs:
         # Get the scheduler
-        scheduler_name = config.get('scheduler', DEFAULT_SCHEDULER_CONFIG)
-        scheduler = scheduler_index[scheduler_name]()
+        scheduler = expt.scheduler
         # Update the job files in data with the latest information
         # from the scheduler
         update_all_job_files(data, scheduler)
@@ -73,6 +71,6 @@ def runcmd(lab_path, config_path, json_output,
     if json_output:
         print(json.dumps(data, indent=4))
     else:
-        display_job_info(data)
+        display_job_info(data, cur_expt_time=cur_expt_time)
 
 runscript = runcmd

payu/telemetry.py

@@ -495,7 +495,8 @@ def update_run_job_file(
             extra_info: Optional[dict[str, Any]] = None,
             manifests: Optional[dict[str, Any]] = None,
             model_restart_datetimes: Optional[dict[str, Any]] = None,
-            timings: Optional[dict[str, Any]] = None
+            timings: Optional[dict[str, Any]] = None,
+            cur_expt_time: Optional[str] = None
         ) -> None:
     """Update the payu-run job file with the current stage and any extra info
     if defined
@@ -530,6 +531,8 @@ def update_run_job_file(
         run_info.update(extra_info)
     if timings:
         run_info.update(get_timings_isoformat(timings))
+    if cur_expt_time:
+        run_info.update({"cur_expt_time": cur_expt_time})
 
     update_job_file(file_path=file_path, data=run_info)