Current expt time for running jobs in payu status

payu/experiment.py

@@ -69,7 +69,7 @@ def wrapper(self, *args, **kwargs):
 
 
 class Experiment(object):
-    def __init__(self, lab, reproduce=False, force=False, metadata_off=False):
+    def __init__(self, lab, reproduce=False, force=False, metadata_off=False, config_path=None):
         self.init_timings()
         self.lab = lab
         # Check laboratory directories are writable
@@ -82,14 +82,14 @@ def __init__(self, lab, reproduce=False, force=False, metadata_off=False):
             self.force = force
 
         # Initialise experiment metadata - uuid and experiment name
-        self.metadata = Metadata(Path(lab.archive_path), disabled=metadata_off)
+        self.metadata = Metadata(Path(lab.archive_path), disabled=metadata_off, config_path=config_path)
         self.metadata.setup()
 
         # TODO: replace with dict, check versions via key-value pairs
         self.modules = set()
 
         # TODO: __init__ should not be a config dumping ground!
-        self.config = read_config()
+        self.config = read_config(config_path)
 
         # Payu experiment type
         self.debug = self.config.get('debug', False)
@@ -857,6 +857,9 @@ def get_model_restart_datetimes(self):
             pass
         return datetimes
 
+    def get_model_cur_expt_time(self):
+        return self.model.get_cur_expt_time()
+
     def archiving(self):
         """
         Determine whether to run archive step based on config.yaml settings.

payu/models/access.py

@@ -283,6 +283,11 @@ def get_restart_datetime(self, restart_path):
         return self.get_restart_datetime_using_submodel(restart_path,
                                                         model_types)
 
+    def get_cur_expt_time(self):
+        """ Use UM submodel to get the current experiment time."""
+        model_types = ['um']
+        return self.get_cur_expt_time_using_submodel(model_types)
+
     def set_model_pathnames(self):
         pass
 

payu/models/access_esm1p6.py

@@ -319,6 +319,11 @@ def get_restart_datetime(self, restart_path):
         return self.get_restart_datetime_using_submodel(restart_path,
                                                         model_types)
 
+    def get_cur_expt_time(self):
+        """ Use UM submodel to get the current experiment time."""
+        model_types = ['um']
+        return self.get_cur_expt_time_using_submodel(model_types)
+
     def set_model_pathnames(self):
         pass
 
@@ -332,4 +337,4 @@ def set_model_output_paths(self):
         pass
 
     def collate(self):
-        pass
+        pass

payu/models/accessom2.py

@@ -12,9 +12,14 @@
 
 import os
 import shutil
+import json
+import warnings
+import cftime
+import logging
 
 from payu.models.model import Model
 
+logger = logging.getLogger(__name__)
 
 class AccessOm2(Model):
 
@@ -94,3 +99,23 @@ def get_restart_datetime(self, restart_path):
 
         return self.get_restart_datetime_using_submodel(restart_path,
                                                         model_types)
+
+    def get_cur_expt_time(self):
+        """Get the current experiment time from file work/atmosphere/log/matmxx.pe00000.log.
+        ---
+        output:
+            cftime.datetime or None if it cannot be determined.
+        """
+        log_path = os.path.join(self.expt.work_path, 'atmosphere', 'log', 
+                                    'matmxx.pe00000.log')
+
+        with open(log_path, 'r') as f:
+            for line in reversed(f.readlines()):
+                if 'cur_exp-datetime' in line:
+                    line_json = json.loads(line)
+                    time_str = line_json.get('cur_exp-datetime', None)
+                    if time_str is not None:
+                        return cftime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S')
+
+        raise ValueError(f"Key 'cur_exp-datetime' not found in {log_path}")
+

payu/models/cesm_cmeps.py

@@ -16,12 +16,15 @@
 import shutil
 import cftime
 from warnings import warn
+import logging
 
 from payu.fsops import make_symlink
 from payu.models.model import Model
 from payu.models.fms import fms_collate
 from payu.models.mom6 import mom6_add_parameter_files, mom6_save_docs_files
 
+logger = logging.getLogger(__name__)
+
 NUOPC_CONFIG = "nuopc.runconfig"
 NUOPC_RUNSEQ = "nuopc.runseq"
 
@@ -422,7 +425,22 @@ def get_components(self):
                 "Access-OM3 comprises a data runoff model, but the runoff model in nuopc.runconfig is set "
                 f"to {self.components['rof']}."
             )
-
+
+    def get_cur_expt_time(self):
+        """Get the current experiment time from file work/log/med.log.
+        ---
+        output:
+            cftime.datetime or None if it cannot be determined.
+        """
+        log_path = os.path.join(self.expt.work_path, 'log', 'med.log')
+
+        with open(log_path, 'r') as f:
+            for line in reversed(f.readlines()):
+                if line.startswith(" memory_write: model date"):
+                    time_str = line.split()[4]
+                    return cftime.datetime.strptime(time_str, '%Y-%m-%dT%H:%M:%S')
+
+        return None
 
 class Runconfig:
     """ Simple class for parsing and editing nuopc.runconfig """

payu/models/model.py

@@ -13,7 +13,6 @@
 from payu import envmod
 from payu.fsops import required_libs
 
-
 class Model(object):
     """Abstract model class."""
 
@@ -449,3 +448,29 @@ def get_restart_datetime_using_submodel(self, restart_path, model_types):
             f'{self.model_type} date-based restart pruning requires one of '
             'these sub-models to determine restart dates.'
         )
+
+    def get_cur_expt_time(self):
+        """For model not implemented experiment time calculate/read-out,
+        leaves a warning and returns None."""
+        print("Current experiment time not implemented for this model.")
+        return None
+
+    def get_cur_expt_time_using_submodel(self, model_types):
+        """
+        Use a specified submodel's get_cur_expt_time method
+
+        Parameters
+        ----------
+        model_types: List of submodels in order of priority. Use first
+        submodel in model_types that is present in the experiment.
+
+        Returns:
+        --------
+        Current experiment time in cftime.datetime
+        """
+        for model_type in model_types:
+            for model in self.expt.models:
+                if model.model_type == model_type:
+                    cur_expt_time = model.get_cur_expt_time()
+                    if cur_expt_time is not None:
+                        return cur_expt_time

payu/models/mom6.py

@@ -10,6 +10,9 @@
 
 # Standard library
 import os
+from datetime import datetime, timedelta
+import cftime
+from collections import deque
 
 # Extensions
 import f90nml
@@ -130,3 +133,39 @@ def archive(self):
         mom6_save_docs_files(self)
 
         super().archive()
+
+    def read_start_date(self, input_path, calendar):
+        """Read the start date from input.nml."""
+        input_nml = f90nml.read(input_path)
+        start_date_list = input_nml.get('ocean_solo_nml', {}).get('date_init', None)
+        if start_date_list is None:
+            raise ValueError(f"Key 'date_init' not found in {input_path}")
+        return cftime.datetime(*start_date_list, calendar=calendar)
+
+    def read_timestep(self, stats_path):
+        """ Read the current timestep from ocean.stats."""
+        with open(stats_path, 'r') as f:
+            line = deque(f, maxlen=1)[0]
+            timestep = float(line.split(',')[1])
+            return timestep
+
+    def get_cur_expt_time(self):
+        """Get the current experiment time from log file.
+        --- 
+        output:
+            cftime.datetime or None if it cannot be determined.
+        """
+        ocean_solo_path = os.path.join(self.expt.work_path, 'INPUT', 'ocean_solo.res')
+        calendar = self.get_calendar(ocean_solo_path)
+
+        input_path = os.path.join(self.expt.work_path, 'input.nml')
+        start_date = self.read_start_date(input_path, calendar)
+
+        stats_path = os.path.join(self.expt.work_path, 'ocean.stats')
+        timestep = self.read_timestep(stats_path)
+
+        if start_date is None or timestep is None:
+            return None
+
+        cur_expt_time = start_date + timedelta(days=timestep)
+        return cur_expt_time

payu/models/mom_mixin.py

@@ -10,6 +10,19 @@
 
 
 class MomMixin:
+    def get_calendar(self, ocean_solo_path):
+        """Get the calendar type from the ocean_solo.res file."""
+        with open(ocean_solo_path, 'r') as ocean_solo:
+            line = ocean_solo.readlines()[0]
+            calendar_int = int(line.split()[0])
+
+        cftime_calendars = {
+            1: "360_day",
+            2: "julian",
+            3: "proleptic_gregorian",
+            4: "noleap"
+        }
+        return cftime_calendars[calendar_int]
 
     def get_restart_datetime(self, restart_path):
         """Given a restart path, parse the restart files and
@@ -21,19 +34,11 @@ def get_restart_datetime(self, restart_path):
                 'Cannot find ocean_solo.res file, which is required for '
                 'date-based restart pruning')
 
-        with open(ocean_solo_path, 'r') as ocean_solo:
-            lines = ocean_solo.readlines()
+        calendar = self.get_calendar(ocean_solo_path)
 
-        calendar_int = int(lines[0].split()[0])
-        cftime_calendars = {
-            1: "360_day",
-            2: "julian",
-            3: "proleptic_gregorian",
-            4: "noleap"
-        }
-        calendar = cftime_calendars[calendar_int]
-
-        last_date_line = lines[-1].split()
+        with open(ocean_solo_path, 'r') as ocean_solo:
+            lines = ocean_solo.readlines()[-1]
+            last_date_line = lines.split()
         date_values = [int(i) for i in last_date_line[:6]]
         year, month, day, hour, minute, second = date_values
         return cftime.datetime(year, month, day, hour, minute, second,

payu/models/um.py

@@ -253,6 +253,44 @@ def get_restart_datetime(self, restart_path):
         # Payu UM always uses proleptic Gregorian calendar
         return cal.date_to_cftime(restart_date, UM_CFTIME_CALENDAR)
 
+    def convert_timestep(self, log_path):
+        """ Convert the timestep to experiment runtime 
+        based on the information in log file"""
+        timestep = None
+        step_per_period = 0
+        secs_per_period = 0
+
+        with open(log_path, 'r') as f:
+            for line in f:
+                if 'STEPS_PER_PERIODim' in line:
+                    step_per_period = int(line.split('=')[-1].strip())
+                if 'SECS_PER_PERIODim' in line:
+                    secs_per_period = int(line.split('=')[-1].strip())
+                if 'Atm_Step: Timestep' in line:
+                    timestep = int(line.split()[-1])
+
+        if timestep is not None and secs_per_period > 0 and step_per_period > 0:
+            secs_per_step = secs_per_period / step_per_period 
+            runtime_sec = timestep * secs_per_step
+            return runtime_sec
+
+        raise ValueError(
+            f"Could not find all required entries in file {log_path}"
+            f" to calculate run time"
+        )
+
+    def get_cur_expt_time(self):
+        """Get the current experiment time from file"""
+        log_path = os.path.join(self.expt.work_path, 'atmosphere', 'atm.fort6.pe0')
+
+        start_date_dir = os.path.join(self.expt.work_path, 'atmosphere')
+        start_date = self.get_restart_datetime(start_date_dir)
+        runtime_sec = self.convert_timestep(log_path)
+        if start_date is None or runtime_sec is None:
+            raise ValueError("Could not determine current experiment time.")
+        cur_expt_time = start_date + datetime.timedelta(seconds=runtime_sec)
+        return cur_expt_time
+
 
 def date_to_um_dump_date(date):
     """

payu/status.py

@@ -8,6 +8,8 @@
 from typing import Any, Optional
 import warnings
 from datetime import datetime
+import json
+import logging
 
 from payu.schedulers import Scheduler
 from payu.telemetry import (
@@ -16,6 +18,7 @@
     remove_job_file
 )
 
+logger = logging.getLogger(__name__)
 
 def find_file_match(pattern: str, path: Path) -> Optional[Path]:
     """Find a file matching the pattern in the given path"""
@@ -132,7 +135,8 @@ def build_job_info(
             archive_path: Path,
             control_path: Path,
             run_number: Optional[int] = None,
-            all_runs: Optional[bool] = False
+            all_runs: Optional[bool] = False,
+            expt=None
         ) -> Optional[dict[str, Any]]:
     """
     Generate a dictionary of jobs information (exit status, stage),
@@ -176,6 +180,10 @@ def build_job_info(
             "start_time": data.get("timings", {}).get("payu_start_time"),
         }
 
+        if data.get("stage") == "archive":
+            # For archive stage, add the model finish time if available
+            run_info["model_finish_time"] = data.get("model_finish_time", None)
+
         run_num = data["payu_current_run"]
         runs.setdefault(run_num, {"run": []})["run"].append(run_info)
 
@@ -197,6 +205,16 @@ def build_job_info(
         for run_num, run_jobs in status_data["runs"].items():
             run_jobs["run"] = [run_jobs["run"][-1]]
 
+    if run_info.get("stage") == "model-run" and expt is not None:
+        try:
+            cur_expt_time = expt.get_model_cur_expt_time()
+            if cur_expt_time is not None:
+                run_info["cur_expt_time"] = cur_expt_time.isoformat()
+        except (FileNotFoundError, IndexError, OSError, json.JSONDecodeError, ValueError, NotImplementedError) as e:
+            logger.debug(f"Cannot parse current experiment time: {e}")
+        except Exception as e:
+            logger.warning(f"Unexpected error while parsing current experiment time: {e}")
+
     return status_data
 
 
@@ -299,10 +317,15 @@ def display_job_info(data: dict[str, Any]) -> None:
             #read out qtime and stime from the job file
             job_file = run_info.get("job_file")
             job_id = run_info.get("job_id")
-            all_job_info = read_job_file(Path(job_file)).get("scheduler_job_info", {})
-            job_info = all_job_info.get("Jobs", {}).get(job_id, {})
+            all_job_info = read_job_file(Path(job_file))
+            job_info = all_job_info.get("scheduler_job_info", {}).get("Jobs", {}).get(job_id, {})
             display_wait_time(job_info.get("qtime", None), job_info.get("stime", None))
 
+            if run_info.get("stage") == "model-run":
+                print_line("Current Expt Time", "cur_expt_time", run_info)
+            if run_info.get("stage") == "archive":
+                model_finish_time = all_job_info.get("model_finish_time", None)
+                print(f"  {'Model Finish Time:':<18} {model_finish_time}")
             exit_status = run_info.get("exit_status")
             if exit_status is not None:
                 status_str = "Success" if exit_status == 0 else "Failed"