Remove cdo/python-cdo from fre-cli, add xarray time averager

environment.yml

@@ -7,7 +7,6 @@ dependencies:
   - noaa-gfdl::analysis_scripts==0.0.1
   - noaa-gfdl::catalogbuilder==2025.01.01
 #  - noaa-gfdl::fre-nctools==2022.02.01
-  - conda-forge::cdo>=2
   - conda-forge::cftime
   - conda-forge::click>=8.2
   - conda-forge::cmor>=3.14
@@ -22,7 +21,6 @@ dependencies:
   - conda-forge::pytest
   - conda-forge::pytest-cov
   - conda-forge::pylint
-  - conda-forge::python-cdo
   - conda-forge::pyyaml
   - conda-forge::xarray>=2024.*
   - conda-forge::netcdf4>=1.7.*

fre/app/freapp.py

@@ -138,8 +138,8 @@ def mask_atmos_plevel(infile, psfile, outfile, warn_no_ps):
               required = True,
               help = "Output file name")
 @click.option("-p", "--pkg",
-              type = click.Choice(["cdo","fre-nctools","fre-python-tools"]),
-              default = "cdo",
+              type = click.Choice(["cdo","fre-nctools","fre-python-tools","xarray"]),
+              default = "fre-python-tools",
               help = "Time average approach")
 @click.option("-v", "--var",
               type = str,
@@ -192,8 +192,8 @@ def gen_time_averages(inf, outf, pkg, var, unwgt, avg_type):
               required = True,
               help = "Frequency of desired climatology: 'mon' or 'yr'")
 @click.option("-p", "--pkg",
-              type = click.Choice(["cdo","fre-nctools","fre-python-tools"]),
-              default = "cdo",
+              type = click.Choice(["cdo","fre-nctools","fre-python-tools","xarray"]),
+              default = "fre-python-tools",
               help = "Time average approach")
 def gen_time_averages_wrapper(cycle_point, dir_, sources, output_interval, input_interval, grid, frequency, pkg):
     """

fre/app/generate_time_averages/__init__.py

@@ -1,3 +1,4 @@
 '''required for generate_time_averages module import functionality'''
 __all__ = ['generate_time_averages', 'timeAverager', 'wrapper', 'combine',
-           'frenctoolsTimeAverager', 'cdoTimeAverager', 'frepytoolsTimeAverager']
+           'frenctoolsTimeAverager', 'cdoTimeAverager', 'frepytoolsTimeAverager',
+           'xarrayTimeAverager']

fre/app/generate_time_averages/cdoTimeAverager.py

@@ -1,89 +1,35 @@
-''' class using (mostly) cdo functions for time-averages '''
+''' stub that redirects pkg='cdo' requests to the xarray time averager '''
 
 import logging
+import warnings
 
-from netCDF4 import Dataset
-import numpy as np
-
-import cdo
-from cdo import Cdo
-
-from .timeAverager import timeAverager
+from .xarrayTimeAverager import xarrayTimeAverager
 
 fre_logger = logging.getLogger(__name__)
 
-class cdoTimeAverager(timeAverager):
+
+class cdoTimeAverager(xarrayTimeAverager):  # pylint: disable=invalid-name
     '''
-    class inheriting from abstract base class timeAverager
-    generates time-averages using cdo (mostly, see weighted approach)
+    Legacy entry-point kept for backward compatibility.
+    CDO/python-cdo has been removed.  All work is now done by xarrayTimeAverager.
     '''
 
-    def generate_timavg(self, infile = None, outfile = None):
+    def generate_timavg(self, infile=None, outfile=None):
         """
-        use cdo package routines via python bindings        
+        Emit a loud warning then delegate to the xarray implementation.
 
-        :param self: This is an instance of the class cdoTimeAverager
-        :param infile: path to history file, or list of paths, default is None
-        :type infile: str, list
-        :param outfile: path to where output file should be stored, default is None
+        :param infile: path to input NetCDF file
+        :type infile: str
+        :param outfile: path to output file
         :type outfile: str
-        :return: 1 if the instance variable self.avg_typ is unsupported, 0 if function has a clean exit
+        :return: 0 on success
         :rtype: int
         """
-
-        if self.avg_type not in ['all', 'seas', 'month']:
-            fre_logger.error('requested unknown avg_type %s.', self.avg_type)
-            raise ValueError(f'requested unknown avg_type {self.avg_type}')
-
-        if self.var is not None:
-            fre_logger.warning('WARNING: variable specification not twr supported for cdo time averaging. ignoring!')
-
-        fre_logger.info('python-cdo version is %s', cdo.__version__)
-
-        _cdo = Cdo()
-
-        wgts_sum = 0
-        if not self.unwgt: #weighted case, cdo ops alone don't support a weighted time-average.
-
-            nc_fin = Dataset(infile, 'r')
-
-            time_bnds = nc_fin['time_bnds'][:].copy()
-            # Ensure float64 precision for consistent results across numpy versions
-            # NumPy 2.0 changed type promotion rules (NEP 50), so explicit casting
-            # is needed to avoid precision differences
-            time_bnds = np.asarray(time_bnds, dtype=np.float64)
-            # Transpose once to avoid redundant operations
-            time_bnds_transposed = np.moveaxis(time_bnds, 0, -1)
-            wgts = time_bnds_transposed[1] - time_bnds_transposed[0]
-            # Use numpy.sum for consistent dtype handling across numpy versions
-            wgts_sum = np.sum(wgts, dtype=np.float64)
-
-            fre_logger.debug('wgts_sum = %s', wgts_sum)
-
-        if self.avg_type == 'all':
-            fre_logger.info('time average over all time requested.')
-            if self.unwgt:
-                _cdo.timmean(input = infile, output = outfile, returnCdf = True)
-            else:
-                _cdo.divc( str(wgts_sum), input = "-timsum -muldpm "+infile, output = outfile)
-            fre_logger.info('done averaging over all time.')
-
-        elif self.avg_type == 'seas':
-            fre_logger.info('seasonal time-averages requested.')
-            _cdo.yseasmean(input = infile, output = outfile, returnCdf = True)
-            fre_logger.info('done averaging over seasons.')
-
-        elif self.avg_type == 'month':
-            fre_logger.info('monthly time-averages requested.')
-            outfile_str = str(outfile)
-            _cdo.ymonmean(input = infile, output = outfile_str, returnCdf = True)
-            fre_logger.info('done averaging over months.')
-
-            fre_logger.warning(" splitting by month")
-            outfile_root = outfile_str.removesuffix(".nc") + '.'
-            _cdo.splitmon(input = outfile_str, output = outfile_root)
-            fre_logger.debug('Done with splitting by month, outfile_root = %s', outfile_root)
-
-        fre_logger.info('done averaging')
-        fre_logger.info('output file created: %s', outfile)
-        return 0
+        msg = (
+            "WARNING *** CDO/python-cdo has been REMOVED from fre-cli. "
+            "pkg='cdo' now uses the xarray time-averager under the hood. "
+            "Please switch to pkg='xarray' or pkg='fre-python-tools'. ***"
+        )
+        warnings.warn(msg, FutureWarning, stacklevel=2)
+        fre_logger.warning(msg)
+        return super().generate_timavg(infile=infile, outfile=outfile)

fre/app/generate_time_averages/frenctoolsTimeAverager.py

@@ -5,7 +5,7 @@
 from subprocess import Popen, PIPE
 from pathlib import Path
 
-from cdo import Cdo
+import xarray as xr
 from .timeAverager import timeAverager
 
 fre_logger = logging.getLogger(__name__)
@@ -80,38 +80,40 @@ def generate_timavg(self, infile=None, outfile=None):
                 month_output_file_paths[month_index] = os.path.join( output_dir,
                                                                      f"{Path(outfile).stem}.{month_index:02d}.nc")
 
-            cdo = Cdo()
-            #Loop through each month and select the corresponding data
-            for month_index in month_indices:
-
-                #month_name = month_names[month_index - 1]
-                nc_monthly_file = nc_month_file_paths[month_index]
-
-                #Select data for the given month
-                cdo.select(f"month={month_index}", input=infile, output=nc_monthly_file)
-
-                #Run timavg command for newly created file
-                month_output_file = month_output_file_paths[month_index]
-                #timavgcsh_command=['timavg.csh', '-mb','-o', month_output_file, nc_monthly_file]
-                timavgcsh_command=[shutil.which('timavg.csh'), '-dmb','-o', month_output_file, nc_monthly_file]
-                fre_logger.info( 'timavgcsh_command is %s', ' '.join(timavgcsh_command) )
-                exitstatus=1
-                with Popen(timavgcsh_command,
-                           stdout=PIPE, stderr=PIPE, shell=False) as subp:
-                    stdout, stderr = subp.communicate()
-                    stdoutput=stdout.decode()
-                    fre_logger.info('output= %s', stdoutput)
-                    stderror=stderr.decode()
-                    fre_logger.info('error = %s', stderror )
-
-                    if subp.returncode != 0:
-                        fre_logger.error('stderror = %s', stderror)
-                        raise ValueError(f'error: timavg.csh had a problem, subp.returncode = {subp.returncode}')
-
-                    fre_logger.info('%s climatology successfully ran',nc_monthly_file)
-                    exitstatus=0
-
-                #Delete files after being used to generate output files
+            with xr.open_dataset(infile) as ds_in:
+                #Loop through each month and select the corresponding data
+                for month_index in month_indices:
+
+                    #month_name = month_names[month_index - 1]
+                    nc_monthly_file = nc_month_file_paths[month_index]
+
+                    #Select data for the given month
+                    month_ds = ds_in.sel(time=ds_in['time'].dt.month == month_index)
+                    month_ds.to_netcdf(nc_monthly_file)
+                    month_ds.close()
+
+                    #Run timavg command for newly created file
+                    month_output_file = month_output_file_paths[month_index]
+                    #timavgcsh_command=['timavg.csh', '-mb','-o', month_output_file, nc_monthly_file]
+                    timavgcsh_command=[shutil.which('timavg.csh'), '-dmb','-o', month_output_file, nc_monthly_file]
+                    fre_logger.info( 'timavgcsh_command is %s', ' '.join(timavgcsh_command) )
+                    exitstatus=1
+                    with Popen(timavgcsh_command,
+                               stdout=PIPE, stderr=PIPE, shell=False) as subp:
+                        stdout, stderr = subp.communicate()
+                        stdoutput=stdout.decode()
+                        fre_logger.info('output= %s', stdoutput)
+                        stderror=stderr.decode()
+                        fre_logger.info('error = %s', stderror )
+
+                        if subp.returncode != 0:
+                            fre_logger.error('stderror = %s', stderror)
+                            raise ValueError(f'error: timavg.csh had a problem, subp.returncode = {subp.returncode}')
+
+                        fre_logger.info('%s climatology successfully ran',nc_monthly_file)
+                        exitstatus=0
+
+                    #Delete files after being used to generate output files
             shutil.rmtree('monthly_nc_files')
 
         if self.avg_type == 'month':   #End here if month variable used

fre/app/generate_time_averages/frepytoolsTimeAverager.py

@@ -9,7 +9,7 @@
 
 fre_logger = logging.getLogger(__name__)
 
-class frepytoolsTimeAverager(timeAverager):
+class NumpyTimeAverager(timeAverager):  # pylint: disable=invalid-name
     '''
     class inheriting from abstract base class timeAverager
     generates time-averages using a python-native approach
@@ -256,3 +256,6 @@ def generate_timavg(self, infile = None, outfile = None):
         fre_logger.debug('input file closed')
 
         return 0
+
+# backward-compatible alias
+frepytoolsTimeAverager = NumpyTimeAverager  # pylint: disable=invalid-name

fre/app/generate_time_averages/generate_time_averages.py

@@ -3,16 +3,19 @@
 import os
 import logging
 import time
+import warnings
 from typing import Optional, List, Union
 
-from cdo import Cdo
+import xarray as xr
 
-from .cdoTimeAverager import cdoTimeAverager
 from .frenctoolsTimeAverager import frenctoolsTimeAverager
-from .frepytoolsTimeAverager import frepytoolsTimeAverager
+from .frepytoolsTimeAverager import NumpyTimeAverager
+from .xarrayTimeAverager import xarrayTimeAverager
 
 fre_logger = logging.getLogger(__name__)
 
+VALID_PKGS = ['cdo', 'fre-nctools', 'fre-python-tools', 'xarray']
+
 def generate_time_average(infile: Union[str, List[str]] = None,
                           outfile: str = None,
                           pkg: str = None,
@@ -26,7 +29,9 @@
     :type infile: str, list
     :param outfile: path to where output file should be stored
     :type outfile: str
-    :param pkg: which package to use to calculate climatology (cdo, fre-nctools, fre-python-tools)
+    :param pkg: which package to use to calculate climatology
+                ('xarray', 'fre-python-tools', 'fre-nctools', or 'cdo')
+                'cdo' is kept for backward compat but silently uses xarray.
     :type pkg: str
     :param var: optional, not currently supported and defaults to None
     :type var: str
@@ -41,26 +46,26 @@
     fre_logger.debug('called generate_time_average')
     if None in [infile, outfile, pkg]:
         raise ValueError('infile, outfile, and pkg are required inputs')
-    if pkg not in ['cdo', 'fre-nctools', 'fre-python-tools']:
-        raise ValueError(f'argument pkg = {pkg} not known, must be one of: cdo, fre-nctools, fre-python-tools')
+    if pkg not in VALID_PKGS:
+        raise ValueError(f'argument pkg = {pkg} not known, must be one of: {", ".join(VALID_PKGS)}')
     exitstatus = 1
     myavger = None
 
-    # multiple files case Use cdo to merge multiple files if present
+    # multiple files case - merge multiple files if present
     merged = False
     orig_infile_list = None
     if all ( [ type(infile).__name__ == 'list',
                len(infile) > 1 ] ) :
         fre_logger.info('list input argument detected')
         infile_str = [str(item) for item in infile]
 
-        _cdo = Cdo()
         merged_file = "merged_output.nc"
 
-        fre_logger.info('calling cdo mergetime')
+        fre_logger.info('merging input files with xarray')
         fre_logger.debug('output: %s', merged_file)
         fre_logger.debug('inputs: \n %s', ' '.join(infile_str) )
-        _cdo.mergetime(input = ' '.join(infile_str), output = merged_file)
+        with xr.open_mfdataset(infile_str, combine='by_coords') as ds:
+            ds.to_netcdf(merged_file)
 
         # preserve the original file names for later
         orig_infile_list = infile
@@ -69,11 +74,27 @@
         fre_logger.info('file merging success')
 
     if pkg == 'cdo':
-        fre_logger.info('creating a cdoTimeAverager')
-        myavger = cdoTimeAverager( pkg = pkg,
-                                   var = var,
-                                   unwgt = unwgt,
-                                   avg_type = avg_type )
+        # CDO has been removed — warn loudly, use xarray instead
+        msg = (
+            "WARNING *** CDO/python-cdo has been REMOVED from fre-cli. "
+            "pkg='cdo' now uses the xarray time-averager under the hood. "
+            "Please switch to pkg='xarray' or pkg='fre-python-tools'. ***"
+        )
+        warnings.warn(msg, FutureWarning, stacklevel=2)
+        fre_logger.warning(msg)
+        fre_logger.info('creating an xarrayTimeAverager (via pkg=cdo redirect)')
+        myavger = xarrayTimeAverager( pkg = pkg,
+                                      var = var,
+                                      unwgt = unwgt,
+                                      avg_type = avg_type )
+
+    elif pkg == 'xarray':
+        fre_logger.info('creating an xarrayTimeAverager')
+        myavger = xarrayTimeAverager( pkg = pkg,
+                                      var = var,
+                                      unwgt = unwgt,
+                                      avg_type = avg_type )
+
     elif pkg == 'fre-nctools':
         fre_logger.info('creating a frenctoolsTimeAverager')
         myavger = frenctoolsTimeAverager( pkg = pkg,
@@ -87,11 +108,11 @@
             var = orig_infile_list[0].split('/').pop().split('.')[-2]
             fre_logger.warning('extracted var = %s from orig_infile_list[0] = %s', var, orig_infile_list[0] )
 
-        fre_logger.info('creating a frepytoolsTimeAverager')
-        myavger = frepytoolsTimeAverager( pkg = pkg,
-                                          var = var,
-                                          unwgt = unwgt,
-                                          avg_type = avg_type )
+        fre_logger.info('creating a NumpyTimeAverager')
+        myavger = NumpyTimeAverager( pkg = pkg,
+                                     var = var,
+                                     unwgt = unwgt,
+                                     avg_type = avg_type )
 
     # workload
     if myavger is not None:

fre/app/generate_time_averages/tests/test_cdoTimeAverager.py

@@ -10,3 +10,13 @@ def test_cdotimavg_init_error():
         test_avgr = cdo_timavg.cdoTimeAverager(pkg = 'cdo', var = None, unwgt = False, avg_type = 'FOO')
         test_avgr.generate_timavg(infile = None,
                                   outfile = None)
+
+def test_cdotimavg_warns_future():
+    ''' test that FutureWarning is emitted when generate_timavg is called '''
+    with pytest.warns(FutureWarning, match='CDO/python-cdo has been REMOVED'):
+        test_avgr = cdo_timavg.cdoTimeAverager(pkg = 'cdo', var = None, unwgt = False, avg_type = 'all')
+        # this will fail because infile is None, but the warning should fire first
+        try:
+            test_avgr.generate_timavg(infile = 'nonexistent.nc', outfile = 'out.nc')
+        except (FileNotFoundError, ValueError, OSError):
+            pass