Allow fix_file to return dataset objects

doc/conf.py

@@ -460,10 +460,12 @@
     'iris': ('https://scitools-iris.readthedocs.io/en/stable/', None),
     'esmf_regrid': ('https://iris-esmf-regrid.readthedocs.io/en/stable/', None),
     'matplotlib': ('https://matplotlib.org/stable/', None),
+    'ncdata': ('https://ncdata.readthedocs.io/en/stable/', None),
     'numpy': ('https://numpy.org/doc/stable/', None),
     'pyesgf': ('https://esgf-pyclient.readthedocs.io/en/stable/', None),
     'python': ('https://docs.python.org/3/', None),
     'scipy': ('https://docs.scipy.org/doc/scipy/', None),
+    'xarray': ('https://docs.xarray.dev/en/stable/', None),
 }
 
 # -- Extlinks extension -------------------------------------------------------

doc/develop/fixing_data.rst

@@ -126,9 +126,9 @@ Then we have to create the class for the fix deriving from
 Next we must choose the method to use between the ones offered by the
 Fix class:
 
-- ``fix_file``: should be used only to fix errors that prevent data loading.
-  As a rule of thumb, you should only use it if the execution halts before
-  reaching the checks.
+- ``fix_file``: you need to fix errors that prevent loading the data with Iris
+  or perform operations that are more efficient with other packages (e.g.,
+  loading files with lots of variables is much faster with Xarray than Iris).
 
 - ``fix_metadata``: you want to change something in the cube that is not
   the data (e.g., variable or coordinate names, data units).

doc/quickstart/configure.rst

@@ -934,6 +934,7 @@ the preprocessing chain.
 
 Currently supported preprocessor steps:
 
+* :func:`~esmvalcore.preprocessor.fix_file`
 * :func:`~esmvalcore.preprocessor.load`
 
 Here is an example on how to ignore specific warnings during the preprocessor

doc/recipe/preprocessor.rst

@@ -272,20 +272,22 @@ ESMValCore deals with those issues by applying specific fixes for those
 datasets that require them. Fixes are applied at three different preprocessor
 steps:
 
-    - ``fix_file``: apply fixes directly to a copy of the file.
-      Copying the files is costly, so only errors that prevent Iris to load the
-      file are fixed here.
-      See :func:`esmvalcore.preprocessor.fix_file`.
-
-    - ``fix_metadata``: metadata fixes are done just before concatenating the
-      cubes loaded from different files in the final one.
-      Automatic metadata fixes are also applied at this step.
-      See :func:`esmvalcore.preprocessor.fix_metadata`.
-
-    - ``fix_data``: data fixes are applied before starting any operation that
-      will alter the data itself.
-      Automatic data fixes are also applied at this step.
-      See :func:`esmvalcore.preprocessor.fix_data`.
+- ``fix_file``: apply fixes to data before loading them with Iris.
+  This is mainly intended to fix errors that prevent data loading with Iris
+  (e.g., those related to ``missing_value`` or ``_FillValue``) or
+  operations that are more efficient with other packages (e.g., loading
+  files with lots of variables is much faster with Xarray than Iris).  See
+  :func:`esmvalcore.preprocessor.fix_file`.
+
+- ``fix_metadata``: metadata fixes are done just before concatenating the
+  cubes loaded from different files in the final one.
+  Automatic metadata fixes are also applied at this step.
+  See :func:`esmvalcore.preprocessor.fix_metadata`.
+
+- ``fix_data``: data fixes are applied before starting any operation that
+  will alter the data itself.
+  Automatic data fixes are also applied at this step.
+  See :func:`esmvalcore.preprocessor.fix_data`.
 
 To get an overview on data fixes and how to implement new ones, please go to
 :ref:`fixing_data`.

environment.yml

@@ -25,6 +25,7 @@ dependencies:
   - jinja2
   - libnetcdf !=4.9.1  # to avoid hdf5 warnings
   - nc-time-axis
+  - ncdata
   - nested-lookup
   - netcdf4
   - numpy !=1.24.3

esmvalcore/cmor/_fixes/cmip6/cesm2.py

@@ -22,15 +22,16 @@
 
     def _fix_formula_terms(
         self,
-        filepath,
+        file,
         output_dir,
         add_unique_suffix=False,
+        ignore_warnings=None,
     ):
         """Fix ``formula_terms`` attribute."""
         new_path = self.get_fixed_filepath(
-            output_dir, filepath, add_unique_suffix=add_unique_suffix
+            output_dir, file, add_unique_suffix=add_unique_suffix
         )
-        copyfile(filepath, new_path)
+        copyfile(file, new_path)
         dataset = Dataset(new_path, mode="a")
         dataset.variables["lev"].formula_terms = "p0: p0 a: a b: b ps: ps"
         dataset.variables[
@@ -39,7 +40,7 @@
         dataset.close()
         return new_path
 
-    def fix_file(self, filepath, output_dir, add_unique_suffix=False):
+    def fix_file(self, file, output_dir, add_unique_suffix=False):
         """Fix hybrid pressure coordinate.
 
         Adds missing ``formula_terms`` attribute to file.
@@ -53,7 +54,7 @@
 
         Parameters
         ----------
-        filepath : str
+        file : str
             Path to the original file.
         output_dir: Path
             Output directory for fixed files.
@@ -67,7 +68,7 @@
 
         """
         new_path = self._fix_formula_terms(
-            filepath, output_dir, add_unique_suffix=add_unique_suffix
+            file, output_dir, add_unique_suffix=add_unique_suffix
         )
         dataset = Dataset(new_path, mode="a")
         dataset.variables["a_bnds"][:] = dataset.variables["a_bnds"][::-1, :]

esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py

@@ -15,7 +15,13 @@
 class Cl(BaseCl):
     """Fixes for cl."""
 
-    def fix_file(self, filepath, output_dir, add_unique_suffix=False):
+    def fix_file(
+        self,
+        file,
+        output_dir,
+        add_unique_suffix=False,
+        ignore_warnings=None,
+    ):
         """Fix hybrid pressure coordinate.
 
         Adds missing ``formula_terms`` attribute to file.
@@ -29,7 +35,7 @@
 
         Parameters
         ----------
-        filepath : str
+        file: str
             Path to the original file.
         output_dir: Path
             Output directory for fixed files.
@@ -43,7 +49,7 @@
 
         """
         new_path = self._fix_formula_terms(
-            filepath, output_dir, add_unique_suffix=add_unique_suffix
+            file, output_dir, add_unique_suffix=add_unique_suffix
         )
         dataset = Dataset(new_path, mode="a")
         dataset.variables["a_bnds"][:] = dataset.variables["a_bnds"][:, ::-1]

esmvalcore/cmor/_fixes/emac/emac.py

@@ -40,7 +40,13 @@ class AllVars(EmacFix):
         "kg/m**2s": "kg m-2 s-1",
     }
 
-    def fix_file(self, filepath, output_dir, add_unique_suffix=False):
+    def fix_file(
+        self,
+        file,
+        output_dir,
+        add_unique_suffix=False,
+        ignore_warnings=None,
+    ):
         """Fix file.
 
         Fixes hybrid pressure level coordinate.
@@ -53,11 +59,11 @@ def fix_file(self, filepath, output_dir, add_unique_suffix=False):
 
         """
         if "alevel" not in self.vardef.dimensions:
-            return filepath
+            return file
         new_path = self.get_fixed_filepath(
-            output_dir, filepath, add_unique_suffix=add_unique_suffix
+            output_dir, file, add_unique_suffix=add_unique_suffix
         )
-        copyfile(filepath, new_path)
+        copyfile(file, new_path)
         with Dataset(new_path, mode="a") as dataset:
             if "formula_terms" in dataset.variables["lev"].ncattrs():
                 del dataset.variables["lev"].formula_terms

esmvalcore/cmor/_fixes/fix.py

@@ -11,7 +11,9 @@
 from typing import TYPE_CHECKING, Any, Optional
 
 import dask
+import ncdata
 import numpy as np
+import xarray as xr
 from cf_units import Unit
 from iris.coords import Coord, CoordExtent
 from iris.cube import Cube, CubeList
@@ -27,7 +29,10 @@
 )
 from esmvalcore.cmor.fixes import get_time_bounds
 from esmvalcore.cmor.table import get_var_info
-from esmvalcore.iris_helpers import has_unstructured_grid, safe_convert_units
+from esmvalcore.iris_helpers import (
+    has_unstructured_grid,
+    safe_convert_units,
+)
 
 if TYPE_CHECKING:
     from esmvalcore.cmor.table import CoordinateInfo, VariableInfo
@@ -75,34 +80,49 @@
 
     def fix_file(
         self,
-        filepath: Path,
+        file: str | Path | Cube | CubeList | xr.Dataset | ncdata.NcData,
         output_dir: Path,
         add_unique_suffix: bool = False,
-    ) -> str | Path:
-        """Apply fixes to the files prior to creating the cube.
+        ignore_warnings: Optional[list[dict[str, Any]]] = None,
+    ) -> str | Path | Cube | CubeList | xr.Dataset | ncdata.NcData:
+        """Fix files before loading them into a :class:`~iris.cube.CubeList`.
+
+        This is mainly intended to fix errors that prevent loading the data
+        with Iris (e.g., those related to ``missing_value`` or ``_FillValue``)
+        or operations that are more efficient with other packages (e.g.,
+        loading files with lots of variables is much faster with Xarray than
+        Iris).
 
-        Should be used only to fix errors that prevent loading or cannot be
-        fixed in the cube (e.g., those related to `missing_value` or
-        `_FillValue`).
+        Warning
+        -------
+        A path should only be returned if it points to the original (unchanged)
+        file (i.e., a fix was not necessary). If a fix is necessary, this
+        function should return a :class:`~iris.cube.Cube`,
+        :class:`~iris.cube.CubeList`, :class:`~ncdata.NcData` or
+        :class:`~xarray.Dataset` object. Under no circumstances a copy of the
+        input data should be created (this is very inefficient).
 
         Parameters
         ----------
-        filepath:
-            File to fix.
+        file:
+            Data to fix or path to them. Original files should not be
+            overwritten.
         output_dir:
             Output directory for fixed files.
         add_unique_suffix:
-            Adds a unique suffix to `output_dir` for thread safety.
+            Adds a unique suffix to ``output_dir`` for thread safety.
+        ignore_warnings:
+            Keyword arguments passed to :func:`warnings.filterwarnings` used to
+            ignore warnings during data loading. Each list element corresponds
+            to one call to :func:`warnings.filterwarnings`.
 
         Returns
         -------
-        str or pathlib.Path
-            Path to the corrected file. It can be different from the original
-            filepath if a fix has been applied, but if not it should be the
-            original filepath.
+        str | pathlib.Path | iris.cube.Cube | iris.cube.CubeList | xr.Dataset | ncdata.NcData:
+            Fixed data or a path to them.
 
         """
-        return filepath
+        return file
 
     def fix_metadata(self, cubes: Sequence[Cube]) -> Sequence[Cube]:
         """Apply fixes to the metadata of the cube.
@@ -118,7 +138,7 @@
 
         Returns
         -------
-        Iterable[iris.cube.Cube]
+        Sequence[iris.cube.Cube]
             Fixed cubes. They can be different instances.
 
         """

esmvalcore/cmor/_fixes/ipslcm/ipsl_cm6.py

@@ -17,7 +17,13 @@
 class AllVars(Fix):
     """Fixes for all IPSLCM variables."""
 
-    def fix_file(self, filepath, output_dir, add_unique_suffix=False):
+    def fix_file(
+        self,
+        file,
+        output_dir,
+        add_unique_suffix=False,
+        ignore_warnings=None,
+    ):
         """Select IPSLCM variable in filepath.
 
         This is done only if input file is a multi-variable one. This
@@ -34,28 +40,28 @@
         """
         if "_" + self.extra_facets.get(
             "group", "non-sense"
-        ) + ".nc" not in str(filepath):
+        ) + ".nc" not in str(file):
             # No need to filter the file
-            logger.debug("Not filtering for %s", filepath)
-            return filepath
+            logger.debug("Not filtering for %s", file)
+            return file
 
         if not self.extra_facets.get("use_cdo", False):
             # The configuration developer doesn't provide CDO, while ESMValTool
             # licence policy doesn't allow to include it in dependencies
             # Or he considers that plain Iris load is quick enough for
             # that file
-            logger.debug("In ipsl-cm6.py : CDO not activated for %s", filepath)
-            return filepath
+            logger.debug("In ipsl-cm6.py : CDO not activated for %s", file)
+            return file
 
         # Proceed with CDO selvar
         varname = self.extra_facets.get(VARNAME_KEY, self.vardef.short_name)
-        alt_filepath = str(filepath).replace(".nc", "_cdo_selected.nc")
+        alt_filepath = str(file).replace(".nc", "_cdo_selected.nc")
         outfile = self.get_fixed_filepath(
             output_dir, alt_filepath, add_unique_suffix=add_unique_suffix
         )
         tim1 = time.time()
-        logger.debug("Using CDO for selecting %s in %s", varname, filepath)
-        command = ["cdo", "-selvar,%s" % varname, str(filepath), str(outfile)]
+        logger.debug("Using CDO for selecting %s in %s", varname, file)
+        command = ["cdo", "-selvar,%s" % varname, str(file), str(outfile)]
         subprocess.run(command, check=True)
         logger.debug("CDO selection done in %.2f seconds", time.time() - tim1)
         return outfile