Allow fix_file to return dataset objects

doc/conf.py

@@ -460,10 +460,12 @@
     'iris': ('https://scitools-iris.readthedocs.io/en/stable/', None),
     'esmf_regrid': ('https://iris-esmf-regrid.readthedocs.io/en/stable/', None),
     'matplotlib': ('https://matplotlib.org/stable/', None),
+    'ncdata': ('https://ncdata.readthedocs.io/en/stable/', None),
     'numpy': ('https://numpy.org/doc/stable/', None),
     'pyesgf': ('https://esgf-pyclient.readthedocs.io/en/stable/', None),
     'python': ('https://docs.python.org/3/', None),
     'scipy': ('https://docs.scipy.org/doc/scipy/', None),
+    'xarray': ('https://docs.xarray.dev/en/stable/', None),
 }
 
 # -- Extlinks extension -------------------------------------------------------

doc/develop/fixing_data.rst

@@ -126,9 +126,9 @@ Then we have to create the class for the fix deriving from
 Next we must choose the method to use between the ones offered by the
 Fix class:
 
-- ``fix_file``: should be used only to fix errors that prevent data loading.
-  As a rule of thumb, you should only use it if the execution halts before
-  reaching the checks.
+- ``fix_file``: you need to fix errors that prevent loading the data with Iris
+  or perform operations that are more efficient with other packages (e.g.,
+  loading files with lots of variables is much faster with Xarray than Iris).
 
 - ``fix_metadata``: you want to change something in the cube that is not
   the data (e.g., variable or coordinate names, data units).

doc/recipe/preprocessor.rst

@@ -272,20 +272,22 @@ ESMValCore deals with those issues by applying specific fixes for those
 datasets that require them. Fixes are applied at three different preprocessor
 steps:
 
-    - ``fix_file``: apply fixes directly to a copy of the file.
-      Copying the files is costly, so only errors that prevent Iris to load the
-      file are fixed here.
-      See :func:`esmvalcore.preprocessor.fix_file`.
-
-    - ``fix_metadata``: metadata fixes are done just before concatenating the
-      cubes loaded from different files in the final one.
-      Automatic metadata fixes are also applied at this step.
-      See :func:`esmvalcore.preprocessor.fix_metadata`.
-
-    - ``fix_data``: data fixes are applied before starting any operation that
-      will alter the data itself.
-      Automatic data fixes are also applied at this step.
-      See :func:`esmvalcore.preprocessor.fix_data`.
+- ``fix_file``: apply fixes to data before loading them with Iris.
+  This is mainly intended to fix errors that prevent data loading with Iris
+  (e.g., those related to ``missing_value`` or ``_FillValue``) or
+  operations that are more efficient with other packages (e.g., loading
+  files with lots of variables is much faster with Xarray than Iris). See
+  :func:`esmvalcore.preprocessor.fix_file`.
+
+- ``fix_metadata``: metadata fixes are done just before concatenating the
+  cubes loaded from different files in the final one.
+  Automatic metadata fixes are also applied at this step.
+  See :func:`esmvalcore.preprocessor.fix_metadata`.
+
+- ``fix_data``: data fixes are applied before starting any operation that
+  will alter the data itself.
+  Automatic data fixes are also applied at this step.
+  See :func:`esmvalcore.preprocessor.fix_data`.
 
 To get an overview on data fixes and how to implement new ones, please go to
 :ref:`fixing_data`.

environment.yml

@@ -25,6 +25,7 @@ dependencies:
   - jinja2
   - libnetcdf !=4.9.1  # to avoid hdf5 warnings
   - nc-time-axis
+  - ncdata
   - nested-lookup
   - netcdf4
   - numpy !=1.24.3

esmvalcore/cmor/_fixes/cmip6/cesm2.py

@@ -1,5 +1,6 @@
 """Fixes for CESM2 model."""
 
+from pathlib import Path
 from shutil import copyfile
 
 import iris
@@ -22,25 +23,30 @@ class Cl(Fix):
 
     def _fix_formula_terms(
         self,
-        filepath,
-        output_dir,
-        add_unique_suffix=False,
-    ):
+        file: str | Path,
+        output_dir: str | Path,
+        add_unique_suffix: bool = False,
+    ) -> Path:
         """Fix ``formula_terms`` attribute."""
         new_path = self.get_fixed_filepath(
             output_dir,
-            filepath,
+            file,
             add_unique_suffix=add_unique_suffix,
         )
-        copyfile(filepath, new_path)
+        copyfile(file, new_path)
         with Dataset(new_path, mode="a") as dataset:
             dataset.variables["lev"].formula_terms = "p0: p0 a: a b: b ps: ps"
             dataset.variables[
                 "lev"
             ].standard_name = "atmosphere_hybrid_sigma_pressure_coordinate"
         return new_path
 
-    def fix_file(self, filepath, output_dir, add_unique_suffix=False):
+    def fix_file(
+        self,
+        file: str | Path,
+        output_dir: str | Path,
+        add_unique_suffix: bool = False,
+    ) -> Path:
         """Fix hybrid pressure coordinate.
 
         Adds missing ``formula_terms`` attribute to file.
@@ -54,7 +60,7 @@ def fix_file(self, filepath, output_dir, add_unique_suffix=False):
 
         Parameters
         ----------
-        filepath : str
+        file : str
             Path to the original file.
         output_dir: Path
             Output directory for fixed files.
@@ -68,7 +74,7 @@ def fix_file(self, filepath, output_dir, add_unique_suffix=False):
 
         """
         new_path = self._fix_formula_terms(
-            filepath,
+            file,
             output_dir,
             add_unique_suffix=add_unique_suffix,
         )

esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py

@@ -16,7 +16,7 @@
 class Cl(BaseCl):
     """Fixes for cl."""
 
-    def fix_file(self, filepath, output_dir, add_unique_suffix=False):
+    def fix_file(self, file, output_dir, add_unique_suffix=False):
         """Fix hybrid pressure coordinate.
 
         Adds missing ``formula_terms`` attribute to file.
@@ -30,7 +30,7 @@ def fix_file(self, filepath, output_dir, add_unique_suffix=False):
 
         Parameters
         ----------
-        filepath : str
+        file: str
             Path to the original file.
         output_dir: Path
             Output directory for fixed files.
@@ -44,7 +44,7 @@ def fix_file(self, filepath, output_dir, add_unique_suffix=False):
 
         """
         new_path = self._fix_formula_terms(
-            filepath,
+            file,
             output_dir,
             add_unique_suffix=add_unique_suffix,
         )

esmvalcore/cmor/_fixes/emac/emac.py

@@ -41,7 +41,7 @@ class AllVars(EmacFix):
         "kg/m**2s": "kg m-2 s-1",
     }
 
-    def fix_file(self, filepath, output_dir, add_unique_suffix=False):
+    def fix_file(self, file, output_dir, add_unique_suffix=False):
         """Fix file.
 
         Fixes hybrid pressure level coordinate.
@@ -54,13 +54,13 @@ def fix_file(self, filepath, output_dir, add_unique_suffix=False):
 
         """
         if "alevel" not in self.vardef.dimensions:
-            return filepath
+            return file
         new_path = self.get_fixed_filepath(
             output_dir,
-            filepath,
+            file,
             add_unique_suffix=add_unique_suffix,
         )
-        copyfile(filepath, new_path)
+        copyfile(file, new_path)
         with Dataset(new_path, mode="a") as dataset:
             if "formula_terms" in dataset.variables["lev"].ncattrs():
                 del dataset.variables["lev"].formula_terms

esmvalcore/cmor/_fixes/fix.py

@@ -27,11 +27,17 @@
 )
 from esmvalcore.cmor.fixes import get_time_bounds
 from esmvalcore.cmor.table import get_var_info
-from esmvalcore.iris_helpers import has_unstructured_grid, safe_convert_units
+from esmvalcore.iris_helpers import (
+    has_unstructured_grid,
+    safe_convert_units,
+)
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
+    import ncdata
+    import xarray as xr
+
     from esmvalcore.cmor.table import CoordinateInfo, VariableInfo
     from esmvalcore.config import Session
 
@@ -77,34 +83,43 @@ def __init__(
 
     def fix_file(
         self,
-        filepath: Path,
+        file: str | Path | xr.Dataset | ncdata.NcData,
         output_dir: Path,
         add_unique_suffix: bool = False,
-    ) -> str | Path:
-        """Apply fixes to the files prior to creating the cube.
+    ) -> str | Path | xr.Dataset | ncdata.NcData:
+        """Fix files before loading them into a :class:`~iris.cube.CubeList`.
+
+        This is mainly intended to fix errors that prevent loading the data
+        with Iris (e.g., those related to ``missing_value`` or ``_FillValue``)
+        or operations that are more efficient with other packages (e.g.,
+        loading files with lots of variables is much faster with Xarray than
+        Iris).
 
-        Should be used only to fix errors that prevent loading or cannot be
-        fixed in the cube (e.g., those related to `missing_value` or
-        `_FillValue`).
+        Warning
+        -------
+        A path should only be returned if it points to the original (unchanged)
+        file (i.e., a fix was not necessary). If a fix is necessary, this
+        function should return a :class:`~ncdata.NcData` or
+        :class:`~xarray.Dataset` object.  Under no circumstances a copy of the
+        input data should be created (this is very inefficient).
 
         Parameters
         ----------
-        filepath:
-            File to fix.
+        file:
+            Data to fix or path to them. Original files should not be
+            overwritten.
         output_dir:
             Output directory for fixed files.
         add_unique_suffix:
-            Adds a unique suffix to `output_dir` for thread safety.
+            Adds a unique suffix to ``output_dir`` for thread safety.
 
         Returns
         -------
-        str or pathlib.Path
-            Path to the corrected file. It can be different from the original
-            filepath if a fix has been applied, but if not it should be the
-            original filepath.
+        str | pathlib.Path | xr.Dataset | ncdata.NcData:
+            Fixed data or a path to them.
 
         """
-        return filepath
+        return file
 
     def fix_metadata(self, cubes: Sequence[Cube]) -> Sequence[Cube]:
         """Apply fixes to the metadata of the cube.
@@ -120,7 +135,7 @@ def fix_metadata(self, cubes: Sequence[Cube]) -> Sequence[Cube]:
 
         Returns
         -------
-        Iterable[iris.cube.Cube]
+        Sequence[iris.cube.Cube]
             Fixed cubes. They can be different instances.
 
         """

esmvalcore/cmor/_fixes/ipslcm/ipsl_cm6.py

@@ -17,7 +17,7 @@
 class AllVars(Fix):
     """Fixes for all IPSLCM variables."""
 
-    def fix_file(self, filepath, output_dir, add_unique_suffix=False):
+    def fix_file(self, file, output_dir, add_unique_suffix=False):
         """Select IPSLCM variable in filepath.
 
         This is done only if input file is a multi-variable one. This
@@ -35,30 +35,30 @@
         if "_" + self.extra_facets.get(
             "group",
             "non-sense",
-        ) + ".nc" not in str(filepath):
+        ) + ".nc" not in str(file):
             # No need to filter the file
-            logger.debug("Not filtering for %s", filepath)
-            return filepath
+            logger.debug("Not filtering for %s", file)
+            return file
 
         if not self.extra_facets.get("use_cdo", False):
             # The configuration developer doesn't provide CDO, while ESMValTool
             # licence policy doesn't allow to include it in dependencies
             # Or he considers that plain Iris load is quick enough for
             # that file
-            logger.debug("In ipsl-cm6.py : CDO not activated for %s", filepath)
-            return filepath
+            logger.debug("In ipsl-cm6.py : CDO not activated for %s", file)
+            return file
 
         # Proceed with CDO selvar
         varname = self.extra_facets.get(VARNAME_KEY, self.vardef.short_name)
-        alt_filepath = str(filepath).replace(".nc", "_cdo_selected.nc")
+        alt_filepath = str(file).replace(".nc", "_cdo_selected.nc")
         outfile = self.get_fixed_filepath(
             output_dir,
             alt_filepath,
             add_unique_suffix=add_unique_suffix,
         )
         tim1 = time.time()
-        logger.debug("Using CDO for selecting %s in %s", varname, filepath)
-        command = ["cdo", f"-selvar,{varname}", str(filepath), str(outfile)]
+        logger.debug("Using CDO for selecting %s in %s", varname, file)
+        command = ["cdo", f"-selvar,{varname}", str(file), str(outfile)]
         subprocess.run(command, check=True)
         logger.debug("CDO selection done in %.2f seconds", time.time() - tim1)
         return outfile

esmvalcore/cmor/fix.py

@@ -19,6 +19,9 @@
     from collections.abc import Sequence
     from pathlib import Path
 
+    import ncdata
+    import xarray as xr
+
     from esmvalcore.config import Session
 
 logger = logging.getLogger(__name__)
@@ -35,18 +38,26 @@ def fix_file(  # noqa: PLR0913
     session: Session | None = None,
     frequency: str | None = None,
     **extra_facets,
-) -> str | Path:
-    """Fix files before ESMValTool can load them.
+) -> str | Path | xr.Dataset | ncdata.NcData:
+    """Fix files before loading them into a :class:`~iris.cube.CubeList`.
 
-    These fixes are only for issues that prevent iris from loading the cube or
-    that cannot be fixed after the cube is loaded.
+    This is mainly intended to fix errors that prevent loading the data with
+    Iris (e.g., those related to ``missing_value`` or ``_FillValue``) or
+    operations that are more efficient with other packages (e.g., loading files
+    with lots of variables is much faster with Xarray than Iris).
 
-    Original files are not overwritten.
+    Warning
+    -------
+    A path should only be returned if it points to the original (unchanged)
+    file (i.e., a fix was not necessary). If a fix is necessary, this function
+    should return a :class:`~ncdata.NcData` or :class:`~xarray.Dataset` object.
+    Under no circumstances a copy of the input data should be created (this is
+    very inefficient).
 
     Parameters
     ----------
     file:
-        Path to the original file.
+        Path to the original file. Original files are not overwritten.
     short_name:
         Variable's short name.
     project:
@@ -58,7 +69,7 @@ def fix_file(  # noqa: PLR0913
     output_dir:
         Output directory for fixed files.
     add_unique_suffix:
-        Adds a unique suffix to `output_dir` for thread safety.
+        Adds a unique suffix to ``output_dir`` for thread safety.
     session:
         Current session which includes configuration and directory information.
     frequency:
@@ -69,8 +80,8 @@ def fix_file(  # noqa: PLR0913
 
     Returns
     -------
-    str or pathlib.Path
-        Path to the fixed file.
+    str | pathlib.Path | xr.Dataset | ncdata.NcData:
+        Fixed data or a path to them.
 
     """
     # Update extra_facets with variable information given as regular arguments

esmvalcore/exceptions.py

@@ -55,3 +55,11 @@ class ESMValCoreDeprecationWarning(ESMValCoreUserWarning):
 
 class MissingConfigParameter(ESMValCoreUserWarning):
     """Config parameter is missing."""
+
+
+class ESMValCorePreprocessorWarning(ESMValCoreUserWarning):
+    """Custom preprocessor warning."""
+
+
+class ESMValCoreLoadWarning(ESMValCorePreprocessorWarning):
+    """Custom load warning."""