Tolerant handling of standard_name and dimension coordinate loading (#6338)

* Basic LOAD_PROBLEMS object.

* Minimum docstring for LOAD_PROBLEMS.

* Working handling of names and dimension coords.

* Warn user of load problems.

* Docstring and UX improvements.

* What's New entry.

* Fix doctests.

* Fix tests.

* Fix doctests for 3.13.

* New tests for actions.

* Updated load problems tests for names.

* Licence header.

* Tidy up code modifications.

* Convenience - get_latest_load_problem.

* Tests for _add_or_capture.

* Test for build_raw_cube.

* Make more methods private.

* Fix doctest.

* Factor out add_method.

* Remove stack_trace TODO.

* Correct docstring of _add_or_capture.

* Extra comments.

* Refactor to create the LoadProblems class.

* Fix doctest.

* What's New improvements.

* Encase coordinate name in quotes.

* More robust LoadProblems testing.

* Remove TODO tags following review discussion.

* Propose mail-archive for addition to linkcheck_ignore.

Author: trexfeathers

docs/src/conf.py

@@ -405,6 +405,7 @@ def _dotv(version):
     "https://twitter.com/scitools_iris",
     "https://stackoverflow.com/questions/tagged/python-iris",
     "https://www.flaticon.com/",
+    "https://www.mail-archive.com/[email protected]/msg39091.html",
 ]
 
 # list of sources to exclude from the build.

docs/src/whatsnew/latest.rst

@@ -73,6 +73,12 @@ This document explains the changes made to Iris for this release
    render text in the bottom right of the plot figure.
    (:issue:`6247`, :pull:`6332`)
 
+#. `@trexfeathers`_ and `@pp-mo`_ added :const:`iris.loading.LOAD_PROBLEMS` to
+   capture objects that could not be loaded correctly, increasing transparency
+   and helping users to fix loading problems via the Iris API. As a first pass,
+   this is currently limited to ``standard_name`` and dimension coordinates from
+   NetCDF files. (:issue:`6317`, :pull:`6338`)
+
 
 🐛 Bugs Fixed
 =============
@@ -99,6 +105,19 @@ This document explains the changes made to Iris for this release
    Once conversion from unittest to pytest is completed, :class:`iris.tests.IrisTest`
    class will be deprecated.
 
+#. `@trexfeathers`_ and `@pp-mo`_ significantly changed Iris' NetCDF loading
+   warnings as part of the :const:`~iris.loading.LOAD_PROBLEMS` work. Several
+   loading operations no longer raise their own warnings; instead their loading
+   problems are captured in :const:`~iris.loading.LOAD_PROBLEMS`. When
+   :const:`~iris.loading.LOAD_PROBLEMS` is populated, a single
+   :class:`~iris.warnings.IrisLoadWarning` is raised; see below.
+   As a first pass, this changed behaviour is currently limited to
+   ``standard_name`` and dimension coordinates from NetCDF files. (:pull:`6338`)
+
+   .. code-block:: none
+
+       IrisLoadWarning: Not all file objects were parsed correctly. See iris.loading.LOAD_PROBLEMS for details.
+
 
 🚀 Performance Enhancements
 ===========================
@@ -166,7 +185,7 @@ This document explains the changes made to Iris for this release
    necessary. (:issue:`6285`, :pull:`6288`)
 
 #. `@trexfeathers`_ improved the handling of benchmark environments, especially
-    when working across Python versions. (:pull:`6329`)
+   when working across Python versions. (:pull:`6329`)
 
 #. `@trexfeathers`_ temporarily pinned Sphinx to `<8.2`.
    (:pull:`6344`, :issue:`6345`)

lib/iris/common/mixin.py

@@ -73,7 +73,6 @@ class LimitedAttributeDict(dict):
 
     """
 
-    #: Attributes with special CF meaning, forbidden in Iris attribute dictionaries.
     CF_ATTRS_FORBIDDEN = (
         "standard_name",
         "long_name",
@@ -94,6 +93,15 @@ class LimitedAttributeDict(dict):
         "scale_factor",
         "_FillValue",
     )
+    """Attributes with special CF meaning, forbidden in Iris attribute dictionaries."""
+
+    IRIS_RAW = "IRIS_RAW"
+    """Key used by Iris to store ALL attributes when problems are encountered during loading.
+
+    See Also
+    --------
+    iris.loading.LOAD_PROBLEMS: The destination for captured loading problems.
+    """
 
     def __init__(self, *args, **kwargs):
         dict.__init__(self, *args, **kwargs)

lib/iris/fileformats/_nc_load_rules/actions.py

@@ -45,6 +45,7 @@
 from iris.config import get_logger
 import iris.fileformats.cf
 import iris.fileformats.pp as pp
+from iris.loading import LOAD_PROBLEMS
 import iris.warnings
 
 from . import helpers as hh
@@ -104,6 +105,10 @@ def inner(engine, *args, **kwargs):
 @action_function
 def action_default(engine):
     """Perform standard operations for every cube."""
+    # Future pattern (iris#6319).
+    hh.build_and_add_names(engine)
+
+    # Legacy pattern.
     hh.build_cube_metadata(engine)
 
 
@@ -286,6 +291,7 @@ def action_build_dimension_coordinate(engine, providescoord_fact):
     cf_var = engine.cf_var.cf_group[var_name]
     rule_name = f"fc_build_coordinate_({coord_type})"
     coord_grid_class, coord_name = _COORDTYPE_GRIDTYPES_AND_COORDNAMES[coord_type]
+    succeed = None
     if coord_grid_class is None:
         # Coordinates not identified with a specific grid-type class (latlon,
         # rotated or projected) are always built, but can have no coord-system.
@@ -367,9 +373,28 @@ def action_build_dimension_coordinate(engine, providescoord_fact):
             assert coord_grid_class in grid_classes
 
     if succeed:
-        hh.build_dimension_coordinate(
+        hh.build_and_add_dimension_coordinate(
             engine, cf_var, coord_name=coord_name, coord_system=coord_system
         )
+
+    else:
+        message = f"Dimension coordinate {var_name} not created. Debug info:\n"
+        if succeed is None:
+            message += "An unexpected error occurred"
+            error = NotImplementedError(message)
+        else:
+            message += rule_name
+            error = ValueError(message)
+
+        try:
+            raise error
+        except error.__class__ as error:
+            _ = LOAD_PROBLEMS.record(
+                filename=engine.filename,
+                loaded=hh.build_raw_cube(cf_var, engine.filename),
+                exception=error,
+            )
+
     return rule_name
 
 

lib/iris/fileformats/_nc_load_rules/engine.py

@@ -19,6 +19,10 @@
 
 """
 
+from iris.coords import _DimensionalMetadata
+from iris.cube import Cube
+from iris.fileformats.cf import CFDataVariable
+
 from .actions import run_actions
 
 
@@ -74,6 +78,11 @@ class Engine:
 
     """
 
+    cf_var: CFDataVariable | None
+    cube: Cube | None
+    cube_parts: dict[str, list[tuple[_DimensionalMetadata, str]]] | None
+    filename: str | None
+
     def __init__(self):
         """Init new engine."""
         self.reset()