FIX: Convert volumes to_cfradial1 containing sweeps with different ra…

…nge and azimuth shapes (#234)

* FIX: use join="outer" in concat of CfRadial1 export/transform
* FIX: make objects mergable, don't raise when attempting to drop missing variables
* FIX: only drop Dataset attrs, not DataArray attrs
* FIX: use combine_by_coords instead of concat, fixup tests
* add history.md entry

docs/history.md

@@ -4,6 +4,7 @@
 
 This is the first version which uses datatree directly from xarray. Thus, xarray is pinned to version >= 2024.10.0.
 
+* FIX: Convert volumes to_cfradial1 containing sweeps with different range and azimuth shapes, raise for different range bin sizes ({issue}`233`) by [@syedhamidali](https://github.com/syedhamidali), ({pull}`234`) by [@kmuehlbauer](https://github.com/kmuehlbauer).
 * FIX: Correctly handle 8bit/16bit, big-endian/little-endian in nexrad reader (PHI and ZDR) ({issue}`230`) by [@syedhamidali](https://github.com/syedhamidali), ({pull}`231`) by [@kmuehlbauer](https://github.com/kmuehlbauer).
 * ENH: Refactoring all xradar backends to use `from_dict` datatree constructor. Test for `_get_required_root`, `_get_subgroup`, and `_get_radar_calibration` were also added ({pull}`221`) by [@aladinor](https://github.com/aladinor)
 * ENH: Added pytests to the missing functions in the `test_xradar` and `test_iris` in order to increase codecov in ({pull}`228`) by [@syedhamidali](https://github.com/syedhamidali).

tests/transform/test_cfradial.py

@@ -2,39 +2,71 @@
 # Copyright (c) 2024, openradar developers.
 # Distributed under the MIT License. See LICENSE for more info.
 
+import pytest
 import xarray as xr
-from open_radar_data import DATASETS
+from xarray import MergeError
 
 import xradar as xd
 
 
-def test_to_cfradial1():
+def test_to_cfradial1(cfradial1_file):
     """Test the conversion from DataTree to CfRadial1 format."""
-    file = DATASETS.fetch("cfrad.20080604_002217_000_SPOL_v36_SUR.nc")
-    dtree = xd.io.open_cfradial1_datatree(file)
+    with xd.io.open_cfradial1_datatree(cfradial1_file) as dtree:
 
-    # Call the conversion function
-    ds_cf1 = xd.transform.to_cfradial1(dtree)
+        # Call the conversion function
+        ds_cf1 = xd.transform.to_cfradial1(dtree)
 
-    # Verify key attributes and data structures in the resulting dataset
-    assert isinstance(ds_cf1, xr.Dataset), "Output is not a valid xarray Dataset"
-    assert "Conventions" in ds_cf1.attrs and ds_cf1.attrs["Conventions"] == "Cf/Radial"
-    assert "sweep_mode" in ds_cf1.variables, "Missing sweep_mode in converted dataset"
-    assert ds_cf1.attrs["version"] == "1.2", "Incorrect CfRadial version"
+        # Verify key attributes and data structures in the resulting dataset
+        assert isinstance(ds_cf1, xr.Dataset), "Output is not a valid xarray Dataset"
+        assert (
+            "Conventions" in ds_cf1.attrs and ds_cf1.attrs["Conventions"] == "Cf/Radial"
+        )
+        assert (
+            "sweep_mode" in ds_cf1.variables
+        ), "Missing sweep_mode in converted dataset"
+        assert ds_cf1.attrs["version"] == "1.2", "Incorrect CfRadial version"
 
 
-def test_to_cfradial2():
+def test_to_cfradial2(cfradial1_file):
     """Test the conversion from CfRadial1 to CfRadial2 DataTree format."""
-    file = DATASETS.fetch("cfrad.20080604_002217_000_SPOL_v36_SUR.nc")
-    dtree = xd.io.open_cfradial1_datatree(file)
+    with xd.io.open_cfradial1_datatree(cfradial1_file) as dtree:
 
-    # Convert to CfRadial1 dataset first
-    ds_cf1 = xd.transform.to_cfradial1(dtree)
+        # Convert to CfRadial1 dataset first
+        ds_cf1 = xd.transform.to_cfradial1(dtree)
 
-    # Call the conversion back to CfRadial2
-    dtree_cf2 = xd.transform.to_cfradial2(ds_cf1)
+        # Call the conversion back to CfRadial2
+        dtree_cf2 = xd.transform.to_cfradial2(ds_cf1)
 
-    # Verify key attributes and data structures in the resulting datatree
-    assert isinstance(dtree_cf2, xr.DataTree), "Output is not a valid DataTree"
-    assert "radar_parameters" in dtree_cf2, "Missing radar_parameters in DataTree"
-    assert dtree_cf2.attrs == ds_cf1.attrs, "Attributes mismatch between formats"
+        # Verify key attributes and data structures in the resulting datatree
+        assert isinstance(dtree_cf2, xr.DataTree), "Output is not a valid DataTree"
+        assert "radar_parameters" in dtree_cf2, "Missing radar_parameters in DataTree"
+        assert dtree_cf2.attrs == ds_cf1.attrs, "Attributes mismatch between formats"
+
+
+def test_to_cfradial1_with_different_range_shapes(nexradlevel2_bzfile):
+    with xd.io.open_nexradlevel2_datatree(nexradlevel2_bzfile) as dtree:
+        ds_cf1 = xd.transform.to_cfradial1(dtree)
+        # Verify key attributes and data structures in the resulting dataset
+        assert isinstance(ds_cf1, xr.Dataset), "Output is not a valid xarray Dataset"
+        assert (
+            "Conventions" in ds_cf1.attrs and ds_cf1.attrs["Conventions"] == "Cf/Radial"
+        )
+        assert (
+            "sweep_mode" in ds_cf1.variables
+        ), "Missing sweep_mode in converted dataset"
+        assert ds_cf1.attrs["version"] == "1.2", "Incorrect CfRadial version"
+        assert ds_cf1.sizes.mapping == {"time": 5400, "range": 1832, "sweep": 11}
+
+        # Call the conversion back to CfRadial2
+        dtree_cf2 = xd.transform.to_cfradial2(ds_cf1)
+        # Verify key attributes and data structures in the resulting datatree
+        assert isinstance(dtree_cf2, xr.DataTree), "Output is not a valid DataTree"
+        # todo: this needs to be fixed in nexrad level2reader
+        # assert "radar_parameters" in dtree_cf2, "Missing radar_parameters in DataTree"
+        assert dtree_cf2.attrs == ds_cf1.attrs, "Attributes mismatch between formats"
+
+
+def test_to_cfradial1_error_with_different_range_bin_sizes(gamic_file):
+    with xd.io.open_gamic_datatree(gamic_file) as dtree:
+        with pytest.raises(MergeError):
+            xd.transform.to_cfradial1(dtree)

xradar/io/backends/common.py

@@ -63,7 +63,8 @@ def _fix_angle(da):
 def _attach_sweep_groups(dtree, sweeps):
     """Attach sweep groups to DataTree."""
     for i, sw in enumerate(sweeps):
-        dtree[f"sweep_{i}"] = xr.DataTree(sw.drop_attrs())
+        # remove attributes only from Dataset's not DataArrays
+        dtree[f"sweep_{i}"] = xr.DataTree(sw.drop_attrs(deep=False))
     return dtree
 
 

xradar/io/export/cfradial1.py

@@ -57,9 +57,8 @@ def _calib_mapper(calib_params):
             attrs=data_array.attrs,
         )
     radar_calib_renamed = xr.Dataset(new_data_vars)
-    dummy_ds = radar_calib_renamed.rename_vars({"r_calib": "fake_coord"})
-    del dummy_ds["fake_coord"]
-    return dummy_ds
+    radar_calib_renamed = radar_calib_renamed.drop_vars("r_calib", errors="ignore")
+    return radar_calib_renamed
 
 
 def _main_info_mapper(dtree):
@@ -135,12 +134,15 @@ def _variable_mapper(dtree, dim0=None):
             # Convert to a dataset and append to the list
             sweep_datasets.append(data)
 
-    result_dataset = xr.concat(
+    # need to use combine_by_coords to correctly test for
+    # incompatible attrs on DataArray's
+    result_dataset = xr.combine_by_coords(
         sweep_datasets,
-        dim="time",
+        data_vars="all",
         compat="no_conflicts",
-        join="right",
-        combine_attrs="drop_conflicts",
+        join="outer",
+        coords="minimal",
+        combine_attrs="no_conflicts",
     )
 
     drop_variables = [
@@ -304,11 +306,11 @@ def to_cfradial1(dtree=None, filename=None, calibs=True):
 
     # Add additional parameters if they exist in dtree
     if "radar_parameters" in dtree:
-        radar_params = dtree["radar_parameters"].to_dataset()
+        radar_params = dtree["radar_parameters"].to_dataset().reset_coords()
         dataset.update(radar_params)
 
     if "georeferencing_correction" in dtree:
-        radar_georef = dtree["georeferencing_correction"].to_dataset()
+        radar_georef = dtree["georeferencing_correction"].to_dataset().reset_coords()
         dataset.update(radar_georef)
 
     # Ensure that the data type of sweep_mode and similar variables matches

xradar/transform/cfradial.py

@@ -106,11 +106,11 @@ def to_cfradial1(dtree=None, filename=None, calibs=True):
 
     # Add additional parameters if they exist in dtree
     if "radar_parameters" in dtree:
-        radar_params = dtree["radar_parameters"].to_dataset()
+        radar_params = dtree["radar_parameters"].to_dataset().reset_coords()
         dataset.update(radar_params)
 
     if "georeferencing_correction" in dtree:
-        radar_georef = dtree["georeferencing_correction"].to_dataset()
+        radar_georef = dtree["georeferencing_correction"].to_dataset().reset_coords()
         dataset.update(radar_georef)
 
     # Ensure that the data type of sweep_mode and similar variables matches