From ccc1c9d38955b6daa3b09ff860cc5fca6623db64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20M=C3=BChlbauer?= Date: Sun, 3 Nov 2024 17:12:14 +0100 Subject: [PATCH] FIX: Convert volumes to_cfradial1 containing sweeps with different range and azimuth shapes (#234) * FIX: use join="outer" in concat of CfRadial1 export/transform * FIX: make objects mergable, don't raise when attempting to drop missing variables * FIX: only drop Dataset attrs, not DataArray attrs * FIX: use combine_by_coords instead of concat, fixup tests * add history.md entry --- docs/history.md | 1 + tests/transform/test_cfradial.py | 76 +++++++++++++++++++++++--------- xradar/io/backends/common.py | 3 +- xradar/io/export/cfradial1.py | 20 +++++---- xradar/transform/cfradial.py | 4 +- 5 files changed, 70 insertions(+), 34 deletions(-) diff --git a/docs/history.md b/docs/history.md index e4ff41c9..de8481ad 100644 --- a/docs/history.md +++ b/docs/history.md @@ -4,6 +4,7 @@ This is the first version which uses datatree directly from xarray. Thus, xarray is pinned to version >= 2024.10.0. +* FIX: Convert volumes to_cfradial1 containing sweeps with different range and azimuth shapes, raise for different range bin sizes ({issue}`233`) by [@syedhamidali](https://github.com/syedhamidali), ({pull}`234`) by [@kmuehlbauer](https://github.com/kmuehlbauer). * FIX: Correctly handle 8bit/16bit, big-endian/little-endian in nexrad reader (PHI and ZDR) ({issue}`230`) by [@syedhamidali](https://github.com/syedhamidali), ({pull}`231`) by [@kmuehlbauer](https://github.com/kmuehlbauer). * ENH: Refactoring all xradar backends to use `from_dict` datatree constructor. Test for `_get_required_root`, `_get_subgroup`, and `_get_radar_calibration` were also added ({pull}`221`) by [@aladinor](https://github.com/aladinor) * ENH: Added pytests to the missing functions in the `test_xradar` and `test_iris` in order to increase codecov in ({pull}`228`) by [@syedhamidali](https://github.com/syedhamidali). diff --git a/tests/transform/test_cfradial.py b/tests/transform/test_cfradial.py index 5767c5c2..f846b55f 100644 --- a/tests/transform/test_cfradial.py +++ b/tests/transform/test_cfradial.py @@ -2,39 +2,71 @@ # Copyright (c) 2024, openradar developers. # Distributed under the MIT License. See LICENSE for more info. +import pytest import xarray as xr -from open_radar_data import DATASETS +from xarray import MergeError import xradar as xd -def test_to_cfradial1(): +def test_to_cfradial1(cfradial1_file): """Test the conversion from DataTree to CfRadial1 format.""" - file = DATASETS.fetch("cfrad.20080604_002217_000_SPOL_v36_SUR.nc") - dtree = xd.io.open_cfradial1_datatree(file) + with xd.io.open_cfradial1_datatree(cfradial1_file) as dtree: - # Call the conversion function - ds_cf1 = xd.transform.to_cfradial1(dtree) + # Call the conversion function + ds_cf1 = xd.transform.to_cfradial1(dtree) - # Verify key attributes and data structures in the resulting dataset - assert isinstance(ds_cf1, xr.Dataset), "Output is not a valid xarray Dataset" - assert "Conventions" in ds_cf1.attrs and ds_cf1.attrs["Conventions"] == "Cf/Radial" - assert "sweep_mode" in ds_cf1.variables, "Missing sweep_mode in converted dataset" - assert ds_cf1.attrs["version"] == "1.2", "Incorrect CfRadial version" + # Verify key attributes and data structures in the resulting dataset + assert isinstance(ds_cf1, xr.Dataset), "Output is not a valid xarray Dataset" + assert ( + "Conventions" in ds_cf1.attrs and ds_cf1.attrs["Conventions"] == "Cf/Radial" + ) + assert ( + "sweep_mode" in ds_cf1.variables + ), "Missing sweep_mode in converted dataset" + assert ds_cf1.attrs["version"] == "1.2", "Incorrect CfRadial version" -def test_to_cfradial2(): +def test_to_cfradial2(cfradial1_file): """Test the conversion from CfRadial1 to CfRadial2 DataTree format.""" - file = DATASETS.fetch("cfrad.20080604_002217_000_SPOL_v36_SUR.nc") - dtree = xd.io.open_cfradial1_datatree(file) + with xd.io.open_cfradial1_datatree(cfradial1_file) as dtree: - # Convert to CfRadial1 dataset first - ds_cf1 = xd.transform.to_cfradial1(dtree) + # Convert to CfRadial1 dataset first + ds_cf1 = xd.transform.to_cfradial1(dtree) - # Call the conversion back to CfRadial2 - dtree_cf2 = xd.transform.to_cfradial2(ds_cf1) + # Call the conversion back to CfRadial2 + dtree_cf2 = xd.transform.to_cfradial2(ds_cf1) - # Verify key attributes and data structures in the resulting datatree - assert isinstance(dtree_cf2, xr.DataTree), "Output is not a valid DataTree" - assert "radar_parameters" in dtree_cf2, "Missing radar_parameters in DataTree" - assert dtree_cf2.attrs == ds_cf1.attrs, "Attributes mismatch between formats" + # Verify key attributes and data structures in the resulting datatree + assert isinstance(dtree_cf2, xr.DataTree), "Output is not a valid DataTree" + assert "radar_parameters" in dtree_cf2, "Missing radar_parameters in DataTree" + assert dtree_cf2.attrs == ds_cf1.attrs, "Attributes mismatch between formats" + + +def test_to_cfradial1_with_different_range_shapes(nexradlevel2_bzfile): + with xd.io.open_nexradlevel2_datatree(nexradlevel2_bzfile) as dtree: + ds_cf1 = xd.transform.to_cfradial1(dtree) + # Verify key attributes and data structures in the resulting dataset + assert isinstance(ds_cf1, xr.Dataset), "Output is not a valid xarray Dataset" + assert ( + "Conventions" in ds_cf1.attrs and ds_cf1.attrs["Conventions"] == "Cf/Radial" + ) + assert ( + "sweep_mode" in ds_cf1.variables + ), "Missing sweep_mode in converted dataset" + assert ds_cf1.attrs["version"] == "1.2", "Incorrect CfRadial version" + assert ds_cf1.sizes.mapping == {"time": 5400, "range": 1832, "sweep": 11} + + # Call the conversion back to CfRadial2 + dtree_cf2 = xd.transform.to_cfradial2(ds_cf1) + # Verify key attributes and data structures in the resulting datatree + assert isinstance(dtree_cf2, xr.DataTree), "Output is not a valid DataTree" + # todo: this needs to be fixed in nexrad level2reader + # assert "radar_parameters" in dtree_cf2, "Missing radar_parameters in DataTree" + assert dtree_cf2.attrs == ds_cf1.attrs, "Attributes mismatch between formats" + + +def test_to_cfradial1_error_with_different_range_bin_sizes(gamic_file): + with xd.io.open_gamic_datatree(gamic_file) as dtree: + with pytest.raises(MergeError): + xd.transform.to_cfradial1(dtree) diff --git a/xradar/io/backends/common.py b/xradar/io/backends/common.py index 25e15f16..1ab8c69a 100644 --- a/xradar/io/backends/common.py +++ b/xradar/io/backends/common.py @@ -63,7 +63,8 @@ def _fix_angle(da): def _attach_sweep_groups(dtree, sweeps): """Attach sweep groups to DataTree.""" for i, sw in enumerate(sweeps): - dtree[f"sweep_{i}"] = xr.DataTree(sw.drop_attrs()) + # remove attributes only from Dataset's not DataArrays + dtree[f"sweep_{i}"] = xr.DataTree(sw.drop_attrs(deep=False)) return dtree diff --git a/xradar/io/export/cfradial1.py b/xradar/io/export/cfradial1.py index 0b420984..491b652a 100644 --- a/xradar/io/export/cfradial1.py +++ b/xradar/io/export/cfradial1.py @@ -57,9 +57,8 @@ def _calib_mapper(calib_params): attrs=data_array.attrs, ) radar_calib_renamed = xr.Dataset(new_data_vars) - dummy_ds = radar_calib_renamed.rename_vars({"r_calib": "fake_coord"}) - del dummy_ds["fake_coord"] - return dummy_ds + radar_calib_renamed = radar_calib_renamed.drop_vars("r_calib", errors="ignore") + return radar_calib_renamed def _main_info_mapper(dtree): @@ -135,12 +134,15 @@ def _variable_mapper(dtree, dim0=None): # Convert to a dataset and append to the list sweep_datasets.append(data) - result_dataset = xr.concat( + # need to use combine_by_coords to correctly test for + # incompatible attrs on DataArray's + result_dataset = xr.combine_by_coords( sweep_datasets, - dim="time", + data_vars="all", compat="no_conflicts", - join="right", - combine_attrs="drop_conflicts", + join="outer", + coords="minimal", + combine_attrs="no_conflicts", ) drop_variables = [ @@ -304,11 +306,11 @@ def to_cfradial1(dtree=None, filename=None, calibs=True): # Add additional parameters if they exist in dtree if "radar_parameters" in dtree: - radar_params = dtree["radar_parameters"].to_dataset() + radar_params = dtree["radar_parameters"].to_dataset().reset_coords() dataset.update(radar_params) if "georeferencing_correction" in dtree: - radar_georef = dtree["georeferencing_correction"].to_dataset() + radar_georef = dtree["georeferencing_correction"].to_dataset().reset_coords() dataset.update(radar_georef) # Ensure that the data type of sweep_mode and similar variables matches diff --git a/xradar/transform/cfradial.py b/xradar/transform/cfradial.py index c88914d1..d1cde26a 100644 --- a/xradar/transform/cfradial.py +++ b/xradar/transform/cfradial.py @@ -106,11 +106,11 @@ def to_cfradial1(dtree=None, filename=None, calibs=True): # Add additional parameters if they exist in dtree if "radar_parameters" in dtree: - radar_params = dtree["radar_parameters"].to_dataset() + radar_params = dtree["radar_parameters"].to_dataset().reset_coords() dataset.update(radar_params) if "georeferencing_correction" in dtree: - radar_georef = dtree["georeferencing_correction"].to_dataset() + radar_georef = dtree["georeferencing_correction"].to_dataset().reset_coords() dataset.update(radar_georef) # Ensure that the data type of sweep_mode and similar variables matches