diff --git a/imap_processing/cli.py b/imap_processing/cli.py index bdb815371..f629b97ed 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -879,11 +879,36 @@ def do_processing( # noqa: PLR0912 f"Expected exactly one DE science dependency. " f"Got {l1b_de_paths}" ) - anc_paths = dependencies.get_file_paths(data_type="ancillary") - if len(anc_paths) != 1: + + # Get ancillary dependencies + anc_dependencies = dependencies.get_processing_inputs( + data_type="ancillary" + ) + if len(anc_dependencies) != 2: + raise ValueError( + f"Expected two ancillary dependencies (cal-prod and " + f"backgrounds). Got " + f"{[anc_dep.descriptor for anc_dep in anc_dependencies]}" + ) + + # Create mapping from descriptor to path + anc_path_dict = { + dep.descriptor.split("-", 1)[1]: dep.imap_file_paths[ + 0 + ].construct_path() + for dep in anc_dependencies + } + + # Verify we have both required ancillary files + if ( + "cal-prod" not in anc_path_dict + or "backgrounds" not in anc_path_dict + ): raise ValueError( - f"Expected exactly one ancillary dependency. Got {anc_paths}" + f"Missing required ancillary files. Expected 'cal-prod' and " + f"'backgrounds', got {list(anc_path_dict.keys())}" ) + # Load goodtimes dependency goodtimes_paths = dependencies.get_file_paths( source="hi", data_type="l1b", descriptor="goodtimes" @@ -893,10 +918,12 @@ def do_processing( # noqa: PLR0912 f"Expected exactly one goodtimes dependency. " f"Got {goodtimes_paths}" ) + datasets = hi_l1c.hi_l1c( load_cdf(l1b_de_paths[0]), - anc_paths[0], + anc_path_dict["cal-prod"], load_cdf(goodtimes_paths[0]), + anc_path_dict["backgrounds"], ) elif self.data_level == "l2": science_paths = dependencies.get_file_paths(source="hi", data_type="l1c") diff --git a/imap_processing/hi/hi_l1c.py b/imap_processing/hi/hi_l1c.py index 266fc3f77..4cc848df2 100644 --- a/imap_processing/hi/hi_l1c.py +++ b/imap_processing/hi/hi_l1c.py @@ -13,10 +13,12 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.cdf.utils import parse_filename_like from imap_processing.hi.utils import ( + BackgroundConfig, CalibrationProductConfig, HiConstants, create_dataset_variables, full_dataarray, + iter_background_events_by_config, iter_qualified_events_by_config, parse_sensor_number, ) @@ -44,6 +46,7 @@ def hi_l1c( de_dataset: xr.Dataset, calibration_prod_config_path: Path, goodtimes_ds: xr.Dataset, + background_config_path: Path, ) -> list[xr.Dataset]: """ High level IMAP-Hi l1c processing function. @@ -56,6 +59,8 @@ def hi_l1c( Calibration product configuration file. goodtimes_ds : xarray.Dataset Goodtimes dataset with cull_flags. + background_config_path : pathlib.Path + Background configuration file. Returns ------- @@ -65,7 +70,7 @@ def hi_l1c( logger.info("Running Hi l1c processing") l1c_dataset = generate_pset_dataset( - de_dataset, calibration_prod_config_path, goodtimes_ds + de_dataset, calibration_prod_config_path, goodtimes_ds, background_config_path ) return [l1c_dataset] @@ -75,6 +80,7 @@ def generate_pset_dataset( de_dataset: xr.Dataset, calibration_prod_config_path: Path, goodtimes_ds: xr.Dataset, + background_config_path: Path, ) -> xr.Dataset: """ Generate IMAP-Hi l1c pset xarray dataset from l1b product. @@ -87,6 +93,8 @@ def generate_pset_dataset( Calibration product configuration file. goodtimes_ds : xarray.Dataset Goodtimes dataset with cull_flags. + background_config_path : pathlib.Path + Background configuration file. Returns ------- @@ -100,6 +108,8 @@ def generate_pset_dataset( logical_source_parts = parse_filename_like(de_dataset.attrs["Logical_source"]) # read calibration product configuration file config_df = CalibrationProductConfig.from_csv(calibration_prod_config_path) + # read background configuration file + background_df = BackgroundConfig.from_csv(background_config_path) pset_dataset = empty_pset_dataset( de_dataset.ccsds_met.data.mean(), @@ -119,8 +129,17 @@ def generate_pset_dataset( ) # Calculate and add the exposure time to the pset_dataset pset_dataset.update(pset_exposure(pset_dataset.coords, de_dataset, goodtimes_ds)) - # Get the backgrounds - pset_dataset.update(pset_backgrounds(pset_dataset.coords)) + + # Compute backgrounds (background counts computed internally) + pset_dataset.update( + pset_backgrounds( + pset_dataset.coords, + background_df, + de_dataset, + goodtimes_ds, + pset_dataset["exposure_times"], + ) + ) return pset_dataset @@ -347,10 +366,9 @@ def pset_counts( Returns ------- dict[str, xarray.DataArray] - Dictionary containing new exposure_times DataArray to be added to the PSET - dataset. + Dictionary containing counts DataArray. """ - # Generate exposure time variable filled with zeros + # Generate counts variable filled with zeros counts_var = create_dataset_variables( ["counts"], coords=pset_coords, @@ -417,43 +435,237 @@ def pset_counts( spin_bin_indices, 1, ) + return counts_var -def pset_backgrounds(pset_coords: dict[str, xr.DataArray]) -> dict[str, xr.DataArray]: +def _compute_background_counts( + pset_coords: dict[str, xr.DataArray], + background_config_df: pd.DataFrame, + l1b_de_dataset: xr.Dataset, + goodtimes_ds: xr.Dataset, +) -> xr.DataArray: """ - Calculate pointing set backgrounds and background uncertainties. + Compute background counts by filtering and binning direct events. + + Background counts are computed across all esa_energy_steps and spin_angle_bins + since backgrounds are isotropic and do not depend on ESA energy step or spin angle. Parameters ---------- pset_coords : dict[str, xarray.DataArray] The PSET coordinates from the xarray.Dataset. + background_config_df : pandas.DataFrame + Background configuration DataFrame with MultiIndex + (calibration_prod, background_index). + l1b_de_dataset : xarray.Dataset + The L1B dataset for the pointing being processed. + goodtimes_ds : xarray.Dataset + Goodtimes dataset with cull_flags. + + Returns + ------- + xarray.DataArray + Background counts with dims (epoch, calibration_prod, background_index). + """ + # Create background_counts as xarray DataArray with proper coordinates + # Note: esa_energy_step and spin_angle_bin are NOT included since backgrounds + # are isotropic and computed across all ESA steps and spin angles + background_indices = ( + background_config_df.index.get_level_values("background_index") + .unique() + .sort_values() + .values + ) + + bg_coords = { + "epoch": pset_coords["epoch"], + "calibration_prod": pset_coords["calibration_prod"], + "background_index": background_indices, + } + + background_counts = xr.DataArray( + np.zeros( + ( + len(bg_coords["epoch"]), + len(bg_coords["calibration_prod"]), + len(bg_coords["background_index"]), + ), + ), + dims=[ + "epoch", + "calibration_prod", + "background_index", + ], + coords=bg_coords, + ) + + # Process direct events + de_ds = l1b_de_dataset.drop_dims("epoch") + + good_mask = de_ds["trigger_id"].data != de_ds["trigger_id"].attrs["FILLVAL"] + if not np.any(good_mask): + return background_counts + + if not np.all(good_mask): + raise ValueError( + "An event with trigger_id=FILLVAL should only occur for a pointing " + "with no events that gets a single fill event. Events with mixed " + "valid and FILLVAL trigger_ids found." + ) + + # Remove DEs not in Goodtimes/angles + goodtimes_mask = good_time_and_phase_mask( + de_ds.event_met.values, + de_ds.nominal_bin.values, + goodtimes_ds, + ) + de_ds = de_ds.isel(event_met=goodtimes_mask) + + n_events = len(de_ds["event_met"]) + if n_events == 0: + return background_counts + + for cal_prod in pset_coords["calibration_prod"].values: + # Check that cal_prod exists in background_config_df + if cal_prod not in background_config_df.index.get_level_values( + "calibration_prod" + ): + raise ValueError( + f"Calibration product {cal_prod} not found in background " + f"configuration. Available calibration products: " + f"{sorted(background_config_df.index.get_level_values('calibration_prod').unique().tolist())}" + ) + + # Take a cross-section of the background configuration DataFrame + # to get rows relevant to the current calibration product + cal_prod_rows = background_config_df.xs(cal_prod, level="calibration_prod") + + # Use iter_background_events_by_config to get filtered events + for config_row, filtered_de_ds in iter_background_events_by_config( + de_ds, cal_prod_rows + ): + background_idx = config_row.Index + + if len(filtered_de_ds["event_met"]) == 0: + continue + + # Count all filtered events + # (no binning by spin angle since backgrounds are isotropic) + count = len(filtered_de_ds["event_met"]) + + background_counts.loc[ + dict( + epoch=pset_coords["epoch"].values[0], + calibration_prod=cal_prod, + background_index=background_idx, + ) + ] += count + + return background_counts + + +def pset_backgrounds( + pset_coords: dict[str, xr.DataArray], + background_config_df: pd.DataFrame, + l1b_de_dataset: xr.Dataset, + goodtimes_ds: xr.Dataset, + exposure_times: xr.DataArray, +) -> dict[str, xr.DataArray]: + """ + Calculate pointing set backgrounds from direct events. + + Computes background counts internally by filtering and binning events + according to the background configuration, then calculates background + rates and uncertainties. + + Parameters + ---------- + pset_coords : dict[str, xarray.DataArray] + The PSET coordinates from the xarray.Dataset. + background_config_df : pandas.DataFrame + Background configuration DataFrame with MultiIndex + (calibration_prod, background_index). + l1b_de_dataset : xarray.Dataset + The L1B dataset for the pointing being processed. + goodtimes_ds : xarray.Dataset + Goodtimes dataset with cull_flags. + exposure_times : xarray.DataArray + Exposure times with dims (epoch, esa_energy_step, spin_angle_bin). Returns ------- dict[str, xarray.DataArray] - Dictionary containing background_rates and background_rates_unc DataArrays - to be added to the PSET dataset. + Dictionary containing background_rates and background_rates_uncertainty + DataArrays to be added to the PSET dataset. """ - # TODO: This is just a placeholder setting backgrounds to zero. The background - # algorithm will be determined in flight. attr_mgr = ImapCdfAttributes() attr_mgr.add_instrument_global_attrs("hi") attr_mgr.add_instrument_variable_attrs(instrument="hi", level=None) - return { + # Create output arrays + output_vars = { var_name: full_dataarray( var_name, attr_mgr.get_variable_attributes(f"hi_pset_{var_name}", check_schema=False), pset_coords, - fill_value=fill_val, ) - for var_name, fill_val in [ - ("background_rates", 0), - ("background_rates_uncertainty", 1), - ] + for var_name in ["background_rates", "background_rates_uncertainty"] } + # Get total exposure time + total_exposure_time = float(exposure_times.sum()) + + if total_exposure_time <= 0: + output_vars["background_rates"].values[:] = 0 + output_vars["background_rates_uncertainty"].values[:] = 0 + return output_vars + + # Compute background counts: shape (epoch, calibration_prod, background_index) + background_counts = _compute_background_counts( + pset_coords, background_config_df, l1b_de_dataset, goodtimes_ds + ) + + # Compute count rates: shape (epoch, calibration_prod, background_index) + count_rates = background_counts / total_exposure_time + + # Convert background config DataFrame to xarray Dataset + config_ds = background_config_df.to_xarray() + if not config_ds["calibration_prod"].equals(pset_coords["calibration_prod"]): + raise ValueError( + f"Calibration products in pset_coords and background_config_df " + f"do not match. pset_coords: {pset_coords['calibration_prod'].values}, " + f"background_config_df: {config_ds['calibration_prod'].values}" + ) + scaling_factors_da = config_ds["scaling_factor"] + uncertainties_da = config_ds["uncertainty"] + + # Compute scaled rates + scaled_rates = count_rates * scaling_factors_da + + # Compute uncertainties (Poisson + scaling factor, combined in quadrature) + poisson_unc = ( + np.sqrt(background_counts) / total_exposure_time + ) * scaling_factors_da + scaling_unc = count_rates * uncertainties_da + combined_unc = np.sqrt(poisson_unc**2 + scaling_unc**2) + + # Sum over background_index dimension to get final rates + total_rates = scaled_rates.sum(dim="background_index", skipna=True) + total_unc = np.sqrt((combined_unc**2).sum(dim="background_index", skipna=True)) + + # Broadcast to (epoch, esa_energy_step, calibration_prod, spin_angle_bin) + # Backgrounds are isotropic and independent of ESA step, so we + # broadcast across esa_energy_step and spin_angle_bin dimensions. + output_vars["background_rates"].values[:] = total_rates.values[ + :, np.newaxis, :, np.newaxis + ] + output_vars["background_rates_uncertainty"].values[:] = total_unc.values[ + :, np.newaxis, :, np.newaxis + ] + + return output_vars + def pset_exposure( pset_coords: dict[str, xr.DataArray], diff --git a/imap_processing/tests/hi/test_hi_l1c.py b/imap_processing/tests/hi/test_hi_l1c.py index de7891a46..8f7b2f2b8 100644 --- a/imap_processing/tests/hi/test_hi_l1c.py +++ b/imap_processing/tests/hi/test_hi_l1c.py @@ -33,10 +33,19 @@ def hi_goodtimes_dataset(hi_l1_test_data_path): @mock.patch("imap_processing.hi.hi_l1c.generate_pset_dataset") -def test_hi_l1c(mock_generate_pset_dataset, hi_test_cal_prod_config_path): +def test_hi_l1c( + mock_generate_pset_dataset, + hi_test_cal_prod_config_path, + hi_test_background_config_path, +): """Test coverage for hi_l1c function""" mock_generate_pset_dataset.return_value = xr.Dataset() - pset = hi_l1c.hi_l1c(xr.Dataset(), hi_test_cal_prod_config_path, xr.Dataset())[0] + pset = hi_l1c.hi_l1c( + xr.Dataset(), + hi_test_cal_prod_config_path, + xr.Dataset(), + hi_test_background_config_path, + )[0] # Empty attributes, global values get added in post-processing assert pset.attrs == {} @@ -47,6 +56,7 @@ def test_generate_pset_dataset( hi_l1b_de_dataset, hi_goodtimes_dataset, hi_test_cal_prod_config_path, + hi_test_background_config_path, use_fake_spin_data_for_time, use_fake_repoint_data_for_time, imap_ena_sim_metakernel, @@ -64,7 +74,10 @@ def test_generate_pset_dataset( goodtimes = hi_goodtimes_dataset l1c_dataset = hi_l1c.generate_pset_dataset( - l1b_dataset, hi_test_cal_prod_config_path, goodtimes + l1b_dataset, + hi_test_cal_prod_config_path, + goodtimes, + hi_test_background_config_path, ) assert l1c_dataset.epoch.data[0] == l1b_dataset.epoch.data[0].astype(np.int64) @@ -97,6 +110,7 @@ def test_generate_pset_dataset_uses_midpoint_time( mock_pset_exposure, mock_pset_backgrounds, hi_test_cal_prod_config_path, + hi_test_background_config_path, ): """Test that generate_pset_dataset uses midpoint ET for pset_geometry.""" # Create a mock L1B dataset @@ -129,12 +143,20 @@ def test_generate_pset_dataset_uses_midpoint_time( # Mock the return values for the sub-functions mock_pset_geometry.return_value = {} mock_pset_counts.return_value = {} - mock_pset_exposure.return_value = {} + # pset_exposure must return exposure_times for pset_backgrounds to use + mock_exposure_times = xr.DataArray( + np.ones((1, n_energy_steps, 3600), dtype=np.float32), + dims=["epoch", "esa_energy_step", "spin_angle_bin"], + ) + mock_pset_exposure.return_value = {"exposure_times": mock_exposure_times} mock_pset_backgrounds.return_value = {} # Call generate_pset_dataset _ = hi_l1c.generate_pset_dataset( - mock_l1b_dataset, hi_test_cal_prod_config_path, xr.Dataset() + mock_l1b_dataset, + hi_test_cal_prod_config_path, + xr.Dataset(), + hi_test_background_config_path, ) # Calculate expected midpoint ET @@ -240,6 +262,7 @@ def test_pset_counts( hi_l1b_de_dataset, hi_goodtimes_dataset, hi_test_cal_prod_config_path, + hi_test_background_config_path, ): """Test coverage for pset_counts function.""" cal_config_df = utils.CalibrationProductConfig.from_csv( @@ -264,6 +287,7 @@ def test_pset_counts_empty_l1b( hi_l1b_de_dataset, hi_goodtimes_dataset, hi_test_cal_prod_config_path, + hi_test_background_config_path, ): """Test coverage for pset_counts function when the input L1b contains no counts.""" # Make a copy and modify it - @@ -520,30 +544,245 @@ def mock_iter(de_ds, config_df, esa_energy_steps): assert counts_var["counts"].data[0, 0, 0, 1800] == 5 -def test_pset_backgrounds(): +@pytest.mark.external_test_data +def test_pset_backgrounds( + hi_test_background_config_path, + hi_test_cal_prod_config_path, + hi_l1b_de_dataset, + hi_goodtimes_dataset, + use_fake_spin_data_for_time, + use_fake_repoint_data_for_time, +): """Test coverage for pset_backgrounds function.""" - # Create some fake coordinates to use + # Setup required SPICE data + use_fake_spin_data_for_time(482372987.999) + l1b_met = hi_l1b_de_dataset["ccsds_met"].values[0] + seconds_per_day = 24 * 60 * 60 + use_fake_repoint_data_for_time( + np.asarray([l1b_met - 15 * 60, l1b_met + seconds_per_day]), + np.asarray([l1b_met, l1b_met + seconds_per_day + 1]), + ) + + # Load the background config + background_df = utils.BackgroundConfig.from_csv(hi_test_background_config_path) + + # Create empty pset dataset to get coordinates + cal_config_df = utils.CalibrationProductConfig.from_csv( + hi_test_cal_prod_config_path + ) + empty_pset = hi_l1c.empty_pset_dataset( + l1b_met, + hi_l1b_de_dataset.esa_energy_step, + cal_config_df.cal_prod_config.calibration_product_numbers, + HIAPID.H90_SCI_DE.sensor, + ) + + # Create exposure_times for the test + exposure_times_data = np.full( + ( + len(empty_pset.coords["epoch"]), + len(empty_pset.coords["esa_energy_step"]), + len(empty_pset.coords["spin_angle_bin"]), + ), + 1.0, + dtype=np.float32, + ) + exposure_times = xr.DataArray( + exposure_times_data, + dims=["epoch", "esa_energy_step", "spin_angle_bin"], + coords={ + "epoch": empty_pset.coords["epoch"], + "esa_energy_step": empty_pset.coords["esa_energy_step"], + "spin_angle_bin": empty_pset.coords["spin_angle_bin"], + }, + ) + + # Call pset_backgrounds with the new signature + backgrounds_vars = hi_l1c.pset_backgrounds( + empty_pset.coords, + background_df, + hi_l1b_de_dataset, + hi_goodtimes_dataset, + exposure_times, + ) + + assert "background_rates" in backgrounds_vars + assert backgrounds_vars["background_rates"].data.shape == ( + len(empty_pset.coords["epoch"]), + len(empty_pset.coords["esa_energy_step"]), + len(empty_pset.coords["calibration_prod"]), + len(empty_pset.coords["spin_angle_bin"]), + ) + + assert "background_rates_uncertainty" in backgrounds_vars + assert backgrounds_vars["background_rates_uncertainty"].data.shape == ( + len(empty_pset.coords["epoch"]), + len(empty_pset.coords["esa_energy_step"]), + len(empty_pset.coords["calibration_prod"]), + len(empty_pset.coords["spin_angle_bin"]), + ) + + +@mock.patch("imap_processing.hi.hi_l1c.good_time_and_phase_mask") +def test_compute_background_counts_missing_cal_prod_raises_error( + mock_good_time_and_phase_mask, + hi_test_background_config_path, +): + """Test _compute_background_counts raises ValueError with invalid bkgnd config.""" + # Mock good_time_and_phase_mask to return all True + mock_good_time_and_phase_mask.side_effect = lambda a, b, c: np.ones( + a.shape, dtype=bool + ) + # Load the background config (has cal prods 0 and 1) + background_df = utils.BackgroundConfig.from_csv(hi_test_background_config_path) + + # Create minimal pset_coords with a calibration product (999) that's + # NOT in the background config + missing_cal_prod = 999 + pset_coords = { + "epoch": xr.DataArray(np.array([0], dtype=np.int64), dims=["epoch"]), + "calibration_prod": xr.DataArray( + np.array([0, 1, missing_cal_prod], dtype=np.int32), + dims=["calibration_prod"], + ), + } + + hi_l1b_de_dataset = xr.Dataset( + { + "coincidence_type": xr.DataArray( + np.array([15], dtype=np.uint8), dims=["event_met"] + ), + "trigger_id": xr.DataArray( + np.array([0], dtype=np.float64), + dims=["event_met"], + attrs={"FILLVAL": 65535}, + ), + "nominal_bin": xr.DataArray( + np.array([0], dtype=np.uint8), dims=["event_met"] + ), + "tof_ab": xr.DataArray( + np.array([50], dtype=np.float32), dims=["event_met"] + ), + "tof_ac1": xr.DataArray( + np.array([50], dtype=np.float32), dims=["event_met"] + ), + "tof_bc1": xr.DataArray( + np.array([50], dtype=np.float32), dims=["event_met"] + ), + "tof_c1c2": xr.DataArray( + np.array([50], dtype=np.float32), dims=["event_met"] + ), + }, + coords={ + "epoch": xr.DataArray(np.array([0], dtype=np.int64), dims=["epoch"]), + "event_met": xr.DataArray( + np.array([0], dtype=np.float64), dims=["event_met"] + ), + }, + ) + + # Verify that calling _compute_background_counts raises ValueError + # with expected message + with pytest.raises( + ValueError, + match=f"Calibration product {missing_cal_prod} not found " + f"in background configuration", + ): + hi_l1c._compute_background_counts( + pset_coords, + background_df, + hi_l1b_de_dataset, + xr.Dataset(), + ) + + +@mock.patch("imap_processing.hi.hi_l1c._compute_background_counts") +def test_pset_backgrounds_cal_prod_mismatch_raises_error( + mock_compute_background_counts, +): + """Test pset_backgrounds raises ValueError when cal prods don't match. + + This tests the validation at lines 634-639 of hi_l1c.py that checks + if calibration products in pset_coords match those in background_config_df. + """ + # Create pset_coords with calibration products [0, 1] n_epoch = 1 - n_energy = 9 - n_cal_prod = 2 + n_energy = 2 n_spin_bins = 3600 pset_coords = { - "epoch": xr.DataArray(np.arange(n_epoch)), - "esa_energy_step": xr.DataArray(np.arange(n_energy) + 1), - "calibration_prod": xr.DataArray(np.arange(n_cal_prod)), - "spin_angle_bin": xr.DataArray(np.arange(n_spin_bins)), + "epoch": xr.DataArray(np.array([0], dtype=np.int64), dims=["epoch"]), + "esa_energy_step": xr.DataArray( + np.arange(n_energy) + 1, dims=["esa_energy_step"] + ), + "calibration_prod": xr.DataArray( + np.array([0, 1], dtype=np.int64), + dims=["calibration_prod"], + ), + "spin_angle_bin": xr.DataArray(np.arange(n_spin_bins), dims=["spin_angle_bin"]), } - backgrounds_vars = hi_l1c.pset_backgrounds(pset_coords) - assert "background_rates" in backgrounds_vars - np.testing.assert_array_equal( - backgrounds_vars["background_rates"].data, - np.zeros((n_epoch, n_energy, n_cal_prod, n_spin_bins)), + + # Create a background config DataFrame with DIFFERENT calibration products [5, 6] + # This simulates a mismatch between pset_coords and background_config_df + background_config_data = { + "coincidence_type_list": ["ABC1C2", "ABC1C2"], + "coincidence_type_values": [[15], [15]], + "tof_ab_low": [0, 0], + "tof_ab_high": [100, 100], + "tof_ac1_low": [0, 0], + "tof_ac1_high": [100, 100], + "tof_bc1_low": [0, 0], + "tof_bc1_high": [100, 100], + "tof_c1c2_low": [0, 0], + "tof_c1c2_high": [100, 100], + "scaling_factor": [1.0, 1.0], + "uncertainty": [0.1, 0.1], + } + # Use calibration products [5, 6] which don't match pset_coords [0, 1] + mismatched_cal_prods = [5, 6] + background_indices = [0, 0] + multi_index = pd.MultiIndex.from_arrays( + [mismatched_cal_prods, background_indices], + names=["calibration_prod", "background_index"], ) - assert "background_rates_uncertainty" in backgrounds_vars - np.testing.assert_array_equal( - backgrounds_vars["background_rates_uncertainty"].data, - np.ones((n_epoch, n_energy, n_cal_prod, n_spin_bins)), + background_df = pd.DataFrame(background_config_data, index=multi_index) + + # Create mock exposure_times + exposure_times = xr.DataArray( + np.ones((n_epoch, n_energy, n_spin_bins), dtype=np.float32), + dims=["epoch", "esa_energy_step", "spin_angle_bin"], + ) + + # Mock _compute_background_counts to return a DataArray with the mismatched + # calibration products (simulating what would happen if the earlier check + # didn't catch the mismatch) + mock_background_counts = xr.DataArray( + np.zeros((n_epoch, len(mismatched_cal_prods), 1)), + dims=["epoch", "calibration_prod", "background_index"], + coords={ + "epoch": pset_coords["epoch"], + "calibration_prod": mismatched_cal_prods, + "background_index": [0], + }, ) + mock_compute_background_counts.return_value = mock_background_counts + + # Create minimal l1b dataset and goodtimes (not used due to mock) + l1b_de_dataset = xr.Dataset() + goodtimes_ds = xr.Dataset() + + # Verify that pset_backgrounds raises ValueError with expected message + with pytest.raises( + ValueError, + match="Calibration products in pset_coords and " + "background_config_df do not match", + ): + hi_l1c.pset_backgrounds( + pset_coords, + background_df, + l1b_de_dataset, + goodtimes_ds, + exposure_times, + ) @mock.patch("imap_processing.hi.hi_l1c.good_time_and_phase_mask") diff --git a/imap_processing/tests/test_cli.py b/imap_processing/tests/test_cli.py index 784af41b5..9535307da 100644 --- a/imap_processing/tests/test_cli.py +++ b/imap_processing/tests/test_cli.py @@ -305,7 +305,10 @@ def test_post_processing_returns_empty_list_if_invoked_with_no_data( "imap_hi_l1b_45sensor-de_20250415_v001.cdf", "imap_hi_l1b_45sensor-goodtimes_20250415_v001.cdf", ], - ["imap_hi_calibration-prod-config_20240101_v001.csv"], + [ + "imap_hi_45sensor-cal-prod_20240101_v001.csv", + "imap_hi_45sensor-backgrounds_20240101_v001.csv", + ], 1, ), (