ULTRA l1b fix repoint filtering (#2380)

* fix repoint filtering

Author: lacoak21

imap_processing/quality_flags.py

@@ -45,6 +45,7 @@ class ImapDEOutliersUltraFlags(FlagNameMixin):
     PHCORR = 2**1  # bit 1
     COINPH = 2**2  # bit 2 # Event validity
     INVALID_ENERGY = 2**3  # bit 3
+    DURINGREPOINT = 2**4  # bit 4 # event during a repointing
 
 
 class ImapHkUltraFlags(FlagNameMixin):
@@ -65,7 +66,6 @@ class ImapAttitudeUltraFlags(FlagNameMixin):
     AUXMISMATCH = 2**1  # bit 1 # aux packet does not match Universal Spin Table
     SPINPHASE = 2**2  # bit 2 # spin phase flagged by Universal Spin Table
     SPINPERIOD = 2**3  # bit 3 # spin period flagged by Universal Spin Table
-    DURINGREPOINT = 2**4  # bit 4 # spin during a repointing
 
 
 class ImapRatesUltraFlags(FlagNameMixin):

imap_processing/tests/ultra/unit/test_ultra_l1b.py

@@ -6,6 +6,8 @@
 
 from imap_processing import imap_module_directory
 from imap_processing.cdf.utils import load_cdf, write_cdf
+from imap_processing.quality_flags import ImapDEOutliersUltraFlags
+from imap_processing.ultra.l1b.de import FILLVAL_FLOAT32
 from imap_processing.ultra.l1b.ultra_l1b import ultra_l1b
 from imap_processing.ultra.utils.ultra_l1_utils import create_dataset
 
@@ -71,6 +73,29 @@ def mock_data_l1b_extendedspin_dict():
     return data_dict
 
 
+@pytest.fixture
+def mock_get_annotated_particle_velocity():
+    """
+    Mock behavior of get_annotated_particle_velocity.
+
+    Returns NaN-filled arrays matching the expected output shape.
+    """
+
+    def side_effect_func(event_times, position, ultra_frame, dps_frame, sc_frame):
+        num_events = event_times.size
+        return (
+            np.full((num_events, 3), np.nan),  # sc_velocity
+            np.full((num_events, 3), np.nan),  # sc_dps_velocity
+            np.full((num_events, 3), np.nan),  # helio_velocity
+        )
+
+    with mock.patch(
+        "imap_processing.ultra.l1b.de.get_annotated_particle_velocity"
+    ) as mocked_func:
+        mocked_func.side_effect = side_effect_func
+        yield mocked_func
+
+
 def test_create_extendedspin_dataset(mock_data_l1b_extendedspin_dict):
     """Tests that dataset is created as expected."""
     dataset = create_dataset(
@@ -101,13 +126,12 @@ def test_create_de_dataset(mock_data_l1b_de_dict):
 
 
 @pytest.mark.external_test_data
-@mock.patch("imap_processing.ultra.l1b.de.get_annotated_particle_velocity")
 def test_cdf_de(
-    mock_get_annotated_particle_velocity,
     de_dataset,
     use_fake_spin_data_for_time,
     ancillary_files,
     use_fake_repoint_data_for_time,
+    mock_get_annotated_particle_velocity,
 ):
     """Tests that CDF file is created and contains same attributes as xarray."""
 
@@ -118,22 +142,6 @@ def test_cdf_de(
     use_fake_spin_data_for_time(511000000, 511000000 + 86400 * 5)
     use_fake_repoint_data_for_time(np.arange(511000000, 511000000 + 86400 * 5, 86400))
 
-    # Mock get_annotated_particle_velocity to avoid needing kernels
-    def side_effect_func(event_times, position, ultra_frame, dps_frame, sc_frame):
-        """
-        Mock behavior of get_annotated_particle_velocity.
-
-        Returns NaN-filled arrays matching the expected output shape.
-        """
-        num_events = event_times.size
-        return (
-            np.full((num_events, 3), np.nan),  # sc_velocity
-            np.full((num_events, 3), np.nan),  # sc_dps_velocity
-            np.full((num_events, 3), np.nan),  # helio_velocity
-        )
-
-    mock_get_annotated_particle_velocity.side_effect = side_effect_func
-
     l1b_de_dataset = ultra_l1b(data_dict, ancillary_files)
 
     assert (
@@ -151,6 +159,31 @@ def side_effect_func(event_times, position, ultra_frame, dps_frame, sc_frame):
     )
 
 
+@pytest.mark.external_test_data
+def test_cdf_de_flags(
+    mock_get_annotated_particle_velocity,
+    de_dataset,
+    use_fake_spin_data_for_time,
+    ancillary_files,
+    use_fake_repoint_data_for_time,
+):
+    """Tests that the de code flags events not in a repointing."""
+    data_dict = {}
+    de_dataset.attrs["Repointing"] = "repoint00000"
+    data_dict[de_dataset.attrs["Logical_source"]] = de_dataset
+    # Create a spin table that cover spin 0-141
+    use_fake_spin_data_for_time(511000000, 511000000 + 86400 * 5)
+    # Use repoint data that will NOT cover the event times to test flag setting
+    use_fake_repoint_data_for_time(np.arange(0, +86400 * 5, 86400))
+
+    l1b_de_dataset = ultra_l1b(data_dict, ancillary_files)
+    # All valid events should be flagged as DURINGREPOINT since the repoint data does
+    # not cover any of the event times
+    valid_events = l1b_de_dataset[0]["event_times"] != FILLVAL_FLOAT32
+    flags = l1b_de_dataset[0]["quality_outliers"].values[valid_events]
+    assert np.all((flags & ImapDEOutliersUltraFlags.DURINGREPOINT.value) != 0)
+
+
 @pytest.mark.external_test_data
 def test_ultra_l1b_extendedspin(
     use_fake_spin_data_for_time, faux_aux_dataset, rates_dataset

imap_processing/ultra/l1b/de.py

@@ -5,7 +5,6 @@
 
 from imap_processing.cdf.utils import parse_filename_like
 from imap_processing.quality_flags import (
-    ImapAttitudeUltraFlags,
     ImapDEOutliersUltraFlags,
     ImapDEScatteringUltraFlags,
 )
@@ -326,10 +325,10 @@ def calculate_de(
         in_pointing = calculate_events_in_pointing(
             repoint_id, event_times[valid_events]
         )
+        events_to_flag = np.zeros(len(quality_flags), dtype=bool)
+        events_to_flag[valid_events] = ~in_pointing
         # Update quality flags for valid events that are not in the pointing
-        quality_flags[valid_events][~in_pointing] |= (
-            ImapAttitudeUltraFlags.DURINGREPOINT.value
-        )
+        quality_flags[events_to_flag] |= ImapDEOutliersUltraFlags.DURINGREPOINT.value
         # Update valid_events to only include times within a pointing
         valid_events[valid_events] &= in_pointing
 

imap_processing/ultra/l1b/quality_flag_filters.py

@@ -16,7 +16,10 @@
 }
 
 DE_QUALITY_FLAG_FILTERS: dict[str, list[FlagNameMixin]] = {
-    "quality_outliers": [ImapDEOutliersUltraFlags.FOV],
+    "quality_outliers": [
+        ImapDEOutliersUltraFlags.FOV,
+        ImapDEOutliersUltraFlags.DURINGREPOINT,
+    ],
     "quality_scattering": [
         ImapDEScatteringUltraFlags.ABOVE_THRESHOLD,
     ],

imap_processing/ultra/l1c/ultra_l1c_pset_bins.py

@@ -292,7 +292,7 @@ def get_deadtime_ratios_by_spin_phase(
     numpy.ndarray
         Nominal deadtime ratios at every spin phase step.
     """
-    if sectored_rates is None:
+    if sectored_rates is None or sectored_rates.epoch.size == 0:
         logger.warning(
             "No sector mode data found in the parameters dataset. Using "
             "static dead time ratios from an ancillary file."
@@ -455,6 +455,12 @@ def get_spacecraft_exposure_times(
     nominal_deadtime_ratios : np.ndarray
         Deadtime ratios at each spin phase step (1ms res).
     """
+    # filter rates dataset to only include data during a pointing
+    rates_time = ttj2000ns_to_met(rates_dataset.epoch.data)
+    pointing_mask = (rates_time >= pointing_range_met[0]) & (
+        rates_time <= pointing_range_met[1]
+    )
+    rates_dataset.isel(epoch=pointing_mask)
     sectored_rates = get_sectored_rates(rates_dataset, params_dataset)
     # Get the number of steps used in the spun pointing lookup tables
     spin_steps = len(pixels_below_scattering)