Skip to content

Ultra l1a - added cdf for apid 882 #1854

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
laspsandoval merged 5 commits into from
Jun 27, 2025
Merged

Ultra l1a - added cdf for apid 882 #1854

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e837dc0
added l1a for apid 882
laspsandoval Jun 26, 2025
0d69087
minor fix
laspsandoval Jun 26, 2025
2f3e87e
fix test
laspsandoval Jun 26, 2025
1510347
minor fix
laspsandoval Jun 27, 2025
a9346eb
apid 882 test update
laspsandoval Jun 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,18 @@ imap_ultra_l1a_90sensor-rates:
Logical_source: imap_ultra_l1a_90sensor-rates
Logical_source_description: IMAP-Ultra Instrument Level-1A Rates Data.

imap_ultra_l1a_45sensor-energy-rates:
<<: *instrument_base
Data_type: L1A_Energy_Rates>Level-1A Energy Rates
Logical_source: imap_ultra_l1a_45sensor-energy-rates
Logical_source_description: IMAP-Ultra Instrument Level-1A Energy Rates Data.

imap_ultra_l1a_90sensor-energy-rates:
<<: *instrument_base
Data_type: L1A_Energy_Rates>Level-1A Energy Rates
Logical_source: imap_ultra_l1a_90sensor-energy-rates
Logical_source_description: IMAP-Ultra Instrument Level-1A Energy Rates Data.

imap_ultra_l1a_45sensor-de:
<<: *instrument_base
Data_type: L1A_DE>Level-1A Direct Event
Expand Down
72 changes: 72 additions & 0 deletions imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,78 @@ sid:
UNITS: " "
DEPEND_0: epoch

ssd0_energy_led:
<<: *default_uint32
CATDESC: SSD0 Energy LED.
FIELDNAM: ssd0_energy_led
LABLAXIS: ssd0 energy led
UNITS: " "
DEPEND_0: epoch

ssd1_energy_led:
<<: *default_uint32
CATDESC: SSD1 Energy LED.
FIELDNAM: ssd1_energy_led
LABLAXIS: ssd1 energy led
UNITS: " "
DEPEND_0: epoch

ssd2_energy_led:
<<: *default_uint32
CATDESC: SSD2 Energy LED.
FIELDNAM: ssd2_energy_led
LABLAXIS: ssd2 energy led
UNITS: " "
DEPEND_0: epoch

ssd3_energy_led:
<<: *default_uint32
CATDESC: SSD3 Energy LED.
FIELDNAM: ssd3_energy_led
LABLAXIS: ssd3 energy led
UNITS: " "
DEPEND_0: epoch

ssd4_energy_led:
<<: *default_uint32
CATDESC: SSD4 Energy LED.
FIELDNAM: ssd4_energy_led
LABLAXIS: ssd4 energy led
UNITS: " "
DEPEND_0: epoch

ssd5_energy_led:
<<: *default_uint32
CATDESC: SSD5 Energy LED.
FIELDNAM: ssd5_energy_led
LABLAXIS: ssd5 energy led
UNITS: " "
DEPEND_0: epoch

ssd6_energy_led:
<<: *default_uint32
CATDESC: SSD6 Energy LED.
FIELDNAM: ssd6_energy_led
LABLAXIS: ssd6 energy led
UNITS: " "
DEPEND_0: epoch

ssd7_energy_led:
<<: *default_uint32
CATDESC: SSD7 Energy LED.
FIELDNAM: ssd7_energy_led
LABLAXIS: ssd7 energy led
UNITS: " "
DEPEND_0: epoch

processed_events:
<<: *default_uint32
CATDESC: Processed Events.
FIELDNAM: processed_events
LABLAXIS: processed events
UNITS: " "
DEPEND_0: epoch

spin:
<<: *default_uint32
CATDESC: Spin number.
Expand Down
26 changes: 26 additions & 0 deletions imap_processing/tests/ultra/unit/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@

from imap_processing import imap_module_directory
from imap_processing.ultra.l0.decom_ultra import (
process_ultra_energy_rates,
process_ultra_events,
process_ultra_rates,
process_ultra_tof,
)
from imap_processing.ultra.l0.ultra_utils import (
ULTRA_AUX,
ULTRA_ENERGY_RATES,
ULTRA_EVENTS,
ULTRA_RATES,
ULTRA_TOF,
Expand Down Expand Up @@ -86,6 +88,19 @@ def ccsds_path_tof():
)


@pytest.fixture
def ccsds_path_functional():
"""Returns the ccsds directory."""
return (
imap_module_directory
/ "tests"
/ "ultra"
/ "data"
/ "l0"
/ "FM45_UltraFM45_Functional_2024-01-22T0105_20240122T010548.CCSDS"
)


@pytest.fixture
def xtce_path():
"""Returns the xtce image rates directory."""
Expand All @@ -107,6 +122,16 @@ def rates_test_path():
return imap_module_directory / "tests" / "ultra" / "data" / "l0" / filename


@pytest.fixture
def energy_rates_test_path():
"""Returns the xtce image rates test data directory."""
filename = (
"ultra45_raw_sc_ultranrgrates_FM45_UltraFM45_Functional"
"_2024-01-22T0105_20240122T010548.csv"
)
return imap_module_directory / "tests" / "ultra" / "data" / "l0" / filename


@pytest.fixture
def aux_test_path():
"""Returns the xtce auxiliary test data directory."""
Expand Down Expand Up @@ -149,6 +174,7 @@ def decom_test_data(request, xtce_path):
ULTRA_TOF.apid[0]: process_ultra_tof,
ULTRA_EVENTS.apid[0]: process_ultra_events,
ULTRA_RATES.apid[0]: process_ultra_rates,
ULTRA_ENERGY_RATES.apid[0]: process_ultra_energy_rates,
ULTRA_TOF.apid[1]: process_ultra_tof,
ULTRA_EVENTS.apid[1]: process_ultra_events,
ULTRA_RATES.apid[1]: process_ultra_rates,
Expand Down
50 changes: 50 additions & 0 deletions imap_processing/tests/ultra/unit/test_decom_apid_882.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import json

import numpy as np
import pandas as pd
import pytest

from imap_processing.ultra.l0.ultra_utils import ENERGY_RATES_KEYS, ULTRA_ENERGY_RATES


@pytest.mark.parametrize(
"decom_test_data",
[
pytest.param(
{
"apid": ULTRA_ENERGY_RATES.apid[0],
"filename": "FM45_UltraFM45_Functional_"
"2024-01-22T0105_20240122T010548.CCSDS",
}
)
],
indirect=True,
)
def test_image_rate_decom(decom_test_data, energy_rates_test_path):
"""This function reads validation data and checks that decom data
matches validation data for image rate packet"""
decom_ultra = decom_test_data

df = pd.read_csv(energy_rates_test_path, index_col="MET")
total_packets = 315

np.testing.assert_array_equal(df.SID, decom_ultra["sid"])
np.testing.assert_array_equal(df.Spin, decom_ultra["spin"])
np.testing.assert_array_equal(df.AbortFlag, decom_ultra["abortflag"])
np.testing.assert_array_equal(df.StartDelay, decom_ultra["startdelay"])

# Spot-check first packet
t0 = decom_ultra["shcoarse"][0]
expected_arr0 = json.loads(df.loc[int(t0)].Counts)
arr = []
for name in ENERGY_RATES_KEYS:
arr.append(decom_ultra[name][0])
assert expected_arr0 == arr

# Spot-check last packet
tn = decom_ultra["shcoarse"][total_packets - 1]
expected_arrn = json.loads(df.loc[int(tn)].Counts)
arr = []
for name in ENERGY_RATES_KEYS:
arr.append(decom_ultra[name][total_packets - 1])
assert expected_arrn == arr
15 changes: 15 additions & 0 deletions imap_processing/tests/ultra/unit/test_ultra_l1a.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from imap_processing.ultra.l0.decom_ultra import get_event_id
from imap_processing.ultra.l0.ultra_utils import (
ULTRA_AUX,
ULTRA_ENERGY_RATES,
ULTRA_EVENTS,
ULTRA_RATES,
ULTRA_TOF,
Expand Down Expand Up @@ -96,6 +97,20 @@ def test_cdf_rates(ccsds_path_theta_0):
)


def test_cdf_energy_rates(ccsds_path_functional):
"""Tests that CDF file can be created."""
test_data = ultra_l1a(ccsds_path_functional, apid_input=ULTRA_ENERGY_RATES.apid[0])
test_data[0].attrs["Data_version"] = "999"
test_data[0].attrs["Repointing"] = "repoint99999"
test_data_path = write_cdf(test_data[0], istp=True)

assert test_data_path.exists()
assert (
test_data_path.name
== "imap_ultra_l1a_45sensor-energy-rates_20240122-repoint99999_v999.cdf"
)


def test_cdf_tof(ccsds_path_theta_0):
"""Tests that CDF file can be created."""
test_data = ultra_l1a(ccsds_path_theta_0, apid_input=ULTRA_TOF.apid[0])
Expand Down
37 changes: 37 additions & 0 deletions imap_processing/ultra/l0/decom_ultra.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
read_image_raw_events_binary,
)
from imap_processing.ultra.l0.ultra_utils import (
ENERGY_RATES_KEYS,
EVENT_FIELD_RANGES,
RATES_KEYS,
ULTRA_ENERGY_RATES,
ULTRA_RATES,
ULTRA_TOF,
)
Expand Down Expand Up @@ -242,3 +244,38 @@ def process_ultra_rates(ds: xr.Dataset) -> xr.Dataset:
ds[key] = xr.DataArray(np.array(values), dims=["epoch"])

return ds


def process_ultra_energy_rates(ds: xr.Dataset) -> xr.Dataset:
Copy link
Copy Markdown
Collaborator

@greglucas greglucas Jun 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is very close to the above function. If you are going to be doing this on other rate-type packets too it might be good to abstract the boilerplate stuff out and pass in the apid/enum to do the lookup of the packet-specific things within a common function instead of needing to duplicate all the looping/logic.

def process_rates_l0(ds, packet_enum):
    # Lookup from whatever enum you passed in
    packet_enum.width, ...

Copy link
Copy Markdown
Contributor Author

@laspsandoval laspsandoval Jun 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. I know. This is the last rates packet, though.

"""
Unpack and decode Ultra ENERGY RATES packets.

Parameters
----------
ds : xarray.Dataset
Energy rates dataset.

Returns
-------
dataset : xarray.Dataset
Dataset containing the decoded and decompressed data.
"""
decom_data = defaultdict(list)

for rate in ds["ratedata"]:
raw_binary_string = convert_to_binary_string(rate.item())
decompressed_data = decompress_binary(
raw_binary_string,
cast(int, ULTRA_ENERGY_RATES.width),
cast(int, ULTRA_ENERGY_RATES.block),
cast(int, ULTRA_ENERGY_RATES.len_array),
cast(int, ULTRA_ENERGY_RATES.mantissa_bit_length),
)

for index in range(cast(int, ULTRA_ENERGY_RATES.len_array)):
decom_data[ENERGY_RATES_KEYS[index]].append(decompressed_data[index])

for key, values in decom_data.items():
ds[key] = xr.DataArray(np.array(values), dims=["epoch"])

return ds
38 changes: 38 additions & 0 deletions imap_processing/ultra/l0/ultra_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,18 @@ class PacketProperties(NamedTuple):
len_array=48,
mantissa_bit_length=12,
)
ULTRA_ENERGY_RATES = PacketProperties(
apid=[882, 946],
logical_source=[
"imap_ultra_l1a_45sensor-energy-rates",
"imap_ultra_l1a_90sensor-energy-rates",
],
addition_to_logical_desc="Image Rates",
width=5,
block=16,
len_array=11,
mantissa_bit_length=12,
)
ULTRA_TOF = PacketProperties(
apid=[883, 947],
logical_source=[
Expand Down Expand Up @@ -324,6 +336,32 @@ class PacketProperties(NamedTuple):
]


ENERGY_RATES_KEYS = [
# SSD0 Energy LED
"ssd0_energy_led",
# SSD1 Energy LED
"ssd1_energy_led",
# SSD2 Energy LED
"ssd2_energy_led",
# SSD3 Energy LED
"ssd3_energy_led",
# SSD4 Energy LED
"ssd4_energy_led",
# SSD5 Energy LED
"ssd5_energy_led",
# SSD6 Energy LED
"ssd6_energy_led",
# SSD7 Energy LED
"ssd7_energy_led",
# Event Active Time
"event_active_time",
# FIFO Valid Events
"fifo_valid_events",
# Processed Events
"processed_events",
]


def parse_event(event_binary: str) -> dict:
"""
Parse a binary string representing a single event.
Expand Down
8 changes: 8 additions & 0 deletions imap_processing/ultra/l1a/ultra_l1a.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@
from imap_processing import imap_module_directory
from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes
from imap_processing.ultra.l0.decom_ultra import (
process_ultra_energy_rates,
process_ultra_events,
process_ultra_rates,
process_ultra_tof,
)
from imap_processing.ultra.l0.ultra_utils import (
ULTRA_AUX,
ULTRA_CMD_TEXT,
ULTRA_ENERGY_RATES,
ULTRA_EVENTS,
ULTRA_HK,
ULTRA_RATES,
Expand Down Expand Up @@ -75,6 +77,12 @@ def ultra_l1a(packet_file: str, apid_input: Optional[int] = None) -> list[xr.Dat
decom_ultra_dataset = process_ultra_rates(datasets_by_apid[apid])
decom_ultra_dataset = decom_ultra_dataset.drop_vars("fastdata_00")
gattr_key = ULTRA_RATES.logical_source[ULTRA_RATES.apid.index(apid)]
elif apid in ULTRA_ENERGY_RATES.apid:
decom_ultra_dataset = process_ultra_energy_rates(datasets_by_apid[apid])
decom_ultra_dataset = decom_ultra_dataset.drop_vars("ratedata")
gattr_key = ULTRA_ENERGY_RATES.logical_source[
ULTRA_ENERGY_RATES.apid.index(apid)
]
elif apid in ULTRA_EVENTS.apid:
decom_ultra_dataset = process_ultra_events(datasets_by_apid[apid])
gattr_key = ULTRA_EVENTS.logical_source[ULTRA_EVENTS.apid.index(apid)]
Expand Down
Loading