1729 add pointing frame kernel job to clipy (IMAP-Science-Operations-Center#1776)

* Add high level function for producing pointing attitude kernel

* Add pointing kernel job to cli Spacecraft.do_processing
Create custom Spacecraft.post_processing to handle list[Path] returned by generate_poiting_attitude_kernel()

* bump imap-data-access version

* add poetry.lock updates

* Allow for Paths to be passed in to default post_processing method

* Fix bug found in integration testing

* Improve pointing kernel name generation

Author: tmplummer

imap_processing/__init__.py

@@ -27,7 +27,7 @@
     "idex": ["l1a", "l1b", "l2a", "l2b", "l2c"],
     "lo": ["l1a", "l1b", "l1c", "l2"],
     "mag": ["l1a", "l1b", "l1c", "l1d", "l2"],
-    "spacecraft": ["l1a"],
+    "spacecraft": ["l1a", "spice"],
     "swapi": ["l1", "l2", "l3a", "l3b"],
     "swe": ["l1a", "l1b", "l2"],
     "ultra": ["l1a", "l1b", "l1c", "l2"],

imap_processing/cli.py

@@ -69,6 +69,7 @@
 from imap_processing.mag.l1c.mag_l1c import mag_l1c
 from imap_processing.mag.l2.mag_l2 import mag_l2
 from imap_processing.spacecraft import quaternions
+from imap_processing.spice import pointing_frame
 from imap_processing.swapi.l1.swapi_l1 import swapi_l1
 from imap_processing.swapi.l2.swapi_l2 import swapi_l2
 from imap_processing.swapi.swapi_utils import read_swapi_lut_table
@@ -437,6 +438,7 @@ def process(self) -> None:
         2. Do the data processing. The result of this step will usually be a list
         of new products (files).
         3. Post-processing actions such as uploading files to the IMAP SDC.
+        4. Final cleanup actions.
         """
         logger.info(f"IMAP Processing Version: {imap_processing._version.__version__}")
         logger.info(f"Processing {self.__class__.__name__} level {self.data_level}")
@@ -446,6 +448,7 @@ def process(self) -> None:
         products = self.do_processing(dependencies)
         logger.info("Beginning postprocessing (uploading data products)")
         self.post_processing(products, dependencies)
+        self.cleanup()
         logger.info("Processing complete")
 
     def pre_processing(self) -> ProcessingInputCollection:
@@ -497,13 +500,23 @@ def do_processing(
         raise NotImplementedError
 
     def post_processing(
-        self, datasets: list[xr.Dataset], dependencies: ProcessingInputCollection
+        self,
+        processed_data: list[xr.Dataset | Path],
+        dependencies: ProcessingInputCollection,
     ) -> None:
         """
         Complete post-processing.
 
-        Default post-processing consists of writing the datasets to local storage
-        and then uploading those newly generated products to the IMAP SDC.
+        Default post-processing consists of the following:
+        For each xarray.Dataset:
+            1. Set `Data_version` global attribute.
+            2. Set `Repointing` global attribute for appropriate products.
+            3. Set `Start_date` global attribute.
+            4. Set `Parents` global attribute.
+            5. Write the xarray.Dataset to a local CDF file.
+        The resulting paths to CDF files as well as any Path included in the
+        `processed_data` input are then uploaded to the IMAP SDC.
+
         Child classes can override this method to customize the
         post-processing actions.
 
@@ -513,12 +526,13 @@ def post_processing(
 
         Parameters
         ----------
-        datasets : list[xarray.Dataset]
-            A list of datasets (products) produced by do_processing method.
+        processed_data : list[xarray.Dataset | Path]
+            A list of datasets (products) and paths produced by the do_processing
+            method.
         dependencies : ProcessingInputCollection
             Object containing dependencies to process.
         """
-        if len(datasets) == 0:
+        if len(processed_data) == 0:
             logger.info("No products to write to CDF file.")
             return
 
@@ -541,16 +555,23 @@ def post_processing(
         # If it is start_date, skip repointing in the output filename.
 
         products = []
-        for ds in datasets:
-            ds.attrs["Data_version"] = self.version
-            if self.repointing is not None:
-                ds.attrs["Repointing"] = self.repointing
-            ds.attrs["Start_date"] = self.start_date
-            ds.attrs["Parents"] = parent_files
-            products.append(write_cdf(ds))
+        for ds in processed_data:
+            if isinstance(ds, xr.Dataset):
+                ds.attrs["Data_version"] = self.version
+                if self.repointing is not None:
+                    ds.attrs["Repointing"] = self.repointing
+                ds.attrs["Start_date"] = self.start_date
+                ds.attrs["Parents"] = parent_files
+                products.append(write_cdf(ds))
+            else:
+                # A path to a product that was already written out
+                products.append(ds)
 
         self.upload_products(products)
 
+    @final
+    def cleanup(self) -> None:
+        """Cleanup from processing."""
         logger.info("Clearing furnished SPICE kernels")
         spiceypy.kclear()
 
@@ -1060,7 +1081,7 @@ class Spacecraft(ProcessInstrument):
 
     def do_processing(
         self, dependencies: ProcessingInputCollection
-    ) -> list[xr.Dataset]:
+    ) -> list[xr.Dataset | Path]:
         """
         Perform Spacecraft specific processing.
 
@@ -1071,26 +1092,41 @@ def do_processing(
 
         Returns
         -------
-        datasets : xr.Dataset
-            Xr.Dataset of products.
+        datasets : list[xarray.Dataset | Path]
+            The list of processed products.
         """
         print(f"Processing Spacecraft {self.data_level}")
 
-        if self.data_level != "l1a":
+        if self.data_level == "l1a":
+            # File path is expected output file path
+            input_files = dependencies.get_file_paths(source="spacecraft")
+            if len(input_files) > 1:
+                raise ValueError(
+                    f"Unexpected dependencies found for Spacecraft L1A: "
+                    f"{input_files}. Expected only one dependency."
+                )
+            datasets = list(quaternions.process_quaternions(input_files[0]))
+            return datasets
+        elif self.data_level == "spice":
+            spice_inputs = dependencies.get_file_paths(
+                data_type=SPICESource.SPICE.value
+            )
+            ah_paths = [path for path in spice_inputs if ".ah" in path.suffixes]
+            if len(ah_paths) != 1:
+                raise ValueError(
+                    f"Unexpected spice dependencies found for Spacecraft "
+                    f"pointing_kernel: {ah_paths}. Expected exactly one "
+                    f"attitude history file."
+                )
+            pointing_kernel_paths = pointing_frame.generate_pointing_attitude_kernel(
+                ah_paths[0]
+            )
+            return pointing_kernel_paths
+        else:
             raise NotImplementedError(
                 f"Spacecraft processing not implemented for level {self.data_level}"
             )
 
-        # File path is expected output file path
-        input_files = dependencies.get_file_paths(source="spacecraft")
-        if len(input_files) > 1:
-            raise ValueError(
-                f"Unexpected dependencies found for Spacecraft L1A: "
-                f"{input_files}. Expected only one dependency."
-            )
-        datasets = list(quaternions.process_quaternions(input_files[0]))
-        return datasets
-
 
 class Swapi(ProcessInstrument):
     """Process SWAPI."""

imap_processing/spice/pointing_frame.py

@@ -9,6 +9,7 @@
 
 import numpy as np
 import spiceypy
+from imap_data_access import SPICEFilePath
 from numpy.typing import NDArray
 
 from imap_processing.spice.geometry import SpiceFrame
@@ -35,6 +36,39 @@
 )
 
 
+def generate_pointing_attitude_kernel(imap_attitude_ck: Path) -> list[Path]:
+    """
+    Generate pointing attitude kernel from input IMAP CK kernel.
+
+    Parameters
+    ----------
+    imap_attitude_ck : Path
+        Location of the IMAP attitude kernel from which to generate pointing
+        attitude.
+
+    Returns
+    -------
+    pointing_kernel_path : list[Path]
+        Location of the new pointing kernels.
+    """
+    pointing_segments = calculate_pointing_attitude_segments(imap_attitude_ck)
+    # get the start and end yyyy_doy strings
+    # TODO: For now just use the input CK start/end dates. It is possible that
+    #    the end date is incorrect b/c the repoint table determines the last
+    #    segment in the pointing kernel.
+    spice_file = SPICEFilePath(imap_attitude_ck.name)
+    pointing_kernel_path = (
+        imap_attitude_ck.parent / f"imap_dps_"
+        f"{spice_file.spice_metadata['start_date'].strftime('%Y_%j')}_"
+        f"{spice_file.spice_metadata['end_date'].strftime('%Y_%j')}_"
+        f"{spice_file.spice_metadata['version']}.ah.bc"
+    )
+    write_pointing_frame_ck(
+        pointing_kernel_path, pointing_segments, imap_attitude_ck.name
+    )
+    return [pointing_kernel_path]
+
+
 @contextmanager
 def open_spice_ck_file(pointing_frame_path: Path) -> Generator[int, None, None]:
     """
@@ -218,7 +252,7 @@ def calculate_pointing_attitude_segments(
             "repoint_id"
         ]
         pointing_start_et = repoint_df.iloc[i_pointing]["repoint_end_et"]
-        pointing_end_et = repoint_df["repoint_start_et"][i_pointing + 1]
+        pointing_end_et = repoint_df.iloc[i_pointing + 1]["repoint_start_et"]
         logger.debug(
             f"Calculating pointing attitude for pointing "
             f"{pointing_segments[i_pointing]['pointing_id']} with time "

imap_processing/tests/spice/test_pointing_frame.py

@@ -1,8 +1,12 @@
 """Test coverage for imap_processing.spice.repoint.py"""
 
+from pathlib import Path
+from unittest import mock
+
 import numpy as np
 import pytest
 import spiceypy
+from imap_data_access import SPICEInput
 
 from imap_processing.spice import IMAP_SC_ID
 from imap_processing.spice.geometry import SpiceFrame
@@ -11,6 +15,7 @@
     _average_quaternions,
     _create_rotation_matrix,
     calculate_pointing_attitude_segments,
+    generate_pointing_attitude_kernel,
     write_pointing_frame_ck,
 )
 from imap_processing.spice.time import TICK_DURATION
@@ -50,6 +55,27 @@ def et_times(pointing_frame_kernels):
     return et_times
 
 
+@mock.patch(
+    "imap_processing.spice.pointing_frame.write_pointing_frame_ck", autospec=True
+)
+@mock.patch(
+    "imap_processing.spice.pointing_frame.calculate_pointing_attitude_segments",
+    autospec=True,
+    return_value=None,
+)
+def test_generate_pointing_attitude_kernel(mock_gen_attitude_segments, mock_write_ck):
+    """Test coverage for generate_pointing_attitude_kernel function."""
+    start_date = "2024_111"
+    end_date = "2024_222"
+    version = "02"
+    ck_path = Path(f"/bogus/file/path/imap_{start_date}_{end_date}_{version}.ah.bc")
+    pointing_ck_path = generate_pointing_attitude_kernel(ck_path)[0]
+    assert pointing_ck_path.name == f"imap_dps_{start_date}_{end_date}_{version}.ah.bc"
+    # Verify that file is valid pointing_attitude kernel with imap-data-access
+    spice_input = SPICEInput(pointing_ck_path.name)
+    assert spice_input.source[0] == "pointing_attitude"
+
+
 @pytest.mark.parametrize(
     "segment_start_offset, segment_end_offset, quaternion, segment_id",
     [

imap_processing/tests/test_cli.py

@@ -314,6 +314,38 @@ def test_spacecraft(mock_spacecraft_l1a, mock_instrument_dependencies):
     assert mock_instrument_dependencies["mock_write_cdf"].call_count == 1
 
 
+@mock.patch(
+    "imap_processing.cli.pointing_frame.generate_pointing_attitude_kernel",
+    autospec=True,
+)
+def test_spacecraft_pointing_kernel(
+    mock_spacecraft_pointing, mock_instrument_dependencies
+):
+    """Test coverage for cli.Spacecraft class"""
+
+    dependency_str = (
+        '[{"type": "spice","files": ["naif0012.tls", '
+        '"imap_sclk_0005.tsc", "imap_2024_100_2024_111_05.ah.bc"]}]'
+    )
+    input_collection = ProcessingInputCollection()
+    input_collection.deserialize(dependency_str)
+    mocks = mock_instrument_dependencies
+    mocks["mock_query"].return_value = [{"file_path": "/path/to/file0"}]
+    mocks["mock_download"].return_value = "file0"
+    mock_spacecraft_pointing.return_value = [
+        Path("imap_dps_2024_100_2024_111_05.ah.bc")
+    ]
+    mocks["mock_write_cdf"].side_effect = ["/path/to/file0"]
+    mocks["mock_pre_processing"].return_value = input_collection
+
+    instrument = Spacecraft(
+        "spice", "pointing_kernel", dependency_str, "20240410", "12345", "v005", False
+    )
+
+    instrument.process()
+    assert mock_spacecraft_pointing.call_count == 1
+
+
 @mock.patch("imap_processing.cli.ultra_l1a.ultra_l1a")
 def test_ultra_l1a(mock_ultra_l1a, mock_instrument_dependencies):
     """Test coverage for cli.Ultra class with l1a data level"""

poetry.lock

@@ -35,7 +35,7 @@ exclude = ["imap_processing/tests"]
 [tool.poetry.dependencies]
 astropy-healpix = ">=1.0"
 cdflib = "^1.3.1"
-imap-data-access = ">=0.28.0"
+imap-data-access = ">=0.29.0"
 python = ">=3.10,<4"
 space_packet_parser = "^5.0.1"
 spiceypy = ">=6.0.0"