Hi Processing fixes for SIT4 (IMAP-Science-Operations-Center#1785)

* Fix L1C PSET ISTP warning

* Add epoch dimension to L2 solid_angle variable

* Add epoch dimension to L2 solid_angle variable - test coverage

* Convert obs_date to int64 after calculation

* Don't check schema when getting variable attributes for label or delta variables

* Make energy dependent attributes the default

* Fix Hi L2 do_processing

Author: tmplummer

imap_processing/cdf/config/imap_enamaps_l2-common_variable_attrs.yaml

@@ -162,7 +162,7 @@ sensitivity:
   DISPLAY_TYPE: image
   DICT_KEY: SPASE>Particle>ParticleType:Atom,ParticleQuantity:GeometricFactor,Qualifier:Directional,CoordinateSystemName:HAE,CoordinateRepresentation:Spherical
 
-exposure_factor: &exposure_factor_energy_independent
+exposure_factor: &exposure_factor
   <<: *default_float32
   CATDESC: Exact or approximate time over which counts are accumulated.
   FIELDNAM: Exposure Times
@@ -173,7 +173,7 @@ exposure_factor: &exposure_factor_energy_independent
   DISPLAY_TYPE: no_plot
   DICT_KEY: SPASE>TemporalDescription:Exposure,Qualifier:Directional,CoordinateSystemName:HAE,CoordinateRepresentation:Spherical
 
-obs_date: &obs_date_energy_independent
+obs_date: &obs_date
   <<: *default_int64
   datatype: int64
   CATDESC: Exposure time weighted mean collection date of data in a pixel.
@@ -195,15 +195,16 @@ obs_date_range:
   VAR_TYPE: data
   LABLAXIS: epoch
   DISPLAY_TYPE: image
+  TIME_SCALE: Terrestrial Time
 
 # These copied metadata vars will allow for variables
 # to be either energy-dependent or independent.
 # The tiling-specific YAML files will override the DEPENDs and LABL_PTRs.
-exposure_factor_energy_dependent:
-  <<: *exposure_factor_energy_independent
+exposure_factor_energy_independent:
+  <<: *exposure_factor
 
-obs_date_energy_dependent:
-  <<: *obs_date_energy_independent
+obs_date_energy_independent:
+  <<: *obs_date
 
 solid_angle:
   <<: *default_float32

imap_processing/cdf/config/imap_enamaps_l2-healpix_variable_attrs.yaml

@@ -75,25 +75,25 @@ sensitivity:
 # These data variables will have an extra (energy) dimension
 # only if the energy dimension is present in the L1C data.
 # The default is energy-independent.
-exposure_factor:
+exposure_factor_energy_independent:
   DEPEND_1: pixel_index
   LABL_PTR_1: pixel_index_label
 
-obs_date:
+obs_date_energy_independent:
   DEPEND_1: pixel_index
   LABL_PTR_1: pixel_index_label
 
 obs_date_range:
   DEPEND_1: pixel_index
   LABL_PTR_1: pixel_index_label
 
-exposure_factor_energy_dependent:
+exposure_factor:
   DEPEND_1: energy
   DEPEND_2: pixel_index
   LABL_PTR_1: energy_label
   LABL_PTR_2: pixel_index_label
 
-obs_date_energy_dependent:
+obs_date:
   DEPEND_1: energy
   DEPEND_2: pixel_index
   LABL_PTR_1: energy_label

imap_processing/cdf/config/imap_enamaps_l2-rectangular_variable_attrs.yaml

@@ -90,33 +90,41 @@ sensitivity:
 # These data variables will have an extra (energy) dimension
 # only if the energy dimension is present in the L1C data.
 # The default is energy-independent.
-exposure_factor:
+exposure_factor_energy_independent:
   DEPEND_1: longitude
   DEPEND_2: latitude
   LABL_PTR_1: longitude_label
   LABL_PTR_2: latitude_label
 
-obs_date:
+obs_date_energy_independent:
   DEPEND_1: longitude
   DEPEND_2: latitude
   LABL_PTR_1: longitude_label
   LABL_PTR_2: latitude_label
 
-obs_date_range:
+obs_date_range_energy_independent:
   DEPEND_1: longitude
   DEPEND_2: latitude
   LABL_PTR_1: longitude_label
   LABL_PTR_2: latitude_label
 
-exposure_factor_energy_dependent:
+exposure_factor:
   DEPEND_1: energy
   DEPEND_2: longitude
   DEPEND_3: latitude
   LABL_PTR_1: energy_label
   LABL_PTR_2: longitude_label
   LABL_PTR_3: latitude_label
 
-obs_date_energy_dependent:
+obs_date:
+  DEPEND_1: energy
+  DEPEND_2: longitude
+  DEPEND_3: latitude
+  LABL_PTR_1: energy_label
+  LABL_PTR_2: longitude_label
+  LABL_PTR_3: latitude_label
+
+obs_date_range:
   DEPEND_1: energy
   DEPEND_2: longitude
   DEPEND_3: latitude

imap_processing/cdf/config/imap_hi_variable_attrs.yaml

@@ -579,18 +579,21 @@ hi_pset_background_rates_uncertainty:
 hi_pset_spin_bin_label:
   CATDESC: Label spin angle bin
   FIELDNAM: Label spin angle bin
+  DEPEND_1: spin_angle_bin
   FORMAT: A4
   VAR_TYPE: metadata
 
 hi_pset_esa_energy_step_label:
   CATDESC: Label esa step
   FIELDNAM: Label esa step
+  DEPEND_1: esa_energy_step
   FORMAT: A4
   VAR_TYPE: metadata
 
 hi_pset_calibration_prod_label:
   CATDESC: Label calibration product
   FIELDNAM: Label calibration product
+  DEPEND_1: calibration_prod
   FORMAT: A4
   VAR_TYPE: metadata
 

imap_processing/cli.py

@@ -740,7 +740,7 @@ def do_processing(
                 )
             datasets = hi_l1c.hi_l1c(load_cdf(science_paths[0]), anc_paths[0])
         elif self.data_level == "l2":
-            science_paths = dependencies.get_file_paths(source="hi", data_type="l2")
+            science_paths = dependencies.get_file_paths(source="hi", data_type="l1c")
             # TODO get ancillary paths
             geometric_factors_path = ""
             esa_energies_path = ""

imap_processing/ena_maps/ena_maps.py

@@ -1119,9 +1119,9 @@ def to_dataset(self) -> xr.Dataset:
             )
         # Add the solid angle variable to the data_1d Dataset
         self.data_1d["solid_angle"] = xr.DataArray(
-            self.solid_angle_points,
+            self.solid_angle_points[np.newaxis, :],
             name="solid_angle",
-            dims=[CoordNames.GENERIC_PIXEL.value],
+            dims=[CoordNames.TIME.value, CoordNames.GENERIC_PIXEL.value],
         )
         # Rewrap each data array in the data_1d to the original 2D grid shape
         rewrapped_data = {}
@@ -1275,8 +1275,14 @@ def build_cdf_dataset(
         # Set the variable attributes
         for var in [*cdf_ds.data_vars, *cdf_ds.coords]:
             try:
+                # Don't check schema on label or delta variables
+                ignore_schema_substrings = ["_label", "_delta"]
+                check_schema = (
+                    False if any(s in var for s in ignore_schema_substrings) else True
+                )
                 var_attrs = cdf_attrs.get_variable_attributes(
                     variable_name=var,
+                    check_schema=check_schema,
                 )
             except KeyError as e:
                 raise KeyError(

imap_processing/hi/l2/hi_l2.py

@@ -136,6 +136,9 @@ def generate_hi_map(
         )
     )
 
+    rect_map.data_1d["obs_date"].data = rect_map.data_1d["obs_date"].data.astype(
+        np.int64
+    )
     # TODO: Figure out how to compute obs_date_range (stddev of obs_date)
     rect_map.data_1d["obs_date_range"] = xr.zeros_like(rect_map.data_1d["obs_date"])
 

imap_processing/tests/ena_maps/test_ena_maps.py

@@ -443,6 +443,11 @@ def test_project_rect_pset_values_to_map_pull_method(
         # operations to be repeated as this test
         rect_map_ds = rectangular_map.to_dataset()
         assert "solid_angle" in rect_map_ds.data_vars
+        assert rect_map_ds.data_vars["solid_angle"].shape == (
+            1,
+            360 / skymap_spacing,
+            180 / skymap_spacing,
+        )
         assert "counts" in rect_map_ds.data_vars
         assert rect_map_ds["counts"].shape == (
             1,

imap_processing/tests/hi/test_hi_l2.py

@@ -4,6 +4,7 @@
 import pytest
 import xarray as xr
 
+from imap_processing.cdf.utils import write_cdf
 from imap_processing.ena_maps.ena_maps import RectangularSkyMap
 from imap_processing.hi.l2.hi_l2 import (
     calculate_ena_intensity,
@@ -46,6 +47,8 @@ def test_hi_l2(hi_l1_test_data_path):
     np.testing.assert_array_equal(
         l2_dataset["ena_intensity"].dims, ["epoch", "energy", "longitude", "latitude"]
     )
+    # Test ISTP compliance by writing the CDF
+    write_cdf(l2_dataset, istp=True)
 
 
 @pytest.mark.external_test_data

imap_processing/ultra/l2/ultra_l2.py

@@ -118,10 +118,10 @@ def get_variable_attributes_optional_energy_dependence(
     # These variables must get metadata with a different key if they are energy
     # dependent.
     if (variable_name in INCONSISTENTLY_ENERGY_DEPENDENT_VARIABLES) and (
-        (CoordNames.ENERGY_L2.value in variable_dims)
-        or (CoordNames.ENERGY_ULTRA_L1C.value in variable_dims)
+        (CoordNames.ENERGY_L2.value not in variable_dims)
+        and (CoordNames.ENERGY_ULTRA_L1C.value not in variable_dims)
     ):
-        variable_name = f"{variable_name}_energy_dependent"
+        variable_name = f"{variable_name}_energy_independent"
 
     metadata = cdf_attrs.get_variable_attributes(
         variable_name=variable_name,
@@ -462,14 +462,6 @@ def ultra_l2(
 
         map_dataset = rectangular_skymap.to_dataset()
 
-        # Reshape the solid_angle to have an epoch dimension at the start
-        map_dataset["solid_angle"] = map_dataset["solid_angle"].expand_dims(
-            {
-                CoordNames.TIME.value: 1,
-            },
-            axis=0,
-        )
-
         # Add longitude_delta, latitude_delta to the map dataset
         map_dataset["longitude_delta"] = rectangular_skymap.spacing_deg / 2
         map_dataset["latitude_delta"] = rectangular_skymap.spacing_deg / 2