V1.6.0_release_prep_fixes (#440)

* fix aspect missing class vars

* more logging

* remove parse tree from packet via keyword to save memory (just needed for JSON export)

* parse tree removal as option

* added documentation string for param

* fix: Applying a mark to a fixture function pytest deprecation

* fix: Applying a mark to a fixture function pytest deprecation

Author: nicHoch

stixcore/idb/tests/test_idb.py

@@ -17,7 +17,6 @@
 
 
 @pytest.fixture
-@pytest.mark.remote_data
 def idb():
     return IDBManager(test_data.idb.DIR).get_idb(VERSION)
 
@@ -35,7 +34,7 @@ def test_idb_setup(idb):
 
 def test_idb_setup_fails():
     with pytest.raises(sqlite3.Error) as e:
-        _idb = IDB(Path(os.path.abspath(__file__)).parent / "data")
+        _ = IDB(Path(os.path.abspath(__file__)).parent / "data")
 
     assert len(str(e.value)) > 0
 

stixcore/idb/tests/test_manager.py

@@ -10,7 +10,6 @@
 
 
 @pytest.fixture
-@pytest.mark.remote_data
 def idb_manager():
     return IDBManager(test_data.idb.DIR)
 

stixcore/io/product_processors/fits/processors.py

@@ -641,7 +641,7 @@ def write_fits(self, product, path=None, *, version=0):
             hdul = fits.HDUList(hdul)
 
             filetowrite = path / filename
-            logger.debug(f"Writing fits file to {filetowrite}")
+            logger.info(f"Writing fits file to {filetowrite}")
             hdul.writeto(filetowrite, overwrite=True, checksum=True)
             created_files.append(filetowrite)
         return created_files

stixcore/processing/LBtoL0.py

@@ -81,13 +81,15 @@ def process_tm_type(files, tm_type, processor, spice_kernel_path, config, idbm):
     IDBManager.instance = idbm
     CONFIG = config
 
+    logger.info(f"Start Processing TM type: {tm_type} with {len(files)} files")
+
     RidLutManager.instance = RidLutManager(Path(CONFIG.get("Publish", "rid_lut_file")), update=False)
 
     # Stand alone packet data
     if (tm_type[0] == 21 and tm_type[-2] not in {20, 21, 22, 23, 24, 42}) or tm_type[0] != 21:
         for file in files:
-            levelb = Product(file)
             logger.info(f"processing file: {file}")
+            levelb = Product(file)
             tmp = Product._check_registered_widget(
                 level="L0",
                 service_type=levelb.service_type,
@@ -97,7 +99,7 @@ def process_tm_type(files, tm_type, processor, spice_kernel_path, config, idbm):
                 control=None,
             )
             try:
-                level0 = tmp.from_levelb(levelb, parent=file.name)
+                level0 = tmp.from_levelb(levelb, parent=file.name, keep_parse_tree=False)
                 if level0:
                     fits_files = processor.write_fits(level0)
                     all_files.extend(fits_files)
@@ -113,6 +115,7 @@ def process_tm_type(files, tm_type, processor, spice_kernel_path, config, idbm):
     else:
         # for each file
         for file in files:
+            logger.info(f"processing file: {file}")
             levelb = Product(file)
             complete, _ = levelb.extract_sequences()
 
@@ -129,7 +132,7 @@ def process_tm_type(files, tm_type, processor, spice_kernel_path, config, idbm):
                             data=None,
                             control=None,
                         )
-                        level0 = tmp.from_levelb(comp, parent=file.name)
+                        level0 = tmp.from_levelb(comp, parent=file.name, keep_parse_tree=False)
                         fits_files = processor.write_fits(level0)
                         all_files.extend(fits_files)
                     except NotCombineException as nc:
@@ -146,6 +149,7 @@ def process_tm_type(files, tm_type, processor, spice_kernel_path, config, idbm):
                         logger.error("%s", e)
                         if CONFIG.getboolean("Logging", "stop_on_error", fallback=False):
                             raise e
+    logger.info(f"Finished Processing TM type: {tm_type} created {len(all_files)} files")
     return all_files
 
 

stixcore/processing/pipeline.py

@@ -246,9 +246,9 @@ def __init__(self, tm_handler):
         self.current_tm = (None, datetime.now())
         self.tm_handler = tm_handler
 
-        self.status_server_thread = threading.Thread(target=self.status_server)
-        self.status_server_thread.daemon = True
-        self.status_server_thread.start()
+        # self.status_server_thread = threading.Thread(target=self.status_server)
+        # self.status_server_thread.daemon = True
+        # self.status_server_thread.start()
 
     @staticmethod
     def get_config():

stixcore/processing/pipeline_cron.py

@@ -8,7 +8,6 @@
 import smtplib
 import warnings
 import importlib
-import threading
 import subprocess
 from pprint import pformat
 from pathlib import Path
@@ -169,9 +168,9 @@ def __init__(self, tm_list):
         self.current_tm = (None, datetime.now())
         self.tm_list = tm_list
 
-        self.status_server_thread = threading.Thread(target=self.status_server)
-        self.status_server_thread.daemon = True
-        self.status_server_thread.start()
+        # self.status_server_thread = threading.Thread(target=self.status_server)
+        # self.status_server_thread.daemon = True
+        # self.status_server_thread.start()
 
     @staticmethod
     def get_config():

stixcore/products/ANC/aspect.py

@@ -239,6 +239,9 @@ class Ephemeris(GenericProduct, L2Mixin):
     """
 
     PRODUCT_PROCESSING_VERSION = 2
+    NAME = "ephemeris"
+    LEVEL = "ANC"
+    TYPE = "asp"
 
     def __init__(
         self,
@@ -260,9 +263,9 @@ def __init__(
             idb_versions=idb_versions,
             **kwargs,
         )
-        self.name = "ephemeris"
-        self.level = "ANC"
-        self.type = "asp"
+        self.name = Ephemeris.NAME
+        self.level = Ephemeris.LEVEL
+        self.type = Ephemeris.TYPE
         self.ssid = 1
         self.service_subtype = 0
         self.service_type = 0
@@ -285,4 +288,4 @@ def fits_daily_file(self):
 
     @classmethod
     def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs):
-        return kwargs["level"] == "ANC" and service_type == 0 and service_subtype == 0 and ssid == 1
+        return kwargs["level"] == Ephemeris.LEVEL and service_type == 0 and service_subtype == 0 and ssid == 1

stixcore/products/level0/housekeepingL0.py

@@ -54,7 +54,7 @@ def fits_daily_file(self):
         return True
 
     @classmethod
-    def from_levelb(cls, levelb, *, parent=""):
+    def from_levelb(cls, levelb, *, parent="", keep_parse_tree=True):
         """Converts level binary HK packets to a L1 product.
 
         Parameters
@@ -65,6 +65,8 @@ def from_levelb(cls, levelb, *, parent=""):
             The parent data file name the binary packed comes from, by default ''
         NIX00405_offset : int, optional
             [description], by default 0
+        keep_parse_tree : bool, optional
+            Whether to keep the parse tree in each packet for debugging and printing, by default True
 
         Returns
         -------
@@ -73,7 +75,7 @@ def from_levelb(cls, levelb, *, parent=""):
             all used IDB versions and time periods
             initialized control table
         """
-        packets, idb_versions = GenericProduct.getLeveL0Packets(levelb)
+        packets, idb_versions = GenericProduct.getLeveL0Packets(levelb, keep_parse_tree=keep_parse_tree)
 
         control = Control()
         control["scet_coarse"] = packets.get("scet_coarse")
@@ -110,8 +112,8 @@ def __init__(
         self.name = "mini"
 
     @classmethod
-    def from_levelb(cls, levelb, parent=""):
-        packets, idb_versions, control = super().from_levelb(levelb, parent=parent)
+    def from_levelb(cls, levelb, parent="", keep_parse_tree=True):
+        packets, idb_versions, control = super().from_levelb(levelb, parent=parent, keep_parse_tree=keep_parse_tree)
 
         # Create array of times as dt from date_obs
         times = SCETime(control["scet_coarse"], control["scet_fine"])
@@ -168,8 +170,8 @@ def __init__(
         self.name = "maxi"
 
     @classmethod
-    def from_levelb(cls, levelb, parent=""):
-        packets, idb_versions, control = super().from_levelb(levelb, parent=parent)
+    def from_levelb(cls, levelb, parent="", keep_parse_tree=True):
+        packets, idb_versions, control = super().from_levelb(levelb, parent=parent, keep_parse_tree=keep_parse_tree)
 
         # Create array of times as dt from date_obs
         times = SCETime(control["scet_coarse"], control["scet_fine"])

stixcore/products/level0/quicklookL0.py

@@ -78,7 +78,7 @@ def fits_daily_file(self):
         return True
 
     @classmethod
-    def from_levelb(cls, levelb, *, parent="", NIX00405_offset=0):
+    def from_levelb(cls, levelb, *, parent="", NIX00405_offset=0, keep_parse_tree=True):
         """Converts level binary packets to a L1 product.
 
         Parameters
@@ -89,6 +89,8 @@ def from_levelb(cls, levelb, *, parent="", NIX00405_offset=0):
             The parent data file name the binary packed comes from, by default ''
         NIX00405_offset : int, optional
             [description], by default 0
+        keep_parse_tree : bool, optional
+            Whether to keep the parse tree in each packet for debugging and printing, by default True
 
         Returns
         -------
@@ -97,7 +99,7 @@ def from_levelb(cls, levelb, *, parent="", NIX00405_offset=0):
             all used IDB versions and time periods
             initialized control table
         """
-        packets, idb_versions = GenericProduct.getLeveL0Packets(levelb)
+        packets, idb_versions = GenericProduct.getLeveL0Packets(levelb, keep_parse_tree=keep_parse_tree)
 
         control = Control.from_packets(packets, NIX00405_offset=NIX00405_offset)
 
@@ -138,8 +140,10 @@ def __init__(
         self.level = "L0"
 
     @classmethod
-    def from_levelb(cls, levelb, parent=""):
-        packets, idb_versions, control = QLProduct.from_levelb(levelb, parent=parent, NIX00405_offset=QLNIX00405_off)
+    def from_levelb(cls, levelb, parent="", keep_parse_tree=True):
+        packets, idb_versions, control = QLProduct.from_levelb(
+            levelb, parent=parent, NIX00405_offset=QLNIX00405_off, keep_parse_tree=keep_parse_tree
+        )
 
         control.add_data("detector_mask", _get_detector_mask(packets))
         control.add_data("pixel_mask", _get_pixel_mask(packets))
@@ -245,8 +249,10 @@ def __init__(
         self.level = "L0"
 
     @classmethod
-    def from_levelb(cls, levelb, parent=""):
-        packets, idb_versions, control = QLProduct.from_levelb(levelb, parent=parent, NIX00405_offset=QLNIX00405_off)
+    def from_levelb(cls, levelb, parent="", keep_parse_tree=True):
+        packets, idb_versions, control = QLProduct.from_levelb(
+            levelb, parent=parent, NIX00405_offset=QLNIX00405_off, keep_parse_tree=keep_parse_tree
+        )
 
         control.add_data("energy_bin_edge_mask", _get_energy_bins(packets, "NIX00266", "NIXD0111"))
         control.add_basic(name="num_energies", nix="NIX00270", packets=packets)
@@ -358,8 +364,10 @@ def __init__(
         self.level = "L0"
 
     @classmethod
-    def from_levelb(cls, levelb, parent=""):
-        packets, idb_versions, control = QLProduct.from_levelb(levelb, parent=parent, NIX00405_offset=QLNIX00405_off)
+    def from_levelb(cls, levelb, parent="", keep_parse_tree=True):
+        packets, idb_versions, control = QLProduct.from_levelb(
+            levelb, parent=parent, NIX00405_offset=QLNIX00405_off, keep_parse_tree=keep_parse_tree
+        )
 
         control.add_data("pixel_mask", _get_pixel_mask(packets))
         control.add_data("compression_scheme_spectra_skm", _get_compression_scheme(packets, "NIX00452"))
@@ -510,8 +518,10 @@ def __init__(
         self.level = "L0"
 
     @classmethod
-    def from_levelb(cls, levelb, parent=""):
-        packets, idb_versions, control = QLProduct.from_levelb(levelb, parent=parent, NIX00405_offset=QLNIX00405_off)
+    def from_levelb(cls, levelb, parent="", keep_parse_tree=True):
+        packets, idb_versions, control = QLProduct.from_levelb(
+            levelb, parent=parent, NIX00405_offset=QLNIX00405_off, keep_parse_tree=keep_parse_tree
+        )
 
         # Control
         control["samples_per_variance"] = np.array(packets.get_value("NIX00279"), np.ubyte)
@@ -600,8 +610,10 @@ def __init__(
         self.level = "L0"
 
     @classmethod
-    def from_levelb(cls, levelb, parent=""):
-        packets, idb_versions, control = QLProduct.from_levelb(levelb, parent=parent, NIX00405_offset=QLNIX00405_off)
+    def from_levelb(cls, levelb, parent="", keep_parse_tree=True):
+        packets, idb_versions, control = QLProduct.from_levelb(
+            levelb, parent=parent, NIX00405_offset=QLNIX00405_off, keep_parse_tree=keep_parse_tree
+        )
 
         control.add_basic(name="num_samples", nix="NIX00089", packets=packets)
 
@@ -683,8 +695,8 @@ def __init__(
         self.type = "cal"
 
     @classmethod
-    def from_levelb(cls, levelb, parent=""):
-        packets, idb_versions, control = QLProduct.from_levelb(levelb, parent=parent)
+    def from_levelb(cls, levelb, parent="", keep_parse_tree=True):
+        packets, idb_versions, control = QLProduct.from_levelb(levelb, parent=parent, keep_parse_tree=keep_parse_tree)
 
         # Control
         control.add_basic(name="integration_time", nix="NIX00122", packets=packets, dtype=np.uint32, attr="value")
@@ -831,8 +843,8 @@ def __init__(
         self.level = "L0"
 
     @classmethod
-    def from_levelb(cls, levelb, parent=""):
-        packets, idb_versions = GenericProduct.getLeveL0Packets(levelb)
+    def from_levelb(cls, levelb, parent="", keep_parse_tree=True):
+        packets, idb_versions = GenericProduct.getLeveL0Packets(levelb, keep_parse_tree=keep_parse_tree)
 
         control = Control()
         control["scet_coarse"] = packets.get("scet_coarse")