FIX: timer ensure timer LSB is initialized

FIX: adjusting timestamps before heartbeat

FIX: adjusting timestamps before heartbeat

ENH: apply SPIDR rollover with single partition if no heartbeat is received

tpx3awkward/_utils.py

@@ -205,11 +205,13 @@ def _ingest_raw_data(data):
     y = np.zeros_like(data, dtype="u2")
     tot = np.zeros_like(data, dtype="u4")
     ts = np.zeros_like(data, dtype="u8")
-    heartbeat_lsb = np.uint64(0)
-    heartbeat_msb = np.uint64(0)
+    heartbeat_lsb = None  #np.uint64(0)
+    heartbeat_msb = None  #np.uint64(0)
     heartbeat_time = np.uint64(0)
+    hb_init_flag = False    # Indicate when the heartbeat was set for the first time
 
     photon_count, chip_indx, msg_run_count, expected_msg_count = 0, 0, 0, 0
+
     for msg in data:
         if is_packet_header(msg):
             # Type 1: packet header (id'd via TPX3 magic number)
@@ -226,26 +228,26 @@ def _ingest_raw_data(data):
         elif matches_nibble(msg, 0xB):
             # Type 2: photon event (id'd via 0xB upper nibble)
             chips[photon_count] = chip_indx
-            # heartbeat_time = np.uint64(0)
             _x, _y, _tot, _ts = decode_message(msg, chip_indx, heartbeat_time=heartbeat_time)
             x[photon_count] = _x
             y[photon_count] = _y
             tot[photon_count] = _tot
             ts[photon_count] = _ts
-
-            if (photon_count > 0) and (_ts > ts[photon_count-1]) and (_ts - ts[photon_count-1] > 2**30):
+
+            # Adjust timestamps that were set before the first heartbeat was received
+            if hb_init_flag and (photon_count > 0):
                 prev_ts = ts[:photon_count]   # This portion needs to be adjusted
                 # Find what the current timestamp would be without global heartbeat
                 _, _, _, _ts_0 = decode_message(msg, chip_indx, heartbeat_time=np.uint64(0))
-                # Check if there is a SPIDR rollover in the beginning of the file but before heartbeat was received
+                # Check if there is a SPIDR rollover in the beginning of the file but before the received
                 head_max = max(prev_ts[:10])
                 tail_min = min(prev_ts[-10:])
                 if (head_max > tail_min) and (head_max - tail_min > 2**32):
-                    prev_ts[prev_ts < 2**33] += np.uint64(2**34)
-                    _ts_0 += 2**34
-                # Ajust already processed timestamps
+                    prev_ts[prev_ts < (tail_min+head_max)/2] += np.uint64(2**34)
+                    _ts_0 += np.uint64(2**34)
                 ts[:photon_count] = prev_ts + (_ts - _ts_0)
 
+            hb_init_flag = False
             photon_count += 1
             msg_run_count += 1
 
@@ -258,14 +260,16 @@ def _ingest_raw_data(data):
             # Type 4: global timestap (id'd via 0x4 upper nibble)
             subheader = (msg >> np.uint(56)) & np.uint64(0x0F)
             if subheader == 0x4:
-                # timer lsb, 32 bits of time
+                # timer LSB, 32 bits of time -- needs to be received first, before MSB
                 heartbeat_lsb = (msg >> np.uint(16)) & np.uint64(0xFFFFFFFF)
             elif subheader == 0x5:
-                # timer msb
-                time_msg = (msg >> np.uint(16)) & np.uint64(0xFFFF)
-                heartbeat_msb = time_msg << np.uint(32)
-                heartbeat_time = (heartbeat_msb | heartbeat_lsb) # << np.uint(4)
-                # TODO the c++ code has large jump detection, do not understand why
+                # timer MSB -- only matters if LSB has been received already
+                if heartbeat_lsb is not None:
+                    if heartbeat_msb is None:
+                        hb_init_flag = True
+                    heartbeat_msb = ((msg >> np.uint(16)) & np.uint64(0xFFFF)) << np.uint(32)
+                    heartbeat_time = (heartbeat_msb | heartbeat_lsb)
+                    # TODO the c++ code has large jump detection, do not understand why
             else:
                 raise Exception(f"Unknown subheader {subheader} in the Global Timestamp message")
             pass
@@ -280,6 +284,12 @@ def _ingest_raw_data(data):
         else:
             raise Exception(f"Not supported: {msg}")
 
+    # Check if there were no heartbeat messages and adjust for potential SPIDR rollovers
+    if heartbeat_msb is None:
+        head_max = max(ts[:10])
+        tail_min = min(ts[-10:])
+        if (head_max > tail_min) and (head_max - tail_min > 2**32):
+            ts[ts < (tail_min+head_max)/2] += np.uint64(2**34)
 
     # Sort the timestamps
     indx = np.argsort(ts[:photon_count], kind="mergesort")