HLA-1161: MagErrAp2 and other magnitude errors in HAP catalogs have incorrect negative values

CHANGELOG.rst

@@ -18,8 +18,16 @@ number of the code change for that issue.  These PRs can be viewed at:
     https://github.com/spacetelescope/drizzlepac/pulls
 
 
-3.6.2rc2 (unreleased)
+3.6.3 (unreleased)
 =====================
+- Added FluxAp1, FluxErrAp1, FluxAp2, and FluxErrAp2 to point sources catalogs 
+  as a part of a bug fix for negative magnitude errors and consequently false 
+  postives in the flagging. [#1700]
+
+
+3.6.2 (27-Nov-2023)
+=====================
+
 - At this time pin Astrocut to versions <=0.9 to avoid conflicts with urllib3
   package.  [#1145]
 

doc/ADRs.md

@@ -1,4 +1,4 @@
-# Updating WCS in Headers
+# Updating WCS in Headers 11/14/23
 
 ## Context
 
@@ -13,7 +13,7 @@ In order to ensure reproducibility and the ability to share consistent datasets
 Headerlets can now be used to easily update and select WCS for datasets. Headerlets are files with the _hlet.fits suffix
 
 
-# Updating APERTURE keyword through poller
+# Updating APERTURE keyword through poller 11/14/23
 
 ## Context
 

drizzlepac/haputils/catalog_utils.py

@@ -1277,15 +1277,16 @@ def measure_sources(self, filter_name):
         photometry_tbl.add_column(flag_col)
 
         # build final output table
-        final_col_order = ["X-Center", "Y-Center", "RA", "DEC", "ID", "MagAp1", "MagErrAp1", "MagAp2", "MagErrAp2",
-                           "MSkyAp2", "StdevAp2", "FluxAp2", "CI", "Flags"]
+        final_col_order = ["X-Center", "Y-Center", "RA", "DEC", "ID", "MagAp1", "MagErrAp1", "FluxAp1", 
+                           "FluxErrAp1", "MagAp2", "MagErrAp2", "MSkyAp2", "StdevAp2", "FluxAp2", 
+                           "FluxErrAp2", "CI", "Flags"]
         output_photometry_table = photometry_tbl[final_col_order]
 
         # format output table columns
         final_col_format = {"X-Center": "10.3f", "Y-Center": "10.3f", "RA": "13.7f", "DEC": "13.7f", "ID": "7d",
-                            "MagAp1": '7.3f', "MagErrAp1": '7.3f', "MagAp2": '7.3f',
-                            "MagErrAp2": '7.3f', "MSkyAp2": '7.3f', "StdevAp2": '7.3f',
-                            "FluxAp2": '10.4f', "CI": "7.3f", "Flags": "5d"}  # TODO: Standardize precision
+                            "MagAp1": '7.3f', "MagErrAp1": '7.3f', "FluxAp1": '10.4f', "FluxErrAp1": '10.4f',
+                            "MagAp2": '7.3f', "MagErrAp2": '7.3f', "MSkyAp2": '7.3f', "StdevAp2": '7.3f',
+                            "FluxAp2": '10.4f', "FluxErrAp2": '10.4f', "CI": "7.3f", "Flags": "5d"}  # TODO: Standardize precision
         for fcf_key in final_col_format.keys():
             output_photometry_table[fcf_key].format = final_col_format[fcf_key]
 
@@ -1297,11 +1298,14 @@ def measure_sources(self, filter_name):
                              "DEC": "Sky coordinate at epoch of observation",
                              "MagAp1": "ABMAG of source based on the inner (smaller) aperture",
                              "MagErrAp1": "Error of MagAp1",
+                             "FluxAp1": "Flux of source based on the outer (smaller) aperture",
+                             "FluxErrAp1": "Flux of source based on the outer (smaller) aperture",
                              "MagAp2": "ABMAG of source based on the outer (larger) aperture",
                              "MagErrAp2": "Error of MagAp2",
                              "MSkyAp2": "Sky estimate from an annulus outside Aperture 2",
                              "StdevAp2": "Standard deviation of sky estimate from annulus outside Aperture 2",
                              "FluxAp2": "Flux of source based on the outer (larger) aperture",
+                             "FluxErrAp2": "Flux of source based on the outer (larger) aperture",
                              "CI": "Concentration Index",
                              "Flags": "Numeric encoding for conditions on detected sources"}
         for fcd_key in final_col_descrip.keys():
@@ -1311,7 +1315,8 @@ def measure_sources(self, filter_name):
         final_col_units = {"X-Center": "pixel", "Y-Center": "pixel", "RA": "degree", "DEC": "degree",
                            "ID": "", "MagAp1": "mag(AB)", "MagErrAp1": "mag(AB)", "MagAp2": "mag(AB)",
                            "MagErrAp2": "mag(AB)", "MSkyAp2": "electron/(s pixel)", "StdevAp2": "electron/(s pixel)",
-                           "FluxAp2": "electron/s", "CI": "mag(AB)", "Flags": ""}
+                           "FluxAp1": "electron/s", "FluxErrAp1": "electron/s","FluxAp2": "electron/s", 
+                           "FluxErrAp2": "electron/s", "CI": "mag(AB)", "Flags": ""}
         for col_title in final_col_units:
             output_photometry_table[col_title].unit = final_col_units[col_title]
 

drizzlepac/haputils/hla_flag_filter.py

@@ -249,7 +249,12 @@ def ci_filter(drizzled_image, catalog_name, catalog_data, proc_type, param_dict,
             merr2 = numpy.nan
         else:
             merr2 = float(merr2)
-        good_snr = merr2 <= 2.5 / (snr * numpy.log(10))
+
+        # SNR calculation based on flux instead of magnitude as negative magnitudes are possible
+        fluxap2 = table_row["FluxAp2"]
+        fluxerr2 = table_row["FluxErrAp2"]
+        good_snr = fluxap2 >= snr*fluxerr2
+
         ci_err = numpy.sqrt(merr1 ** 2 + merr2 ** 2)
 
         if not good_snr:

drizzlepac/haputils/photometry_tools.py

@@ -151,6 +151,8 @@ def iraf_style_photometry(phot_apertures, bg_apertures, data, photflam, photplam
             mag_err = mag
         else:
             mag_err = 1.0857 * flux_error / flux
+            # Set magnitude errors to infinity if flux is negative
+            mag_err[np.logical_not(flux>0)]=np.inf
 
         # Build the final data table
         stacked = np.stack([flux, flux_error, mag, mag_err], axis=1)

tests/hap/test_svm_ibqk07.py

@@ -31,8 +31,8 @@
 POLLER_FILE = "wfc3_bqk_07_input.out"
 WCS_SUB_NAME = "HSC30"
 expected_total_point_sources = {
-"hst_12557_07_wfc3_ir_total_ibqk07_point-cat.ecsv": 4,
-"hst_12557_07_wfc3_uvis_total_ibqk07_point-cat.ecsv": 14}
+"hst_12557_07_wfc3_ir_total_ibqk07_point-cat.ecsv": 2,
+"hst_12557_07_wfc3_uvis_total_ibqk07_point-cat.ecsv": 13}
 expected_total_segment_sources= {
 "hst_12557_07_wfc3_ir_total_ibqk07_segment-cat.ecsv": 2,
 "hst_12557_07_wfc3_uvis_total_ibqk07_segment-cat.ecsv": 19}