Merge branch 'develop' into build/fix_build_workflow

spacetelescope · Jan 22, 2025 · 2b119a0 · 2b119a0
2 parents a21a01f + c1ed93e
commit 2b119a0
Show file tree

Hide file tree

Showing 24 changed files with 1,295 additions and 824 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -1,5 +1,23 @@
 ## What's Changed
 
+1.3.0 (2024-12-19)
+==================
+
+Web Application
+~~~~~~~~~~~~~~~
+* Exclude source-specific WFSS files from observation page by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1651
+* Switch URL for prog info scraping to use the OPO site by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1662
+
+Project & API Documentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added logging configuration to config file, and use it when opening logging by @york-stsci in https://github.com/spacetelescope/jwql/pull/1635
+* Fix bad parens in dark monitor model definitions by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1644
+* Add radius keyword to bokeh.figure.circle calls by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1643
+* Remove bokeh templating code by @bhilbert4 in https://github.com/spacetelescope/jwql/pull/1647
+* Update Bad Pixel Monitor to use Django DB Models by @mfixstsci in https://github.com/spacetelescope/jwql/pull/1497
+* Update Bias Monitor to use Django DB Models by @bsunnquist in https://github.com/spacetelescope/jwql/pull/1503
+
+
 1.2.11 (2024-08-26)
 ===================
 

diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py
diff --git a/jwql/instrument_monitors/common_monitors/bias_monitor.py b/jwql/instrument_monitors/common_monitors/bias_monitor.py
@@ -35,7 +35,6 @@
 import datetime
 import logging
 import os
-from time import sleep
 
 from astropy.io import fits
 from astropy.stats import sigma_clip, sigma_clipped_stats
@@ -47,21 +46,26 @@
 from mpl_toolkits.axes_grid1 import make_axes_locatable  # noqa: E402 (module import not at top)
 import numpy as np  # noqa: E402 (module import not at top)
 from pysiaf import Siaf  # noqa: E402 (module import not at top)
-from sqlalchemy.sql.expression import and_  # noqa: E402 (module import not at top)
 
-from jwql.database.database_interface import session, engine  # noqa: E402 (module import not at top)
-from jwql.database.database_interface import NIRCamBiasQueryHistory, NIRCamBiasStats, NIRISSBiasQueryHistory  # noqa: E402 (module import not at top)
-from jwql.database.database_interface import NIRISSBiasStats, NIRSpecBiasQueryHistory, NIRSpecBiasStats  # noqa: E402 (module import not at top)
 from jwql.instrument_monitors import pipeline_tools  # noqa: E402 (module import not at top)
-from jwql.shared_tasks.shared_tasks import only_one, run_pipeline, run_parallel_pipeline  # noqa: E402 (module import not at top)
+from jwql.shared_tasks.shared_tasks import only_one, run_parallel_pipeline  # noqa: E402 (module import not at top)
 from jwql.utils import instrument_properties, monitor_utils  # noqa: E402 (module import not at top)
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE  # noqa: E402 (module import not at top)
+from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE, ON_GITHUB_ACTIONS, ON_READTHEDOCS  # noqa: E402 (module import not at top)
 from jwql.utils.logging_functions import log_info, log_fail  # noqa: E402 (module import not at top)
-from jwql.utils.monitor_utils import update_monitor_table  # noqa: E402 (module import not at top)
 from jwql.utils.permissions import set_permissions  # noqa: E402 (module import not at top)
-from jwql.utils.utils import copy_files, ensure_dir_exists, filesystem_path, get_config  # noqa: E402 (module import not at top)
+from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config  # noqa: E402 (module import not at top)
 from jwql.website.apps.jwql.monitor_pages.monitor_bias_bokeh import BiasMonitorPlots  # noqa: E402 (module import not at top)
 
+if not ON_GITHUB_ACTIONS and not ON_READTHEDOCS:
+    # Need to set up django apps before we can access the models
+    import django  # noqa: E402 (module level import not at top of file)
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "jwql.website.jwql_proj.settings")
+    django.setup()
+
+    # Import * is okay here because this module specifically only contains database models
+    # for this monitor
+    from jwql.website.apps.jwql.monitor_models.bias import *  # noqa: E402 (module level import not at top of file)
+
 
 class Bias():
     """Class for executing the bias monitor.
@@ -201,15 +205,13 @@ def file_exists_in_database(self, filename):
             ``True`` if filename exists in the bias stats database.
         """
 
-        query = session.query(self.stats_table)
-        results = query.filter(self.stats_table.uncal_filename == filename).all()
+        records = self.stats_table.objects.filter(uncal_filename__iexact=filename).all()
 
-        if len(results) != 0:
+        if len(records) != 0:
             file_exists = True
         else:
             file_exists = False
 
-        session.close()
         return file_exists
 
     def get_amp_medians(self, image, amps):
@@ -346,16 +348,16 @@ def most_recent_search(self):
             where the bias monitor was run.
         """
 
-        query = session.query(self.query_table).filter(and_(self.query_table.aperture == self.aperture,
-                self.query_table.run_monitor == True)).order_by(self.query_table.end_time_mjd).all()  # noqa: E348 (comparison to true)
+        filters = {'aperture__iexact': self.aperture,
+                   'run_monitor': True}
+        record = self.query_table.objects.filter(**filters).order_by('-end_time_mjd').first()
 
-        if len(query) == 0:
+        if record is None:
             query_result = 59607.0  # a.k.a. Jan 28, 2022 == First JWST images (MIRI)
             logging.info(('\tNo query history for {}. Beginning search date will be set to {}.'.format(self.aperture, query_result)))
         else:
-            query_result = query[-1].end_time_mjd
+            query_result = record.end_time_mjd
 
-        session.close()
         return query_result
 
     def process(self, file_list):
@@ -420,18 +422,18 @@ def process(self, file_list):
                              'mean': float(mean),
                              'median': float(median),
                              'stddev': float(stddev),
-                             'collapsed_rows': collapsed_rows.astype(float),
-                             'collapsed_columns': collapsed_columns.astype(float),
-                             'counts': counts.astype(float),
-                             'bin_centers': bin_centers.astype(float),
+                             'collapsed_rows': list(collapsed_rows.astype(float)),
+                             'collapsed_columns': list(collapsed_columns.astype(float)),
+                             'counts': list(counts.astype(float)),
+                             'bin_centers': list(bin_centers.astype(float)),
                              'entry_date': datetime.datetime.now()
                              }
             for key in amp_medians.keys():
                 bias_db_entry[key] = float(amp_medians[key])
 
             # Add this new entry to the bias database table
-            with engine.begin() as connection:
-                connection.execute(self.stats_table.__table__.insert(), bias_db_entry)
+            entry = self.stats_table(**bias_db_entry)
+            entry.save()
 
             # Don't print long arrays of numbers to the log file
             log_dict = {}
@@ -545,8 +547,8 @@ def run(self):
                              'files_found': len(new_files),
                              'run_monitor': monitor_run,
                              'entry_date': datetime.datetime.now()}
-                with engine.begin() as connection:
-                    connection.execute(self.query_table.__table__.insert(), new_entry)
+                entry = self.query_table(**new_entry)
+                entry.save()
                 logging.info('\tUpdated the query history table')
 
             # Update the bias monitor plots

diff --git a/jwql/jwql_monitors/create_initial_preview_and_thumbnail_listfiles.py b/jwql/jwql_monitors/create_initial_preview_and_thumbnail_listfiles.py
diff --git a/jwql/jwql_monitors/generate_preview_images.py b/jwql/jwql_monitors/generate_preview_images.py
@@ -36,8 +36,11 @@
 import numpy as np
 
 from jwql.utils import permissions
-from jwql.utils.constants import IGNORED_SUFFIXES, JWST_INSTRUMENT_NAMES_SHORTHAND, NIRCAM_LONGWAVE_DETECTORS, \
-    NIRCAM_SHORTWAVE_DETECTORS, PREVIEW_IMAGE_LISTFILE, THUMBNAIL_LISTFILE
+from jwql.utils.constants import (IGNORED_SUFFIXES,
+                                  JWST_INSTRUMENT_NAMES_SHORTHAND,
+                                  NIRCAM_LONGWAVE_DETECTORS,
+                                  NIRCAM_SHORTWAVE_DETECTORS
+                                  )
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.protect_module import lock_module
 from jwql.utils.preview_image import PreviewImage
@@ -616,22 +619,6 @@ def generate_preview_images(overwrite, programs=None):
         full_preview_files.extend(r[0])
         full_thumbnail_files.extend(r[1])
 
-    # Filter the preview and thumbnail images by instrument and update the listfiles.
-    # We do this by looking for instrument abbreviations in the filenames. But will
-    # this work for level 3 files?? If an instrument abbreviation is not in the filename,
-    # then the preview/thubnail images won't be caught and added here.
-    for abbrev, inst_name in JWST_INSTRUMENT_NAMES_SHORTHAND.items():
-        inst_previews = [ele for ele in full_preview_files if re.search(abbrev, ele, re.IGNORECASE)]
-        inst_thumbs = [ele for ele in full_thumbnail_files if abbrev in ele]
-
-        # Read in the preview image listfile and the thumbnail image list file
-        # and add these new files to each
-        preview_image_listfile = os.path.join(SETTINGS['preview_image_filesystem'], f"{PREVIEW_IMAGE_LISTFILE}_{inst_name}.txt")
-        update_listfile(preview_image_listfile, inst_previews, 'preview')
-
-        thumbnail_image_listfile = os.path.join(SETTINGS['thumbnail_filesystem'], f"{THUMBNAIL_LISTFILE}_{inst_name}.txt")
-        update_listfile(thumbnail_image_listfile, inst_thumbs, 'thumbnail')
-
     # Complete logging:
     logging.info("Completed.")
 
@@ -842,43 +829,6 @@ def process_program(program, overwrite):
     return preview_image_files, thumbnail_files
 
 
-def update_listfile(filename, file_list, filetype):
-    """Add a list of files to a text file. Designed to add new files to the
-    file containing the list of all preview images and the file containing the
-    list of all thumbnail images.
-
-    Parameters
-    ----------
-    filename : str
-        Name, including path, of the file to be amended/created
-
-    file_list : list
-        List of filenames to be added to filename
-
-    filetype : str
-        Descriptor of the contents of the file being amended. Used only for
-        the logging statement
-    """
-    if len(file_list) > 0:
-        if not os.path.isfile(filename):
-            logging.warning(f"{filetype} image listfile not found!! Expected to be at {filename}. Creating a new file.")
-
-        with open(filename, 'a+') as fobj:
-            # Move read cursor to the start of file.
-            fobj.seek(0)
-
-            # If file is not empty then append '\n'
-            data = fobj.read(100)
-            if len(data) > 0:
-                fobj.write("\n")
-
-            # Append file_list at the end of file
-            for file_to_add in file_list:
-                fobj.write(f'{file_to_add}\n')
-
-        logging.info(f"{filetype} image listfile {filename} updated with new entries.")
-
-
 @lock_module
 def protected_code(overwrite, programs):
     """Protected code ensures only 1 instance of module will run at any given time