Add function to filter out ASIC tuning data from mast results

jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py

@@ -99,6 +99,7 @@
 from jwql.database.database_interface import NIRSpecBadPixelQueryHistory, NIRSpecBadPixelStats
 from jwql.database.database_interface import FGSBadPixelQueryHistory, FGSBadPixelStats
 from jwql.instrument_monitors import pipeline_tools
+from jwql.instrument_monitors.common_monitors.dark_monitor import exclude_asic_tuning
 from jwql.utils import crds_tools, instrument_properties
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.constants import FLAT_EXP_TYPES, DARK_EXP_TYPES
@@ -1022,10 +1023,17 @@ def run(self):
                 # lists to align.
 
                 if new_flat_entries:
+                    # Exclude ASIC tuning data
+                    len_new_flats = len(new_flat_entries)
+                    new_flat_entries = exclude_asic_tuning(new_flat_entries)
+                    len_no_asic = len(new_flat_entries)
+                    num_asic = len_new_flats - len_no_asic
+                    logging.info("\tFiltering out ASIC tuning files removed {} flat files.".format(num_asic))
+
                     new_flat_entries = self.filter_query_results(new_flat_entries, datatype='flat')
                     apcheck_flat_entries = pipeline_tools.aperture_size_check(new_flat_entries, instrument, aperture)
                     lost_to_bad_metadata = len(new_flat_entries) - len(apcheck_flat_entries)
-                    logging.info('{} flat field files ignored due to inconsistency in array size and metadata.'.format(lost_to_bad_metadata))
+                    logging.info('\t{} flat field files ignored due to inconsistency in array size and metadata.'.format(lost_to_bad_metadata))
                     flat_uncal_files = locate_uncal_files(apcheck_flat_entries)
                     flat_uncal_files, run_flats = check_for_sufficient_files(flat_uncal_files, instrument, aperture, flat_file_count_threshold, 'flats')
                     flat_rate_files, flat_rate_files_to_copy = locate_rate_files(flat_uncal_files)
@@ -1034,10 +1042,17 @@ def run(self):
                     flat_uncal_files, flat_rate_files, flat_rate_files_to_copy = None, None, None
 
                 if new_dark_entries:
+                    # Exclude ASIC tuning data
+                    len_new_darks = len(new_dark_entries)
+                    new_dark_entries = exclude_asic_tuning(new_dark_entries)
+                    len_no_asic = len(new_dark_entries)
+                    num_asic = len_new_darks - len_no_asic
+                    logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic))
+
                     new_dark_entries = self.filter_query_results(new_dark_entries, datatype='dark')
                     apcheck_dark_entries = pipeline_tools.aperture_size_check(new_dark_entries, instrument, aperture)
                     lost_to_bad_metadata = len(new_dark_entries) - len(apcheck_dark_entries)
-                    logging.info('{} dark files ignored due to inconsistency in array size and metadata.'.format(lost_to_bad_metadata))
+                    logging.info('\t{} dark files ignored due to inconsistency in array size and metadata.'.format(lost_to_bad_metadata))
                     dark_uncal_files = locate_uncal_files(apcheck_dark_entries)
                     dark_uncal_files, run_darks = check_for_sufficient_files(dark_uncal_files, instrument, aperture, dark_file_count_threshold, 'darks')
                     dark_rate_files, dark_rate_files_to_copy = locate_rate_files(dark_uncal_files)

jwql/instrument_monitors/common_monitors/bias_monitor.py

@@ -51,7 +51,7 @@
 from jwql.database.database_interface import session
 from jwql.database.database_interface import NIRCamBiasQueryHistory, NIRCamBiasStats, NIRISSBiasQueryHistory, NIRISSBiasStats, NIRSpecBiasQueryHistory, NIRSpecBiasStats
 from jwql.instrument_monitors import pipeline_tools
-from jwql.instrument_monitors.common_monitors.dark_monitor import mast_query_darks
+from jwql.instrument_monitors.common_monitors.dark_monitor import exclude_asic_tuning, mast_query_darks
 from jwql.utils import instrument_properties
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.logging_functions import log_info, log_fail
@@ -467,6 +467,14 @@ def run(self):
                 # Query MAST for new dark files for this instrument/aperture
                 logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
                 new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end)
+
+                # Exclude ASIC tuning data
+                len_new_darks = len(new_entries)
+                new_entries = exclude_asic_tuning(new_entries)
+                len_no_asic = len(new_entries)
+                num_asic = len_new_darks - len_no_asic
+                logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic))
+
                 logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries)))
 
                 # Set up a directory to store the data for this aperture

jwql/instrument_monitors/common_monitors/dark_monitor.py

@@ -76,7 +76,8 @@
 from jwql.instrument_monitors import pipeline_tools
 from jwql.jwql_monitors import monitor_mast
 from jwql.utils import calculations, instrument_properties
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS
+from jwql.utils.constants import ASIC_TEMPLATES, JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, \
+                                 RAPID_READPATTERNS
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table
 from jwql.utils.permissions import set_permissions
@@ -85,6 +86,28 @@
 THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'dark_monitor_file_thresholds.txt')
 
 
+def exclude_asic_tuning(mast_results):
+    """Given a list of file information from a MAST query, filter out
+    files taken during ASIC tuning, which will have bad data in terms
+    of results for the instrument monitors.
+
+    Parameters
+    ----------
+    mast_results : list
+        List of dictionaries containing a MAST query result
+
+    Returns
+    -------
+    filtered_results : list
+        Modified list with ASIC tuning entries removed
+    """
+    filtered_results = []
+    for mast_result in mast_results:
+        if mast_result['template'] not in ASIC_TEMPLATES:
+            filtered_results.append(mast_result)
+    return filtered_results
+
+
 def mast_query_darks(instrument, aperture, start_date, end_date, readpatt=None):
     """Use ``astroquery`` to search MAST for dark current data
 
@@ -769,6 +792,14 @@ def run(self):
                     # Query MAST using the aperture and the time of the
                     # most recent previous search as the starting time
                     new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end, readpatt=self.readpatt)
+
+                    # Exclude ASIC tuning data
+                    len_new_darks = len(new_entries)
+                    new_entries = exclude_asic_tuning(new_entries)
+                    len_no_asic = len(new_entries)
+                    num_asic = len_new_darks - len_no_asic
+                    logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic))
+
                     logging.info('\tAperture: {}, Readpattern: {}, new entries: {}'.format(self.aperture, self.readpatt,
                                                                                            len(new_entries)))
 

jwql/instrument_monitors/common_monitors/readnoise_monitor.py

@@ -56,7 +56,7 @@
 from jwql.database.database_interface import NIRSpecReadnoiseQueryHistory, NIRSpecReadnoiseStats
 from jwql.database.database_interface import session
 from jwql.instrument_monitors import pipeline_tools
-from jwql.instrument_monitors.common_monitors.dark_monitor import mast_query_darks
+from jwql.instrument_monitors.common_monitors.dark_monitor import exclude_asic_tuning, mast_query_darks
 from jwql.utils import instrument_properties
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.logging_functions import log_info, log_fail
@@ -555,6 +555,14 @@ def run(self):
                 # Query MAST for new dark files for this instrument/aperture
                 logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
                 new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end)
+
+                # Exclude ASIC tuning data
+                len_new_darks = len(new_entries)
+                new_entries = exclude_asic_tuning(new_entries)
+                len_no_asic = len(new_entries)
+                num_asic = len_new_darks - len_no_asic
+                logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic))
+
                 logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries)))
 
                 # Set up a directory to store the data for this aperture

jwql/utils/constants.py

@@ -117,6 +117,10 @@
                                      'MIRIM_BRIGHTSKY', 'MIRIM_SLITLESSPRISM'],
                             'FGS': ['FGS1_FULL', 'FGS2_FULL']}
 
+# Observing templates used for ASIC tuning. MAST query results that
+# have one of these templates will be ignored
+ASIC_TEMPLATES = ['ISIM ASIC Tuning']
+
 # Bad pixel types by the type of data used to find them
 BAD_PIXEL_TYPES = ['DEAD', 'HOT', 'LOW_QE', 'RC', 'OPEN', 'ADJ_OPEN', 'TELEGRAPH', 'OTHER_BAD_PIXEL']
 DARKS_BAD_PIXEL_TYPES = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH']