Merge pull request #804 from bhilbert4/avoid-asic-tuning-data

Add function to filter out ASIC tuning data from mast results
spacetelescope · Oct 19, 2021 · 87e22a6 · 87e22a6
2 parents 030c166 + b4934b9
commit 87e22a6
Show file tree

Hide file tree

Showing 9 changed files with 164 additions and 124 deletions.
diff --git a/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py b/jwql/instrument_monitors/common_monitors/bad_pixel_monitor.py
@@ -99,12 +99,11 @@
 from jwql.database.database_interface import NIRSpecBadPixelQueryHistory, NIRSpecBadPixelStats
 from jwql.database.database_interface import FGSBadPixelQueryHistory, FGSBadPixelStats
 from jwql.instrument_monitors import pipeline_tools
-from jwql.utils import crds_tools, instrument_properties
+from jwql.utils import crds_tools, instrument_properties, monitor_utils
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.constants import FLAT_EXP_TYPES, DARK_EXP_TYPES
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.mast_utils import mast_query
-from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table
 from jwql.utils.permissions import set_permissions
 from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path
 
@@ -1022,10 +1021,17 @@ def run(self):
                 # lists to align.
 
                 if new_flat_entries:
+                    # Exclude ASIC tuning data
+                    len_new_flats = len(new_flat_entries)
+                    new_flat_entries = monitor_utils.exclude_asic_tuning(new_flat_entries)
+                    len_no_asic = len(new_flat_entries)
+                    num_asic = len_new_flats - len_no_asic
+                    logging.info("\tFiltering out ASIC tuning files removed {} flat files.".format(num_asic))
+
                     new_flat_entries = self.filter_query_results(new_flat_entries, datatype='flat')
                     apcheck_flat_entries = pipeline_tools.aperture_size_check(new_flat_entries, instrument, aperture)
                     lost_to_bad_metadata = len(new_flat_entries) - len(apcheck_flat_entries)
-                    logging.info('{} flat field files ignored due to inconsistency in array size and metadata.'.format(lost_to_bad_metadata))
+                    logging.info('\t{} flat field files ignored due to inconsistency in array size and metadata.'.format(lost_to_bad_metadata))
                     flat_uncal_files = locate_uncal_files(apcheck_flat_entries)
                     flat_uncal_files, run_flats = check_for_sufficient_files(flat_uncal_files, instrument, aperture, flat_file_count_threshold, 'flats')
                     flat_rate_files, flat_rate_files_to_copy = locate_rate_files(flat_uncal_files)
@@ -1034,10 +1040,17 @@ def run(self):
                     flat_uncal_files, flat_rate_files, flat_rate_files_to_copy = None, None, None
 
                 if new_dark_entries:
+                    # Exclude ASIC tuning data
+                    len_new_darks = len(new_dark_entries)
+                    new_dark_entries = monitor_utils.exclude_asic_tuning(new_dark_entries)
+                    len_no_asic = len(new_dark_entries)
+                    num_asic = len_new_darks - len_no_asic
+                    logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic))
+
                     new_dark_entries = self.filter_query_results(new_dark_entries, datatype='dark')
                     apcheck_dark_entries = pipeline_tools.aperture_size_check(new_dark_entries, instrument, aperture)
                     lost_to_bad_metadata = len(new_dark_entries) - len(apcheck_dark_entries)
-                    logging.info('{} dark files ignored due to inconsistency in array size and metadata.'.format(lost_to_bad_metadata))
+                    logging.info('\t{} dark files ignored due to inconsistency in array size and metadata.'.format(lost_to_bad_metadata))
                     dark_uncal_files = locate_uncal_files(apcheck_dark_entries)
                     dark_uncal_files, run_darks = check_for_sufficient_files(dark_uncal_files, instrument, aperture, dark_file_count_threshold, 'darks')
                     dark_rate_files, dark_rate_files_to_copy = locate_rate_files(dark_uncal_files)
@@ -1092,9 +1105,9 @@ def run(self):
 if __name__ == '__main__':
 
     module = os.path.basename(__file__).strip('.py')
-    start_time, log_file = initialize_instrument_monitor(module)
+    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)
 
     monitor = BadPixels()
     monitor.run()
 
-    update_monitor_table(module, start_time, log_file)
+    monitor_utils.update_monitor_table(module, start_time, log_file)
diff --git a/jwql/instrument_monitors/common_monitors/bias_monitor.py b/jwql/instrument_monitors/common_monitors/bias_monitor.py
@@ -51,13 +51,13 @@
 from jwql.database.database_interface import session
 from jwql.database.database_interface import NIRCamBiasQueryHistory, NIRCamBiasStats, NIRISSBiasQueryHistory, NIRISSBiasStats, NIRSpecBiasQueryHistory, NIRSpecBiasStats
 from jwql.instrument_monitors import pipeline_tools
-from jwql.instrument_monitors.common_monitors.dark_monitor import mast_query_darks
-from jwql.utils import instrument_properties
+from jwql.utils import instrument_properties, monitor_utils
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.monitor_utils import update_monitor_table
 from jwql.utils.permissions import set_permissions
-from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, initialize_instrument_monitor
+from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config
+
 
 
 class Bias():
@@ -466,7 +466,15 @@ def run(self):
 
                 # Query MAST for new dark files for this instrument/aperture
                 logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
-                new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end)
+                new_entries = monitor_utils.mast_query_darks(instrument, aperture, self.query_start, self.query_end)
+
+                # Exclude ASIC tuning data
+                len_new_darks = len(new_entries)
+                new_entries = monitor_utils.exclude_asic_tuning(new_entries)
+                len_no_asic = len(new_entries)
+                num_asic = len_new_darks - len_no_asic
+                logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic))
+
                 logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries)))
 
                 # Set up a directory to store the data for this aperture
@@ -524,9 +532,9 @@ def run(self):
 if __name__ == '__main__':
 
     module = os.path.basename(__file__).strip('.py')
-    start_time, log_file = initialize_instrument_monitor(module)
+    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)
 
     monitor = Bias()
     monitor.run()
 
-    update_monitor_table(module, start_time, log_file)
+    monitor_utils.update_monitor_table(module, start_time, log_file)
diff --git a/jwql/instrument_monitors/common_monitors/dark_monitor.py b/jwql/instrument_monitors/common_monitors/dark_monitor.py
@@ -75,86 +75,16 @@
 from jwql.database.database_interface import FGSDarkQueryHistory, FGSDarkPixelStats, FGSDarkDarkCurrent
 from jwql.instrument_monitors import pipeline_tools
 from jwql.jwql_monitors import monitor_mast
-from jwql.utils import calculations, instrument_properties
-from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, RAPID_READPATTERNS
+from jwql.utils import calculations, instrument_properties, monitor_utils
+from jwql.utils.constants import ASIC_TEMPLATES, JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE, JWST_DATAPRODUCTS, \
+                                 RAPID_READPATTERNS
 from jwql.utils.logging_functions import log_info, log_fail
-from jwql.utils.monitor_utils import initialize_instrument_monitor, update_monitor_table
 from jwql.utils.permissions import set_permissions
 from jwql.utils.utils import copy_files, ensure_dir_exists, get_config, filesystem_path
 
 THRESHOLDS_FILE = os.path.join(os.path.split(__file__)[0], 'dark_monitor_file_thresholds.txt')
 
 
-def mast_query_darks(instrument, aperture, start_date, end_date, readpatt=None):
-    """Use ``astroquery`` to search MAST for dark current data
-
-    Parameters
-    ----------
-    instrument : str
-        Instrument name (e.g. ``nircam``)
-
-    aperture : str
-        Detector aperture to search for (e.g. ``NRCA1_FULL``)
-
-    start_date : float
-        Starting date for the search in MJD
-
-    end_date : float
-        Ending date for the search in MJD
-
-    readpatt : str
-        Readout pattern to search for (e.g. ``RAPID``). If None,
-        readout pattern will not be added to the query parameters.
-
-    Returns
-    -------
-    query_results : list
-        List of dictionaries containing the query results
-    """
-
-    # Make sure instrument is correct case
-    if instrument.lower() == 'nircam':
-        instrument = 'NIRCam'
-        dark_template = ['NRC_DARK']
-    elif instrument.lower() == 'niriss':
-        instrument = 'NIRISS'
-        dark_template = ['NIS_DARK']
-    elif instrument.lower() == 'nirspec':
-        instrument = 'NIRSpec'
-        dark_template = ['NRS_DARK']
-    elif instrument.lower() == 'fgs':
-        instrument = 'FGS'
-        dark_template = ['FGS_DARK']
-    elif instrument.lower() == 'miri':
-        instrument = 'MIRI'
-        dark_template = ['MIR_DARKALL', 'MIR_DARKIMG', 'MIR_DARKMRS']
-
-    # monitor_mast.instrument_inventory does not allow list inputs to
-    # the added_filters input (or at least if you do provide a list, then
-    # it becomes a nested list when it sends the query to MAST. The
-    # nested list is subsequently ignored by MAST.)
-    # So query once for each dark template, and combine outputs into a
-    # single list.
-    query_results = []
-    for template_name in dark_template:
-
-        # Create dictionary of parameters to add
-        parameters = {"date_obs_mjd": {"min": start_date, "max": end_date},
-                      "apername": aperture, "exp_type": template_name,
-                     }
-
-        if readpatt is not None:
-            parameters["readpatt"] = readpatt
-
-        query = monitor_mast.instrument_inventory(instrument, dataproduct=JWST_DATAPRODUCTS,
-                                                  add_filters=parameters, return_data=True, caom=False)
-        if 'data' in query.keys():
-            if len(query['data']) > 0:
-                query_results.extend(query['data'])
-
-    return query_results
-
-
 class Dark():
     """Class for executing the dark current monitor.
 
@@ -768,7 +698,15 @@ def run(self):
 
                     # Query MAST using the aperture and the time of the
                     # most recent previous search as the starting time
-                    new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end, readpatt=self.readpatt)
+                    new_entries = monitor_utils.mast_query_darks(instrument, aperture, self.query_start, self.query_end, readpatt=self.readpatt)
+
+                    # Exclude ASIC tuning data
+                    len_new_darks = len(new_entries)
+                    new_entries = monitor_utils.exclude_asic_tuning(new_entries)
+                    len_no_asic = len(new_entries)
+                    num_asic = len_new_darks - len_no_asic
+                    logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic))
+
                     logging.info('\tAperture: {}, Readpattern: {}, new entries: {}'.format(self.aperture, self.readpatt,
                                                                                            len(new_entries)))
 
@@ -1087,9 +1025,9 @@ def stats_by_amp(self, image, amps):
 if __name__ == '__main__':
 
     module = os.path.basename(__file__).strip('.py')
-    start_time, log_file = initialize_instrument_monitor(module)
+    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)
 
     monitor = Dark()
     monitor.run()
 
-    update_monitor_table(module, start_time, log_file)
+    monitor_utils.update_monitor_table(module, start_time, log_file)
diff --git a/jwql/instrument_monitors/common_monitors/readnoise_monitor.py b/jwql/instrument_monitors/common_monitors/readnoise_monitor.py
@@ -56,13 +56,12 @@
 from jwql.database.database_interface import NIRSpecReadnoiseQueryHistory, NIRSpecReadnoiseStats
 from jwql.database.database_interface import session
 from jwql.instrument_monitors import pipeline_tools
-from jwql.instrument_monitors.common_monitors.dark_monitor import mast_query_darks
-from jwql.utils import instrument_properties
+from jwql.utils import instrument_properties, monitor_utils
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_INSTRUMENT_NAMES_MIXEDCASE
 from jwql.utils.logging_functions import log_info, log_fail
 from jwql.utils.monitor_utils import update_monitor_table
 from jwql.utils.permissions import set_permissions
-from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config, initialize_instrument_monitor
+from jwql.utils.utils import ensure_dir_exists, filesystem_path, get_config
 
 
 class Readnoise():
@@ -554,7 +553,15 @@ def run(self):
 
                 # Query MAST for new dark files for this instrument/aperture
                 logging.info('\tQuery times: {} {}'.format(self.query_start, self.query_end))
-                new_entries = mast_query_darks(instrument, aperture, self.query_start, self.query_end)
+                new_entries = monitor_utils.mast_query_darks(instrument, aperture, self.query_start, self.query_end)
+
+                # Exclude ASIC tuning data
+                len_new_darks = len(new_entries)
+                new_entries = monitor_utils.exclude_asic_tuning(new_entries)
+                len_no_asic = len(new_entries)
+                num_asic = len_new_darks - len_no_asic
+                logging.info("\tFiltering out ASIC tuning files removed {} dark files.".format(num_asic))
+
                 logging.info('\tAperture: {}, new entries: {}'.format(self.aperture, len(new_entries)))
 
                 # Set up a directory to store the data for this aperture
@@ -624,9 +631,9 @@ def run(self):
 if __name__ == '__main__':
 
     module = os.path.basename(__file__).strip('.py')
-    start_time, log_file = initialize_instrument_monitor(module)
+    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)
 
     monitor = Readnoise()
     monitor.run()
 
-    update_monitor_table(module, start_time, log_file)
+    monitor_utils.update_monitor_table(module, start_time, log_file)
diff --git a/jwql/jwql_monitors/monitor_mast.py b/jwql/jwql_monitors/monitor_mast.py
@@ -29,8 +29,8 @@
 from jwql.utils.constants import JWST_INSTRUMENT_NAMES, JWST_DATAPRODUCTS
 from jwql.utils.logging_functions import configure_logging, log_info, log_fail
 from jwql.utils.permissions import set_permissions
-from jwql.utils.utils import get_config, initialize_instrument_monitor
-from jwql.utils.monitor_utils import update_monitor_table
+from jwql.utils.utils import get_config
+from jwql.utils import monitor_utils
 from jwql.utils.plotting import bar_chart
 
 
@@ -265,8 +265,8 @@ def monitor_mast():
 
     # Configure logging
     module = os.path.basename(__file__).strip('.py')
-    start_time, log_file = initialize_instrument_monitor(module)
+    start_time, log_file = monitor_utils.initialize_instrument_monitor(module)
 
     # Run the monitors
     monitor_mast()
-    update_monitor_table(module, start_time, log_file)
+    monitor_utils.update_monitor_table(module, start_time, log_file)
diff --git a/jwql/tests/test_dark_monitor.py b/jwql/tests/test_dark_monitor.py
@@ -24,6 +24,7 @@
 import numpy as np
 
 from jwql.instrument_monitors.common_monitors import dark_monitor
+from jwql.utils.monitor_utils import mast_query_darks
 from jwql.utils.utils import get_config
 
 ON_GITHUB_ACTIONS = '/home/runner' in os.path.expanduser('~') or '/Users/runner' in os.path.expanduser('~')
@@ -79,7 +80,7 @@ def test_mast_query_darks():
     readpatt = 'BRIGHT2'
     start_date = Time("2016-01-01T00:00:00").mjd
     end_date = Time("2018-01-01T00:00:00").mjd
-    query = dark_monitor.mast_query_darks(instrument, aperture, readpatt, start_date, end_date)
+    query = mast_query_darks(instrument, aperture, readpatt, start_date, end_date)
     apernames = [entry['apername'] for entry in query]
     filenames = [entry['filename'] for entry in query]
 

diff --git a/jwql/utils/constants.py b/jwql/utils/constants.py
@@ -117,6 +117,10 @@
                                      'MIRIM_BRIGHTSKY', 'MIRIM_SLITLESSPRISM'],
                             'FGS': ['FGS1_FULL', 'FGS2_FULL']}
 
+# Observing templates used for ASIC tuning. MAST query results that
+# have one of these templates will be ignored
+ASIC_TEMPLATES = ['ISIM ASIC Tuning']
+
 # Bad pixel types by the type of data used to find them
 BAD_PIXEL_TYPES = ['DEAD', 'HOT', 'LOW_QE', 'RC', 'OPEN', 'ADJ_OPEN', 'TELEGRAPH', 'OTHER_BAD_PIXEL']
 DARKS_BAD_PIXEL_TYPES = ['HOT', 'RC', 'OTHER_BAD_PIXEL', 'TELEGRAPH']