From 138043dcef55acda364d93684d2e6c6d6d6b679f Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 10 Jan 2019 23:36:40 -0500 Subject: [PATCH 01/10] Use proseco for a quasi-independent hot pix check --- starcheck/data/ACABadPixels | 252 ----------------------- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 60 +++++- 2 files changed, 58 insertions(+), 254 deletions(-) diff --git a/starcheck/data/ACABadPixels b/starcheck/data/ACABadPixels index 5e51aeb2..735e0a38 100644 --- a/starcheck/data/ACABadPixels +++ b/starcheck/data/ACABadPixels @@ -1,255 +1,3 @@ -Bad Pixel List from 2017272 2018002 2018100 Dark Currents - Star.Body.Pixels.BadPixels = [ ... - -512, 511, -1, 0; ... - -1, 0, -512, 511; ... -245, 0, 454, 454; ... - -511, -511, 9, 9; ... - -510, -510, -289, -289; ... - -510, -510, 158, 158; ... - -510, -510, 445, 445; ... - -509, -509, -194, -194; ... - -506, -506, 150, 150; ... - -497, -497, -366, -366; ... - -495, -495, 207, 207; ... - -491, -491, -435, -435; ... - -480, -480, -427, -427; ... - -476, -476, 83, 83; ... - -464, -464, -341, -341; ... - -464, -464, -96, -96; ... - -461, -461, -445, -445; ... - -461, -461, 292, 292; ... - -457, -457, -14, -14; ... - -451, -451, 380, 380; ... - -449, -449, 268, 268; ... - -447, -447, 244, 244; ... - -443, -443, -64, -64; ... - -441, -441, 86, 86; ... - -426, -426, -159, -159; ... - -417, -417, 474, 474; ... - -416, -416, -40, -40; ... - -415, -415, 28, 28; ... - -414, -414, -42, -42; ... - -411, -411, 22, 22; ... - -405, -405, -179, -179; ... - -395, -395, 462, 462; ... - -390, -390, -371, -371; ... - -390, -390, -335, -335; ... - -390, -390, 90, 90; ... - -383, -383, -247, -247; ... - -382, -382, 328, 328; ... - -381, -381, -247, -247; ... - -375, -375, -403, -403; ... - -372, -372, -50, -50; ... - -372, -372, -40, -40; ... - -370, -370, 126, 126; ... - -364, -364, -229, -229; ... - -364, -364, 249, 249; ... - -359, -359, 409, 409; ... - -355, -355, -132, -132; ... - -354, -354, 387, 387; ... - -353, -353, -486, -486; ... - -353, -353, -422, -422; ... - -352, -352, 226, 226; ... - -347, -347, 29, 29; ... - -338, -338, -60, -60; ... - -333, -333, 16, 16; ... - -333, -333, 387, 387; ... - -316, -316, -46, -46; ... - -311, -311, -329, -329; ... - -302, -302, 237, 237; ... - -301, -301, 391, 391; ... - -297, -297, 421, 421; ... - -296, -296, -459, -459; ... - -295, -295, -311, -311; ... - -286, -286, 350, 350; ... - -283, -283, -315, -315; ... - -281, -281, -358, -358; ... - -272, -272, -12, -12; ... - -267, -267, -325, -325; ... - -265, -265, 466, 466; ... - -260, -260, -24, -24; ... - -259, -259, 59, 59; ... - -259, -259, 122, 122; ... - -258, -258, -491, -491; ... - -255, -255, -251, -251; ... - -252, -252, -485, -485; ... - -252, -252, -120, -120; ... - -252, -252, 32, 32; ... - -241, -241, -416, -416; ... - -238, -238, 93, 93; ... - -235, -235, -158, -158; ... - -231, -231, -399, -399; ... - -231, -231, 487, 487; ... - -220, -220, 438, 438; ... - -217, -217, 207, 207; ... - -215, -215, -414, -414; ... - -214, -214, -296, -296; ... - -213, -213, 340, 340; ... - -210, -210, -351, -351; ... - -200, -200, 392, 392; ... - -196, -196, 38, 38; ... - -195, -195, 304, 304; ... - -194, -194, -314, -314; ... - -191, -191, -326, -326; ... - -186, -186, -365, -365; ... - -174, -174, -84, -84; ... - -174, -174, 367, 367; ... - -161, -161, 150, 150; ... - -157, -157, -267, -267; ... - -154, -154, -200, -200; ... - -152, -152, -413, -413; ... - -145, -145, 159, 159; ... - -144, -144, -80, -80; ... - -142, -142, 236, 236; ... - -137, -137, 98, 98; ... - -126, -126, 85, 85; ... - -123, -123, 133, 133; ... - -122, -122, -267, -267; ... - -111, -111, 381, 381; ... - -108, -108, 441, 441; ... - -103, -103, -234, -234; ... - -100, -100, 229, 229; ... - -100, -100, 470, 470; ... - -99, -99, 362, 362; ... - -98, -98, 106, 106; ... - -98, -98, 377, 377; ... - -97, -97, -321, -321; ... - -96, -96, -119, -119; ... - -92, -92, 214, 214; ... - -85, -85, 170, 170; ... - -83, -83, -432, -432; ... - -83, -83, -3, -3; ... - -64, -64, -83, -83; ... - -63, -63, -174, -174; ... - -62, -62, -212, -212; ... - -56, -56, -325, -325; ... - -56, -56, 164, 164; ... - -47, -47, 180, 180; ... - -44, -44, -478, -478; ... - -42, -42, 11, 11; ... - -39, -39, -205, -205; ... - -35, -35, 408, 408; ... - -34, -34, 423, 423; ... - -31, -31, -342, -342; ... - -20, -20, -493, -493; ... - -10, -10, 93, 93; ... - -8, -8, 458, 458; ... - -2, -2, 211, 211; ... - 1, 1, -424, -424; ... - 1, 1, -156, -156; ... - 2, 2, 382, 382; ... - 10, 10, 299, 299; ... - 12, 12, -68, -68; ... - 19, 19, 314, 314; ... - 25, 25, 108, 108; ... - 25, 25, 111, 111; ... - 27, 27, -204, -204; ... - 28, 28, 440, 440; ... - 40, 40, -332, -332; ... - 45, 45, 174, 174; ... - 46, 46, 136, 136; ... - 47, 47, 97, 97; ... - 57, 57, -228, -228; ... - 68, 68, -301, -301; ... - 92, 92, 326, 326; ... - 101, 101, -250, -250; ... - 114, 114, 340, 340; ... - 122, 122, -433, -433; ... - 126, 126, -328, -328; ... - 126, 126, -29, -29; ... - 128, 128, -461, -461; ... - 130, 130, 249, 249; ... - 131, 131, 127, 127; ... - 132, 132, -344, -344; ... - 134, 134, -3, -3; ... - 137, 137, -452, -452; ... - 138, 138, 9, 9; ... - 139, 139, -140, -140; ... - 139, 139, 309, 309; ... - 141, 141, 61, 61; ... - 142, 142, 254, 254; ... - 143, 143, -209, -209; ... - 144, 144, 226, 226; ... - 148, 148, 26, 26; ... - 149, 149, 508, 508; ... - 152, 152, 496, 496; ... - 155, 155, 151, 151; ... - 158, 158, -463, -463; ... - 160, 160, 86, 86; ... - 162, 162, -355, -355; ... - 162, 162, -203, -203; ... - 165, 165, -240, -240; ... - 171, 171, -362, -362; ... - 174, 174, 261, 261; ... - 186, 186, -491, -491; ... - 192, 192, 211, 211; ... - 193, 193, -246, -246; ... - 199, 199, -412, -412; ... - 203, 203, -452, -452; ... - 214, 214, -26, -26; ... - 223, 223, 402, 402; ... - 226, 226, 110, 110; ... - 228, 228, -29, -29; ... - 229, 229, -495, -495; ... - 230, 230, 26, 26; ... - 246, 246, 338, 338; ... - 247, 247, -350, -350; ... - 252, 252, -254, -254; ... - 253, 253, -365, -365; ... - 256, 256, -62, -62; ... - 269, 269, 330, 330; ... - 270, 270, -333, -333; ... - 271, 271, 272, 272; ... - 274, 274, -427, -427; ... - 281, 281, 241, 241; ... - 285, 285, 99, 99; ... - 286, 286, 127, 127; ... - 289, 289, -409, -409; ... - 293, 293, -150, -150; ... - 294, 294, -438, -438; ... - 309, 309, -342, -342; ... - 315, 315, -174, -174; ... - 319, 319, -512, -512; ... - 320, 320, 190, 190; ... - 321, 321, -416, -416; ... - 321, 321, 361, 361; ... - 324, 324, -18, -18; ... - 326, 326, 12, 12; ... - 330, 330, 390, 390; ... - 337, 337, 43, 43; ... - 342, 342, -281, -281; ... - 342, 342, 6, 6; ... - 349, 349, 369, 369; ... - 358, 358, 419, 419; ... - 364, 364, -303, -303; ... - 369, 369, 184, 184; ... - 372, 372, 379, 379; ... - 374, 374, -419, -419; ... - 374, 374, -95, -95; ... - 379, 379, 279, 279; ... - 385, 385, -506, -506; ... - 392, 392, -32, -32; ... - 393, 393, 254, 254; ... - 394, 394, -323, -323; ... - 396, 396, 283, 283; ... - 418, 418, -505, -505; ... - 422, 422, -1, -1; ... - 427, 427, 38, 38; ... - 431, 431, 300, 300; ... - 437, 437, -262, -262; ... - 440, 440, 5, 5; ... - 443, 443, -416, -416; ... - 446, 446, 305, 305; ... - 456, 456, 466, 466; ... - 461, 461, -199, -199; ... - 465, 465, -242, -242; ... - 476, 476, 403, 403; ... - 477, 477, -6, -6; ... - 491, 491, -266, -266; ... - 493, 493, 113, 113; ... - 494, 494, -209, -209; ... - 501, 501, -367, -367; ... - 507, 507, 49, 49; ... - 508, 508, 349, 349; ... ]; diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index a0af665c..ea7fa7e0 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -23,14 +23,47 @@ use warnings; use Inline Python => q{ import numpy as np +from astropy.table import Table + +from mica.archive import aca_dark from chandra_aca.star_probs import guide_count +from chandra_aca.transform import yagzag_to_pixels, count_rate_to_mag import Quaternion from Ska.quatutil import radec2yagzag import agasc +from proseco.core import ACABox +from proseco.guide import get_imposter_mags + def _guide_count(mags, t_ccd): return float(guide_count(np.array(mags), t_ccd)) +def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z, dmag=1.0): + + dark = aca_dark.get_dark_cal_image(date=date.decode('ascii'), + t_ccd_ref=float(t_ccd), aca_image=True) + + guide_stars = [] + for i, y, z, mag, ctype in zip(idxs, yags, zags, mags, types): + ctype = ctype.decode('ascii') + if ctype == 'BOT' or ctype == 'GUI': + star_row, star_col = yagzag_to_pixels(float(y), float(z)) + guide_stars.append({'row': star_row, + 'col': star_col, + 'mag': mag, + 'i': i}) + + guide_stars = Table(guide_stars) + imp_mags, imp_rows, imp_cols = get_imposter_mags(guide_stars, dark, ACABox((dither_y, dither_z))) + + fails = [] + for star, imp_mag, r, c in zip(guide_stars, imp_mags, imp_rows, imp_cols): + if imp_mag < (star['mag'] + dmag): + fails.append({'i': int(star['i']), 'star_row': float(star['row']), 'star_col': float(star['col']), + 'bad2_row': float(r), 'bad2_col': float(c), 'bad2_mag': float(imp_mag)}) + + return fails + def _get_agasc_stars(ra, dec, roll, radius, date, agasc_file): """ @@ -195,10 +228,8 @@ sub set_ACA_bad_pixels { my $pixel_file = shift; my @tmp = io($pixel_file)->slurp; my @lines = grep { /^\s+(\d|-)/ } @tmp; - splice(@lines, 0, 2); # the first two lines are quadrant boundaries foreach (@lines) { my @line = split /;|,/, $_; - #cut out the quadrant boundaries foreach my $i ($line[0]..$line[1]) { foreach my $j ($line[2]..$line[3]) { my $pixel = {'row' => $i, @@ -1239,6 +1270,31 @@ sub check_star_catalog { # store a list of the fid positions my @fid_positions = map {{'y' => $c->{"YANG$_"}, 'z' => $c->{"ZANG$_"}}} @{$self->{fid}}; + + # Make arrays of the items that we need for the hot pixel region check + my (@idxs, @yags, @zags, @mags, @types); + foreach my $i (1..16){ + if ($c->{"TYPE$i"} =~ /BOT|GUI|FID/){ + push @idxs, $i; + push @yags, $c->{"YANG$i"}; + push @zags, $c->{"ZANG$i"}; + # Add zero to get items that look more like float values in the arrays + push @mags, ($c->{"GS_MAG$i"} eq '---') ? 13.94 : $c->{"GS_MAG$i"} + 0.0; + push @types, $c->{"TYPE$i"}; + } + } + + # Run the hot pixel region check on the Python side on FID|GUI|BOT + my @imposters = check_hot_pix(\@idxs, \@yags, \@zags, \@mags, \@types, + $self->{ccd_temp}, $self->{date}, $dither_guide_y, $dither_guide_z); + # Assign warnings based on those hot pixel region checks + for my $imposter (@imposters){ + push @warn, sprintf( + "$alarm [%2d] Imposter 2x2. psmag %.1f (row % 4d, col % 4d) star (% 4d, % 4d) \n", + $imposter->{i}, $imposter->{bad2_mag}, $imposter->{bad2_row}, $imposter->{bad2_col}, + $imposter->{star_row}, $imposter->{star_col}); + } + foreach my $i (1..16) { (my $sid = $c->{"GS_ID$i"}) =~ s/[\s\*]//g; my $type = $c->{"TYPE$i"}; From c296d563cd3a7d56bfa04a3cc3547face5c54d34 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Sun, 13 Jan 2019 08:20:44 -0500 Subject: [PATCH 02/10] Set hot pix check to work with fid lights --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 33 +++++++++++------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index ea7fa7e0..58339454 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -42,25 +42,22 @@ def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z dark = aca_dark.get_dark_cal_image(date=date.decode('ascii'), t_ccd_ref=float(t_ccd), aca_image=True) - - guide_stars = [] + fails = [] for i, y, z, mag, ctype in zip(idxs, yags, zags, mags, types): ctype = ctype.decode('ascii') - if ctype == 'BOT' or ctype == 'GUI': - star_row, star_col = yagzag_to_pixels(float(y), float(z)) - guide_stars.append({'row': star_row, - 'col': star_col, - 'mag': mag, - 'i': i}) - - guide_stars = Table(guide_stars) - imp_mags, imp_rows, imp_cols = get_imposter_mags(guide_stars, dark, ACABox((dither_y, dither_z))) - - fails = [] - for star, imp_mag, r, c in zip(guide_stars, imp_mags, imp_rows, imp_cols): - if imp_mag < (star['mag'] + dmag): - fails.append({'i': int(star['i']), 'star_row': float(star['row']), 'star_col': float(star['col']), - 'bad2_row': float(r), 'bad2_col': float(c), 'bad2_mag': float(imp_mag)}) + if ctype in ['BOT', 'GUI', 'FID']: + if ctype in ['BOT', 'GUI']: + dither = ACABox((float(dither_y), float(dither_z))) + else: + dither = ACABox((5.0, 5.0)) + row, col = yagzag_to_pixels(float(y), float(z)) + entry = Table([{'i': int(i), 'row': row, 'col': col, 'mag': float(mag), 'type': ctype}]) + imp_mags, imp_rows, imp_cols = get_imposter_mags(entry, dark, dither) + if imp_mags[0] < (entry['mag'] + dmag): + fails.append({'i': int(entry['i']), + 'entry_row': float(entry['row']), 'entry_col': float(entry['col']), + 'bad2_row': float(imp_rows[0]), 'bad2_col': float(imp_cols[0]), + 'bad2_mag': float(imp_mags[0])}) return fails @@ -1292,7 +1289,7 @@ sub check_star_catalog { push @warn, sprintf( "$alarm [%2d] Imposter 2x2. psmag %.1f (row % 4d, col % 4d) star (% 4d, % 4d) \n", $imposter->{i}, $imposter->{bad2_mag}, $imposter->{bad2_row}, $imposter->{bad2_col}, - $imposter->{star_row}, $imposter->{star_col}); + $imposter->{entry_row}, $imposter->{entry_col}); } foreach my $i (1..16) { From 65d5f6d5836c6d23266eadf8042f89edc5a29b1f Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Tue, 15 Jan 2019 11:34:58 -0500 Subject: [PATCH 03/10] Add a docstring for check_hot_pix --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 26 +++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index 58339454..120b8aeb 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -39,9 +39,33 @@ def _guide_count(mags, t_ccd): return float(guide_count(np.array(mags), t_ccd)) def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z, dmag=1.0): + """ + Return a list of info to make warnings on guide stars or fid lights with local dark map that gives an + 'imposter_mag' that is brighter than `dmag` fainter than the candidate (star or fid). + As in for a 7th mag star, set an entry in the fails list if there is an imposter brighter than (7 + dmag), where + by default dmag is 1.0 (so warn on imposter mag brighter than 8.0 mag) + + This fetches the dark current before the date of the observation and passes it to + proseco.get_imposter_mags with the star candidate positions to fetch the brightest + 2x2 for each and calculates the mag for that region. If the imposter mag is brighter + than the candidate plus dmag, warn. + + :param idxs: catalog indexes as list or array + :param yags: catalog yangs as list or array + :param zags: catalog zangs as list or array + :param mags: catalog mags (AGASC mags for stars estimated fid mags for fids) list or array + :param types: catalog TYPE (ACQ|BOT|FID|MON|GUI) as list or array + :param date: observation date (bytestring via Inline) + :param dither_y: dither_y in arcsecs (guide dither) + :param dither_z: dither_z in arcsecs (guide dither) + :param dmag: delta mag for warning (default 1.0 mag) + :return fails: list of dictionaries with keys that define the index, the imposter mag, and star or fid + info to make a warning. + """ + dark = aca_dark.get_dark_cal_image(date=date.decode('ascii'), - t_ccd_ref=float(t_ccd), aca_image=True) + t_ccd_ref=float(t_ccd), aca_image=True) fails = [] for i, y, z, mag, ctype in zip(idxs, yags, zags, mags, types): ctype = ctype.decode('ascii') From 28fda24c262a0e79df2579e21b628a6e6390ce47 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Tue, 15 Jan 2019 21:53:22 -0500 Subject: [PATCH 04/10] Update check_hot_pix for style --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 29 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index 120b8aeb..f7101f2d 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -55,6 +55,7 @@ def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z :param zags: catalog zangs as list or array :param mags: catalog mags (AGASC mags for stars estimated fid mags for fids) list or array :param types: catalog TYPE (ACQ|BOT|FID|MON|GUI) as list or array + :param t_ccd: observation t_ccd in deg C (should be max t_ccd in guide phase) :param date: observation date (bytestring via Inline) :param dither_y: dither_y in arcsecs (guide dither) :param dither_z: dither_z in arcsecs (guide dither) @@ -63,22 +64,30 @@ def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z info to make a warning. """ - - dark = aca_dark.get_dark_cal_image(date=date.decode('ascii'), - t_ccd_ref=float(t_ccd), aca_image=True) + idxs = [int(idx) for idx in idxs] + yags = [float(yag) for yag in yags] + zags = [float(zag) for zag in zags] + mags = [float(mag) for mag in mags] + types = [t.decode('ascii') for t in types] + t_ccd = float(t_ccd) + date = date.decode('ascii') + dither_y = float(dither_y) + dither_z = float(dither_z) + + dark = aca_dark.get_dark_cal_image(date=date, t_ccd_ref=t_ccd, aca_image=True) fails = [] - for i, y, z, mag, ctype in zip(idxs, yags, zags, mags, types): - ctype = ctype.decode('ascii') + for idx, yag, zag, mag, ctype in zip(idxs, yags, zags, mags, types): if ctype in ['BOT', 'GUI', 'FID']: if ctype in ['BOT', 'GUI']: - dither = ACABox((float(dither_y), float(dither_z))) + dither = ACABox((dither_y, dither_z)) else: dither = ACABox((5.0, 5.0)) - row, col = yagzag_to_pixels(float(y), float(z)) - entry = Table([{'i': int(i), 'row': row, 'col': col, 'mag': float(mag), 'type': ctype}]) + row, col = yagzag_to_pixels(yag, zag) + entries = Table([{'idx': idx, 'row': row, 'col': col, 'mag': mag, 'type': ctype}]) + entry = entries[0] imp_mags, imp_rows, imp_cols = get_imposter_mags(entry, dark, dither) if imp_mags[0] < (entry['mag'] + dmag): - fails.append({'i': int(entry['i']), + fails.append({'idx': int(entry['idx']), 'entry_row': float(entry['row']), 'entry_col': float(entry['col']), 'bad2_row': float(imp_rows[0]), 'bad2_col': float(imp_cols[0]), 'bad2_mag': float(imp_mags[0])}) @@ -1312,7 +1321,7 @@ sub check_star_catalog { for my $imposter (@imposters){ push @warn, sprintf( "$alarm [%2d] Imposter 2x2. psmag %.1f (row % 4d, col % 4d) star (% 4d, % 4d) \n", - $imposter->{i}, $imposter->{bad2_mag}, $imposter->{bad2_row}, $imposter->{bad2_col}, + $imposter->{idx}, $imposter->{bad2_mag}, $imposter->{bad2_row}, $imposter->{bad2_col}, $imposter->{entry_row}, $imposter->{entry_col}); } From f8314a162ba44d2a1c5689494ff28db785c3708c Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Tue, 15 Jan 2019 22:23:14 -0500 Subject: [PATCH 05/10] Use different offset lims for hot pix warns --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 65 ++++++++++++++++-------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index f7101f2d..433e4d83 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -27,7 +27,7 @@ from astropy.table import Table from mica.archive import aca_dark from chandra_aca.star_probs import guide_count -from chandra_aca.transform import yagzag_to_pixels, count_rate_to_mag +from chandra_aca.transform import yagzag_to_pixels, count_rate_to_mag, mag_to_count_rate import Quaternion from Ska.quatutil import radec2yagzag import agasc @@ -38,17 +38,17 @@ from proseco.guide import get_imposter_mags def _guide_count(mags, t_ccd): return float(guide_count(np.array(mags), t_ccd)) -def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z, dmag=1.0): +def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z): """ Return a list of info to make warnings on guide stars or fid lights with local dark map that gives an - 'imposter_mag' that is brighter than `dmag` fainter than the candidate (star or fid). - As in for a 7th mag star, set an entry in the fails list if there is an imposter brighter than (7 + dmag), where - by default dmag is 1.0 (so warn on imposter mag brighter than 8.0 mag) + 'imposter_mag' that could perturb a centroid. The potential worst-case offsets (ignoring effects + at the background pixels) are returned and checking against offset limits needs to be done + from calling code. This fetches the dark current before the date of the observation and passes it to proseco.get_imposter_mags with the star candidate positions to fetch the brightest - 2x2 for each and calculates the mag for that region. If the imposter mag is brighter - than the candidate plus dmag, warn. + 2x2 for each and calculates the mag for that region. The worse case offset is then + added to an entry for the star index. :param idxs: catalog indexes as list or array :param yags: catalog yangs as list or array @@ -59,9 +59,10 @@ def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z :param date: observation date (bytestring via Inline) :param dither_y: dither_y in arcsecs (guide dither) :param dither_z: dither_z in arcsecs (guide dither) - :param dmag: delta mag for warning (default 1.0 mag) - :return fails: list of dictionaries with keys that define the index, the imposter mag, and star or fid - info to make a warning. + :param yellow_lim: yellow limit centroid offset threshold limit (in arcsecs) + :param red_lim: red limit centroid offset threshold limit (in arcsecs) + :return imposters: list of dictionaries with keys that define the index, the imposter mag, + calculated centroid offset, and star or fid info to make a warning. """ idxs = [int(idx) for idx in idxs] @@ -75,7 +76,19 @@ def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z dither_z = float(dither_z) dark = aca_dark.get_dark_cal_image(date=date, t_ccd_ref=t_ccd, aca_image=True) - fails = [] + + + def imposter_offset(cand_mag, imposter_mag): + """ + For a given candidate star and the pseudomagnitude of the brightest 2x2 imposter + calculate the max offset of the imposter counts are at the edge of the 6x6 + (as if they were in one pixel). This is somewhat the inverse of proseco.get_pixmag_for_offset + """ + cand_counts = mag_to_count_rate(cand_mag) + spoil_counts = mag_to_count_rate(imposter_mag) + return spoil_counts * 3 * 5 / (spoil_counts + cand_counts) + + imposters = [] for idx, yag, zag, mag, ctype in zip(idxs, yags, zags, mags, types): if ctype in ['BOT', 'GUI', 'FID']: if ctype in ['BOT', 'GUI']: @@ -83,16 +96,17 @@ def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z else: dither = ACABox((5.0, 5.0)) row, col = yagzag_to_pixels(yag, zag) + # get_imposter_mags takes a Table of candidates as its first argument, so construct + # a single-candidate table `entries` entries = Table([{'idx': idx, 'row': row, 'col': col, 'mag': mag, 'type': ctype}]) - entry = entries[0] - imp_mags, imp_rows, imp_cols = get_imposter_mags(entry, dark, dither) - if imp_mags[0] < (entry['mag'] + dmag): - fails.append({'idx': int(entry['idx']), - 'entry_row': float(entry['row']), 'entry_col': float(entry['col']), + imp_mags, imp_rows, imp_cols = get_imposter_mags(entries, dark, dither) + offset = imposter_offset(mag, imp_mags[0]) + imposters.append({'idx': int(idx), + 'entry_row': float(row), 'entry_col': float(col), 'bad2_row': float(imp_rows[0]), 'bad2_col': float(imp_cols[0]), - 'bad2_mag': float(imp_mags[0])}) + 'bad2_mag': float(imp_mags[0]), 'offset': float(offset)}) - return fails + return imposters def _get_agasc_stars(ra, dec, roll, radius, date, agasc_file): @@ -1317,12 +1331,19 @@ sub check_star_catalog { # Run the hot pixel region check on the Python side on FID|GUI|BOT my @imposters = check_hot_pix(\@idxs, \@yags, \@zags, \@mags, \@types, $self->{ccd_temp}, $self->{date}, $dither_guide_y, $dither_guide_z); + # Assign warnings based on those hot pixel region checks for my $imposter (@imposters){ - push @warn, sprintf( - "$alarm [%2d] Imposter 2x2. psmag %.1f (row % 4d, col % 4d) star (% 4d, % 4d) \n", - $imposter->{idx}, $imposter->{bad2_mag}, $imposter->{bad2_row}, $imposter->{bad2_col}, - $imposter->{entry_row}, $imposter->{entry_col}); + my $warn = sprintf("$alarm [%2d] Imposter mag %.1f offset %.1f (row % 4d, col % 4d) star (% 4d, % 4d)\n", + $imposter->{idx}, $imposter->{bad2_mag}, $imposter->{offset}, + $imposter->{bad2_row}, $imposter->{bad2_col}, + $imposter->{entry_row}, $imposter->{entry_col}); + if ($imposter->{offset} > 4.0){ + push @warn, $warn; + } + elsif ($imposter->{offset} > 2.5){ + push @orange_warn, $warn; + } } foreach my $i (1..16) { From 798aac41235693aa5e1ffa2e8f4cf480a7c36bd0 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Wed, 16 Jan 2019 07:47:13 -0500 Subject: [PATCH 06/10] Handle get_imposter_mags errors without crashing --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 34 ++++++++++++++++-------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index 433e4d83..f750d62b 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -62,6 +62,7 @@ def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z :param yellow_lim: yellow limit centroid offset threshold limit (in arcsecs) :param red_lim: red limit centroid offset threshold limit (in arcsecs) :return imposters: list of dictionaries with keys that define the index, the imposter mag, + a 'status' key that has value 0 if the code to get the imposter mag ran successfully, calculated centroid offset, and star or fid info to make a warning. """ @@ -95,17 +96,22 @@ def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z dither = ACABox((dither_y, dither_z)) else: dither = ACABox((5.0, 5.0)) - row, col = yagzag_to_pixels(yag, zag) - # get_imposter_mags takes a Table of candidates as its first argument, so construct - # a single-candidate table `entries` - entries = Table([{'idx': idx, 'row': row, 'col': col, 'mag': mag, 'type': ctype}]) - imp_mags, imp_rows, imp_cols = get_imposter_mags(entries, dark, dither) - offset = imposter_offset(mag, imp_mags[0]) - imposters.append({'idx': int(idx), - 'entry_row': float(row), 'entry_col': float(col), - 'bad2_row': float(imp_rows[0]), 'bad2_col': float(imp_cols[0]), - 'bad2_mag': float(imp_mags[0]), 'offset': float(offset)}) - + row, col = yagzag_to_pixels(yag, zag, allow_bad=True) + # Handle any errors in get_imposter_mags with a try/except. This doesn't + # try to pass back a message. Most likely this will only fail if the star + # or fid is completely off the CCD and will have other warning. + try: + # get_imposter_mags takes a Table of candidates as its first argument, so construct + # a single-candidate table `entries` + entries = Table([{'idx': idx, 'row': row, 'col': col, 'mag': mag, 'type': ctype}]) + imp_mags, imp_rows, imp_cols = get_imposter_mags(entries, dark, dither) + offset = imposter_offset(mag, imp_mags[0]) + imposters.append({'idx': int(idx), 'status': int(0), + 'entry_row': float(row), 'entry_col': float(col), + 'bad2_row': float(imp_rows[0]), 'bad2_col': float(imp_cols[0]), + 'bad2_mag': float(imp_mags[0]), 'offset': float(offset)}) + except: + imposters.append({'idx': int(idx), 'status': int(1)}) return imposters @@ -1333,7 +1339,13 @@ sub check_star_catalog { $self->{ccd_temp}, $self->{date}, $dither_guide_y, $dither_guide_z); # Assign warnings based on those hot pixel region checks + IMPOSTER: for my $imposter (@imposters){ + # If the check just fails on the Python side write out a warning and move on. + if ($imposter->{status} == 1){ + push @warn, "$alarm [%d] Processing error when checking for hot pixels.\n"; + next IMPOSTER; + } my $warn = sprintf("$alarm [%2d] Imposter mag %.1f offset %.1f (row % 4d, col % 4d) star (% 4d, % 4d)\n", $imposter->{idx}, $imposter->{bad2_mag}, $imposter->{offset}, $imposter->{bad2_row}, $imposter->{bad2_col}, From c313ffa511176938bf61e55e1ef07d052a4e4ef9 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 17 Jan 2019 09:52:13 -0500 Subject: [PATCH 07/10] Remove some unneeded input casts in check_hot_pix --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index f750d62b..4f03377a 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -65,16 +65,15 @@ def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z a 'status' key that has value 0 if the code to get the imposter mag ran successfully, calculated centroid offset, and star or fid info to make a warning. """ - - idxs = [int(idx) for idx in idxs] - yags = [float(yag) for yag in yags] - zags = [float(zag) for zag in zags] - mags = [float(mag) for mag in mags] + idxs = [idx for idx in idxs] + yags = [yag for yag in yags] + zags = [zag for zag in zags] + mags = [mag for mag in mags] types = [t.decode('ascii') for t in types] - t_ccd = float(t_ccd) + t_ccd = t_ccd date = date.decode('ascii') - dither_y = float(dither_y) - dither_z = float(dither_z) + dither_y = dither_y + dither_z = dither_z dark = aca_dark.get_dark_cal_image(date=date, t_ccd_ref=t_ccd, aca_image=True) From 27d9b4c70c30b91d74810468cfd520585f80cf34 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 17 Jan 2019 09:52:37 -0500 Subject: [PATCH 08/10] Prepend 'centroid' to 'offset' in imposter warning --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index 4f03377a..c88a9e15 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -1345,10 +1345,11 @@ sub check_star_catalog { push @warn, "$alarm [%d] Processing error when checking for hot pixels.\n"; next IMPOSTER; } - my $warn = sprintf("$alarm [%2d] Imposter mag %.1f offset %.1f (row % 4d, col % 4d) star (% 4d, % 4d)\n", - $imposter->{idx}, $imposter->{bad2_mag}, $imposter->{offset}, - $imposter->{bad2_row}, $imposter->{bad2_col}, - $imposter->{entry_row}, $imposter->{entry_col}); + my $warn = sprintf( + "$alarm [%2d] Imposter mag %.1f centroid offset %.1f (row % 4d, col % 4d) star (% 4d, % 4d)\n", + $imposter->{idx}, $imposter->{bad2_mag}, $imposter->{offset}, + $imposter->{bad2_row}, $imposter->{bad2_col}, + $imposter->{entry_row}, $imposter->{entry_col}); if ($imposter->{offset} > 4.0){ push @warn, $warn; } From 7378ad1944e4e5135ca4cd7bd502c8a545a92882 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 17 Jan 2019 10:43:31 -0500 Subject: [PATCH 09/10] Remove more cruft in check_hot_pix --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index c88a9e15..fdd467d0 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -65,15 +65,9 @@ def check_hot_pix(idxs, yags, zags, mags, types, t_ccd, date, dither_y, dither_z a 'status' key that has value 0 if the code to get the imposter mag ran successfully, calculated centroid offset, and star or fid info to make a warning. """ - idxs = [idx for idx in idxs] - yags = [yag for yag in yags] - zags = [zag for zag in zags] - mags = [mag for mag in mags] + types = [t.decode('ascii') for t in types] - t_ccd = t_ccd date = date.decode('ascii') - dither_y = dither_y - dither_z = dither_z dark = aca_dark.get_dark_cal_image(date=date, t_ccd_ref=t_ccd, aca_image=True) From 48c16bc4f81c14d369ab7157d855b7fe070be381 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Thu, 17 Jan 2019 10:43:56 -0500 Subject: [PATCH 10/10] Update warning format for Imposters --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index fdd467d0..4129b35d 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -1340,7 +1340,7 @@ sub check_star_catalog { next IMPOSTER; } my $warn = sprintf( - "$alarm [%2d] Imposter mag %.1f centroid offset %.1f (row % 4d, col % 4d) star (% 4d, % 4d)\n", + "$alarm [%2d] Imposter mag %.1f centroid offset %.1f row, col (%4d, %4d) star (%4d, %4d)\n", $imposter->{idx}, $imposter->{bad2_mag}, $imposter->{offset}, $imposter->{bad2_row}, $imposter->{bad2_col}, $imposter->{entry_row}, $imposter->{entry_col});