From 08fbfabcbe410444f3850e4ee8a68473e13f1b15 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Fri, 23 Aug 2019 12:06:56 -0400 Subject: [PATCH 1/5] Add first draft of using mica stats instead of sybase --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 129 ++++++----------------- starcheck/src/starcheck.pl | 14 --- 2 files changed, 32 insertions(+), 111 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index ebe94690..0dd6836e 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -162,6 +162,37 @@ def _get_agasc_stars(ra, dec, roll, radius, date, agasc_file): return stars_dict + +import numpy as np +import mica.stats.acq_stats +import mica.stats.guide_stats + +ACQS = mica.stats.acq_stats.get_stats() +GUIDES = mica.stats.guide_stats.get_stats() + + +def get_mica_star_stats(agasc_id, time): + time = float(time) + agasc_id = int(agasc_id) + acqs = Table(ACQS[(ACQS['agasc_id'] == agasc_id) + & (ACQS['guide_tstart'] < time)]) + ok = acqs['img_func'] == 'star' + guides = Table(GUIDES[(GUIDES['agasc_id'] == agasc_id) + & (GUIDES['kalman_tstart'] < time)]) + mags = np.concatenate( + [acqs['mag_obs'][acqs['mag_obs'] != 0], + guides['aoacmag_mean'][guides['aoacmag_mean'] != 0]]) + + avg_mag = float(np.mean(mags)) if (len(mags) > 0) else float(13.94) + stats = {'acq': len(acqs), + 'acq_noid': int(np.count_nonzero(~ok)), + 'gui' : len(guides), + 'gui_bad': int(np.count_nonzero(guides['f_track'] < .95)), + 'gui_fail': int(np.count_nonzero(guides['f_track'] < .01)), + 'gui_obc_bad': int(np.count_nonzero(guides['f_obc_bad'] > .05)), + 'avg_mag': avg_mag} + return stats + }; @@ -2572,103 +2603,7 @@ sub star_dbhist { my $obs_tstart_minus_day = $obs_tstart - 86400; - return undef if (not defined $db_handle); - - my %stats = ( - 'agasc_id' => $star_id, - 'acq' => 0, - 'acq_noid' => 0, - 'gui' => 0, - 'gui_bad' => 0, - 'gui_fail' => 0, - 'gui_obc_bad' => 0, - 'avg_mag' => 13.9375, - ); - - - - eval{ - # acq_stats_data - my $sql = SQL::Abstract->new(); - my %acq_where = ( 'agasc_id' => $star_id, - 'type' => { '!=' => 'FID'}, - 'tstart' => { '<' => $obs_tstart_minus_day } - ); - - my ($acq_all_stmt, @acq_all_bind ) = $sql->select('acq_stats_data', - '*', - \%acq_where ); - - my @acq_all = sql_fetchall_array_of_hashref( $db_handle, $acq_all_stmt, @acq_all_bind ); - my @mags; - - if (scalar(@acq_all)){ - my $noid = 0; - for my $attempt (@acq_all){ - if ($attempt->{'obc_id'} =~ 'NOID'){ - $noid++; - } - else{ - push @mags, $attempt->{'mag_obs'}; - } - } - $stats{'acq'} = scalar(@acq_all); - $stats{'acq_noid'} = $noid; - } - - # guide_stats_view - $sql = SQL::Abstract->new(); - my %gui_where = ( 'id' => $star_id, - 'type' => { '!=' => 'FID' }, - 'kalman_tstart' => { '<' => $obs_tstart_minus_day }); - - my ($gui_all_stmt, @gui_all_bind ) = $sql->select('guide_stats_view', - '*', - \%gui_where ); - - my @gui_all = sql_fetchall_array_of_hashref( $db_handle, $gui_all_stmt, @gui_all_bind ); - - - if (scalar(@gui_all)){ - my $bad = 0; - my $fail = 0; - my $obc_bad = 0; - for my $attempt (@gui_all){ - if ($attempt->{'percent_not_tracking'} >= 5){ - $bad++; - } - if ($attempt->{'percent_not_tracking'} == 100){ - $fail++; - } - else{ - if ((defined $attempt->{'mag_obs_mean'}) and ($attempt->{'mag_obs_mean'} < 13.9 )){ - push @mags, $attempt->{'mag_obs_mean'}; - } - } - if ($attempt->{'percent_obc_bad_status'} >= 5){ - $obc_bad++; - } - } - $stats{'gui'} = scalar(@gui_all); - $stats{'gui_bad'} = $bad; - $stats{'gui_fail'} = $fail; - $stats{'gui_obc_bad'} = $obc_bad; - } - - my $mag_sum = 0; - if (scalar(@mags)){ - map { $mag_sum += $_ } @mags; - $stats{'avg_mag'} = $mag_sum / scalar(@mags); - } - }; - if ($@){ - # if we get db errors, just print and move on - print STDERR $@; - - } - - return \%stats; - + return get_mica_star_stats($star_id, $obs_tstart_minus_day); } diff --git a/starcheck/src/starcheck.pl b/starcheck/src/starcheck.pl index 3e128a31..b17df958 100755 --- a/starcheck/src/starcheck.pl +++ b/starcheck/src/starcheck.pl @@ -347,20 +347,6 @@ } -# See if we have database access -my $db_handle; -eval{ - eval 'use Ska::DatabaseUtil'; - $db_handle = Ska::DatabaseUtil::sql_connect( 'sybase-aca-aca_read' ); - -}; -if (($@) or (not defined $db_handle)){ - warning("Unable to connect to Sybase server; links generated for all AGASC ids by default \n"); -} -else{ - Ska::Starcheck::Obsid::set_db_handle($db_handle); - } - # Dark Cal Checker Section use Ska::Starcheck::Dark_Cal_Checker; my $dark_cal_checker; From ff863b1c6d8817737aea68f654cfe255a8d69f69 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Fri, 23 Aug 2019 12:07:33 -0400 Subject: [PATCH 2/5] Use kadi star lookup instead of stats cgi for links --- starcheck/data/characteristics.yaml | 1 - starcheck/src/lib/Ska/Starcheck/Obsid.pm | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/starcheck/data/characteristics.yaml b/starcheck/data/characteristics.yaml index 904fab4d..de767367 100644 --- a/starcheck/data/characteristics.yaml +++ b/starcheck/data/characteristics.yaml @@ -1,5 +1,4 @@ paths: - acq_stat_query: https://icxc.harvard.edu/cgi-bin/aspect/get_stats/get_stats.cgi week_lookup: https://icxc.harvard.edu/cgi-bin/aspect/starcheck/find_shortterm.cgi # string match on oflsids that may not have a maneuver or starcat, and its OK! diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index 0dd6836e..32f30830 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -2140,7 +2140,7 @@ sub print_report { } } - my $acq_stat_lookup = "$config{paths}->{acq_stat_query}?id="; + my $star_stat_lookup = "http://kadi.cfa.harvard.edu/star_hist/?agasc_id="; my $table; @@ -2204,8 +2204,8 @@ sub print_report { # Make the id a URL if there is star history or if star history could # not be checked (no db_handle) my $star_link; - if ((not defined $db_handle) or (($db_stats->{acq} or $db_stats->{gui}))){ - $star_link = sprintf("HREF=\"%s%s\"",$acq_stat_lookup, $c->{"GS_ID${i}"}); + if ($db_stats->{acq} or $db_stats->{gui}){ + $star_link = sprintf("HREF=\"%s%s\"",$star_stat_lookup, $c->{"GS_ID${i}"}); } else{ $star_link = sprintf("A=\"star\""); From c32b4334ad8b854c5d406df8a6ecccd250829d10 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Fri, 23 Aug 2019 12:19:37 -0400 Subject: [PATCH 3/5] Set path to standard test data for 'make test' --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 142e551d..c09f9b2d 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ FLIGHT_ENV = SKA SRC = starcheck/src -#include $(SKA)/include/Makefile.FLIGHT RELATED_LIB = $(SRC)/StarcheckParser.pm BIN = $(SRC)/starcheck.pl $(SRC)/starcheck @@ -15,7 +14,7 @@ DOC_RST = $(SRC)/aca_load_review_cl.rst DOC_HTML = aca_load_review_cl.html -TEST_DATA_TGZ = $(ROOT_FLIGHT)/data/starcheck/JUL0918A_test_data.tar.gz +TEST_DATA_TGZ = ${SKA}/data/starcheck/JUL0918A_test_data.tar.gz # starcheck_characteristics tarball should be installed from # separate starcheck_characteristics project # with "make install_dist" from that project From 8c5b121797c5d13bb2571fa410568313a4690d45 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Fri, 23 Aug 2019 12:31:17 -0400 Subject: [PATCH 4/5] Move imports and add docstring for get_mica_star_stats --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 26 ++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index 32f30830..a3c9324b 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -36,6 +36,13 @@ from proseco.core import ACABox from proseco.catalog import get_effective_t_ccd from proseco.guide import get_imposter_mags +import mica.stats.acq_stats +import mica.stats.guide_stats + +ACQS = mica.stats.acq_stats.get_stats() +GUIDES = mica.stats.guide_stats.get_stats() + + def _yagzag_to_pixels(yag, zag): """ Call chandra_aca.transform.yagzag_to_pixels. @@ -163,17 +170,24 @@ def _get_agasc_stars(ra, dec, roll, radius, date, agasc_file): return stars_dict -import numpy as np -import mica.stats.acq_stats -import mica.stats.guide_stats +def get_mica_star_stats(agasc_id, time): + """ + Get the acq and guide star statistics for a star before a given time. + The time filter is just there to make this play well when run in regression. + The mica acq and guide stats are fetched into globals ACQS and GUIDES + and this method just filters for the relevant ones for a star and returns + a dictionary of summarized statistics. -ACQS = mica.stats.acq_stats.get_stats() -GUIDES = mica.stats.guide_stats.get_stats() + :param agasc_id: agasc id of star + :param time: time used as end of range to retrieve statistics. + :return: dictionary of stats for the observed history of the star + """ -def get_mica_star_stats(agasc_id, time): + # Cast the inputs time = float(time) agasc_id = int(agasc_id) + acqs = Table(ACQS[(ACQS['agasc_id'] == agasc_id) & (ACQS['guide_tstart'] < time)]) ok = acqs['img_func'] == 'star' From 98e8b98ed67035e92db0d87cad0e598fef23c009 Mon Sep 17 00:00:00 2001 From: Jean Connelly Date: Fri, 23 Aug 2019 12:32:09 -0400 Subject: [PATCH 5/5] Remove database Perl imports --- starcheck/src/lib/Ska/Starcheck/Obsid.pm | 2 -- 1 file changed, 2 deletions(-) diff --git a/starcheck/src/lib/Ska/Starcheck/Obsid.pm b/starcheck/src/lib/Ska/Starcheck/Obsid.pm index a3c9324b..b003d1a1 100644 --- a/starcheck/src/lib/Ska/Starcheck/Obsid.pm +++ b/starcheck/src/lib/Ska/Starcheck/Obsid.pm @@ -221,8 +221,6 @@ use Ska::Convert qw(date2time time2date); use RDB; -use SQL::Abstract; -use Ska::DatabaseUtil qw( sql_fetchall_array_of_hashref ); use Carp; # Constants