diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml deleted file mode 100644 index 1ce62dc1..00000000 --- a/.github/workflows/flake8.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: Python flake8 check - -on: [push] - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Lint with flake8 - run: | - pip install flake8 - # Note: only check files in Ska.engarchive package. Many other files in - # the repo are not maintained as PEP8 compliant. - flake8 agasc setup.py --count --ignore=E402,W503,F541 --max-line-length=100 --show-source --statistics diff --git a/.github/workflows/python-linting.yml b/.github/workflows/python-linting.yml new file mode 100644 index 00000000..2c5fccad --- /dev/null +++ b/.github/workflows/python-linting.yml @@ -0,0 +1,11 @@ +name: Check Python formatting using Black and Ruff + +on: [push] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: psf/black@stable + - uses: chartboost/ruff-action@v1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..0ff7e639 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,12 @@ +repos: +- repo: https://github.com/psf/black + rev: 23.9.1 + hooks: + - id: black + language_version: "python3.10" + +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.292 + hooks: + - id: ruff + language_version: "python3.10" diff --git a/agasc/__init__.py b/agasc/__init__.py index a0fa4a4f..5160c538 100644 --- a/agasc/__init__.py +++ b/agasc/__init__.py @@ -9,8 +9,9 @@ def test(*args, **kwargs): - ''' + """ Run py.test unit tests. - ''' + """ import testr + return testr.test(*args, **kwargs) diff --git a/agasc/agasc.py b/agasc/agasc.py index 73a10ee2..131e7870 100644 --- a/agasc/agasc.py +++ b/agasc/agasc.py @@ -3,7 +3,6 @@ import functools import os import re -from packaging.version import Version from pathlib import Path from typing import Optional @@ -12,18 +11,27 @@ import tables from astropy.table import Column, Table from Chandra.Time import DateTime +from packaging.version import Version from .healpix import get_stars_from_healpix_h5, is_healpix from .paths import default_agasc_dir from .supplement.utils import get_supplement_table -__all__ = ['sphere_dist', 'get_agasc_cone', 'get_star', 'get_stars', 'read_h5_table', - 'get_agasc_filename', - 'MAG_CATID_SUPPLEMENT', 'BAD_CLASS_SUPPLEMENT', - 'set_supplement_enabled', 'SUPPLEMENT_ENABLED_ENV'] - -SUPPLEMENT_ENABLED_ENV = 'AGASC_SUPPLEMENT_ENABLED' -SUPPLEMENT_ENABLED_DEFAULT = 'True' +__all__ = [ + "sphere_dist", + "get_agasc_cone", + "get_star", + "get_stars", + "read_h5_table", + "get_agasc_filename", + "MAG_CATID_SUPPLEMENT", + "BAD_CLASS_SUPPLEMENT", + "set_supplement_enabled", + "SUPPLEMENT_ENABLED_ENV", +] + +SUPPLEMENT_ENABLED_ENV = "AGASC_SUPPLEMENT_ENABLED" +SUPPLEMENT_ENABLED_DEFAULT = "True" MAG_CATID_SUPPLEMENT = 100 BAD_CLASS_SUPPLEMENT = 100 @@ -99,7 +107,7 @@ def test_get_aca_catalog(): Whether to use the AGASC supplement in the context / decorator """ if not isinstance(value, bool): - raise TypeError('value must be bool (True|False)') + raise TypeError("value must be bool (True|False)") orig = os.environ.get(SUPPLEMENT_ENABLED_ENV) os.environ[SUPPLEMENT_ENABLED_ENV] = str(value) @@ -129,21 +137,21 @@ def agasc_file(self): @property def ra(self): - if not hasattr(self, '_ra'): + if not hasattr(self, "_ra"): self._ra, self._dec = self.read_ra_dec() return self._ra @property def dec(self): - if not hasattr(self, '_dec'): + if not hasattr(self, "_dec"): self._ra, self._dec = self.read_ra_dec() return self._dec def read_ra_dec(self): # Read the RA and DEC values from the agasc with tables.open_file(self.agasc_file) as h5: - ras = h5.root.data.read(field='RA') - decs = h5.root.data.read(field='DEC') + ras = h5.root.data.read(field="RA") + decs = h5.root.data.read(field="DEC") return ras, decs @@ -155,11 +163,11 @@ def get_ra_decs(agasc_file): def read_h5_table( - h5_file: str | Path | tables.file.File, - row0: Optional[int] = None, - row1: Optional[int] = None, - path="data", - cache=False, + h5_file: str | Path | tables.file.File, + row0: Optional[int] = None, + row1: Optional[int] = None, + path="data", + cache=False, ) -> np.ndarray: """ Read HDF5 table from group ``path`` in ``h5_file``. @@ -209,10 +217,10 @@ def _read_h5_table_cached( def _read_h5_table( - h5_file: str | Path | tables.file.File, - path: str, - row0: None | int, - row1: None | int, + h5_file: str | Path | tables.file.File, + path: str, + row0: None | int, + row1: None | int, ) -> np.ndarray: if isinstance(h5_file, tables.file.File): out = _read_h5_table_from_open_h5_file(h5_file, path, row0, row1) @@ -320,9 +328,9 @@ def get_agasc_filename( continue version_str = match.group(1) rc_str = match.group(2) or "" - if ( - version is not None - and version not in (version_str, version_str + rc_str) + if version is not None and version not in ( + version_str, + version_str + rc_str, ): continue matches.append((Version(version_str.replace("p", ".") + rc_str), path)) @@ -359,10 +367,12 @@ def sphere_dist(ra1, dec1, ra2, dec2): dec1 = np.radians(dec1).astype(np.float64) dec2 = np.radians(dec2).astype(np.float64) - numerator = numexpr.evaluate('sin((dec2 - dec1) / 2) ** 2 + ' # noqa - 'cos(dec1) * cos(dec2) * sin((ra2 - ra1) / 2) ** 2') + numerator = numexpr.evaluate( # noqa: F841 + "sin((dec2 - dec1) / 2) ** 2 + " + "cos(dec1) * cos(dec2) * sin((ra2 - ra1) / 2) ** 2" + ) - dists = numexpr.evaluate('2 * arctan2(numerator ** 0.5, (1 - numerator) ** 0.5)') + dists = numexpr.evaluate("2 * arctan2(numerator ** 0.5, (1 - numerator) ** 0.5)") return np.degrees(dists) @@ -382,8 +392,8 @@ def update_color1_column(stars): This updates ``stars`` in place. """ # Select red stars that have a reliable mag in AGASC 1.7 and later. - color15 = np.isclose(stars['COLOR1'], 1.5) & (stars['RSV3'] > 0) - new_color1 = stars['COLOR2'][color15] * 0.850 + color15 = np.isclose(stars["COLOR1"], 1.5) & (stars["RSV3"] > 0) + new_color1 = stars["COLOR2"][color15] * 0.850 if len(new_color1) > 0: # Ensure no new COLOR1 are within 0.001 of 1.5, so downstream tests of @@ -392,7 +402,7 @@ def update_color1_column(stars): new_color1[fix15] = 1.499 # Insignificantly different from 1.50 # For stars with a reliable mag, now COLOR1 is really the B-V color. - stars['COLOR1'][color15] = new_color1 + stars["COLOR1"][color15] = new_color1 def add_pmcorr_columns(stars, date): @@ -416,25 +426,41 @@ def add_pmcorr_columns(stars, date): # Compute delta year. stars['EPOCH'] is Column, float32. Need to coerce to # ndarray float64 for consistent results between scalar and array cases. - dyear = dates.frac_year - stars['EPOCH'].view(np.ndarray).astype(np.float64) + dyear = dates.frac_year - stars["EPOCH"].view(np.ndarray).astype(np.float64) - pm_to_degrees = dyear / (3600. * 1000.) - dec_pmcorr = np.where(stars['PM_DEC'] != -9999, - stars['DEC'] + stars['PM_DEC'] * pm_to_degrees, - stars['DEC']) - ra_scale = np.cos(np.radians(stars['DEC'])) - ra_pmcorr = np.where(stars['PM_RA'] != -9999, - stars['RA'] + stars['PM_RA'] * pm_to_degrees / ra_scale, - stars['RA']) + pm_to_degrees = dyear / (3600.0 * 1000.0) + dec_pmcorr = np.where( + stars["PM_DEC"] != -9999, + stars["DEC"] + stars["PM_DEC"] * pm_to_degrees, + stars["DEC"], + ) + ra_scale = np.cos(np.radians(stars["DEC"])) + ra_pmcorr = np.where( + stars["PM_RA"] != -9999, + stars["RA"] + stars["PM_RA"] * pm_to_degrees / ra_scale, + stars["RA"], + ) # Add the proper-motion corrected columns to table using astropy.table.Table - stars.add_columns([Column(data=ra_pmcorr, name='RA_PMCORR'), - Column(data=dec_pmcorr, name='DEC_PMCORR')]) + stars.add_columns( + [ + Column(data=ra_pmcorr, name="RA_PMCORR"), + Column(data=dec_pmcorr, name="DEC_PMCORR"), + ] + ) -def get_agasc_cone(ra, dec, radius=1.5, date=None, agasc_file=None, - pm_filter=True, fix_color1=True, use_supplement=None, - cache=False): +def get_agasc_cone( + ra, + dec, + radius=1.5, + date=None, + agasc_file=None, + pm_filter=True, + fix_color1=True, + use_supplement=None, + cache=False, +): """ Get AGASC catalog entries within ``radius`` degrees of ``ra``, ``dec``. @@ -481,7 +507,7 @@ def get_agasc_cone(ra, dec, radius=1.5, date=None, agasc_file=None, # Final filtering using proper-motion corrected positions if pm_filter: - dists = sphere_dist(ra, dec, stars['RA_PMCORR'], stars['DEC_PMCORR']) + dists = sphere_dist(ra, dec, stars["RA_PMCORR"], stars["DEC_PMCORR"]) ok = dists <= radius stars = stars[ok] @@ -492,11 +518,11 @@ def get_agasc_cone(ra, dec, radius=1.5, date=None, agasc_file=None, def get_stars_from_dec_sorted_h5( - ra: float, - dec: float, - radius: float, - agasc_file: str | Path, - cache: bool = False, + ra: float, + dec: float, + radius: float, + agasc_file: str | Path, + cache: bool = False, ) -> Table: """ Returns a table of stars within a given radius of a given RA and Dec. @@ -573,11 +599,12 @@ def get_star(id, agasc_file=None, date=None, fix_color1=True, use_supplement=Non with tables.open_file(agasc_file) as h5: tbl = h5.root.data - id_rows = tbl.read_where('(AGASC_ID == {})'.format(id)) + id_rows = tbl.read_where("(AGASC_ID == {})".format(id)) if len(id_rows) > 1: raise InconsistentCatalogError( - "More than one entry found for {} in AGASC".format(id)) + "More than one entry found for {} in AGASC".format(id) + ) if id_rows is None or len(id_rows) == 0: raise IdNotFound() @@ -596,15 +623,16 @@ def _get_rows_read_where(ids_1d, dates_1d, agasc_file): rows = [] with tables.open_file(agasc_file) as h5: tbl = h5.root.data - for id, date in zip(ids_1d, dates_1d): - id_rows = tbl.read_where('(AGASC_ID == {})'.format(id)) + for id, _date in zip(ids_1d, dates_1d): + id_rows = tbl.read_where("(AGASC_ID == {})".format(id)) if len(id_rows) > 1: raise InconsistentCatalogError( - f'More than one entry found for {id} in AGASC') + f"More than one entry found for {id} in AGASC" + ) if id_rows is None or len(id_rows) == 0: - raise IdNotFound(f'No entry found for {id} in AGASC') + raise IdNotFound(f"No entry found for {id} in AGASC") rows.append(id_rows[0]) return rows @@ -614,19 +642,25 @@ def _get_rows_read_entire(ids_1d, dates_1d, agasc_file): with tables.open_file(agasc_file) as h5: tbl = h5.root.data[:] - agasc_idx = {agasc_id: idx for idx, agasc_id in enumerate(tbl['AGASC_ID'])} + agasc_idx = {agasc_id: idx for idx, agasc_id in enumerate(tbl["AGASC_ID"])} rows = [] - for agasc_id, date in zip(ids_1d, dates_1d): + for agasc_id, _date in zip(ids_1d, dates_1d): if agasc_id not in agasc_idx: - raise IdNotFound(f'No entry found for {agasc_id} in AGASC') + raise IdNotFound(f"No entry found for {agasc_id} in AGASC") rows.append(tbl[agasc_idx[agasc_id]]) return rows -def get_stars(ids, agasc_file=None, dates=None, method_threshold=5000, fix_color1=True, - use_supplement=None): +def get_stars( + ids, + agasc_file=None, + dates=None, + method_threshold=5000, + fix_color1=True, + use_supplement=None, +): """ Get AGASC catalog entries for star ``ids`` at ``dates``. @@ -687,10 +721,10 @@ def get_stars(ids, agasc_file=None, dates=None, method_threshold=5000, fix_color if len(ids_1d) < method_threshold: rows = _get_rows_read_where(ids_1d, dates_1d, agasc_file) - method = 'tables_read_where' + method = "tables_read_where" else: rows = _get_rows_read_entire(ids_1d, dates_1d, agasc_file) - method = 'read_entire_agasc' + method = "read_entire_agasc" t = Table(np.vstack(rows).flatten()) @@ -700,7 +734,7 @@ def get_stars(ids, agasc_file=None, dates=None, method_threshold=5000, fix_color add_pmcorr_columns(t, dates_in if dates_is_scalar else dates) if fix_color1: update_color1_column(t) - t['DATE'] = dates + t["DATE"] = dates update_from_supplement(t, use_supplement) @@ -743,11 +777,15 @@ def update_from_supplement(stars, use_supplement=None): Use the supplement (default=None, see above) """ if use_supplement is None: - supplement_enabled_env = os.environ.get(SUPPLEMENT_ENABLED_ENV, SUPPLEMENT_ENABLED_DEFAULT) - if supplement_enabled_env not in ('True', 'False'): - raise ValueError(f'{SUPPLEMENT_ENABLED_ENV} env var must be either "True" or "False" ' - f'got {supplement_enabled_env}') - supplement_enabled = supplement_enabled_env == 'True' + supplement_enabled_env = os.environ.get( + SUPPLEMENT_ENABLED_ENV, SUPPLEMENT_ENABLED_DEFAULT + ) + if supplement_enabled_env not in ("True", "False"): + raise ValueError( + f'{SUPPLEMENT_ENABLED_ENV} env var must be either "True" or "False" ' + f"got {supplement_enabled_env}" + ) + supplement_enabled = supplement_enabled_env == "True" else: supplement_enabled = use_supplement @@ -763,28 +801,28 @@ def set_star(star, name, value): # Get estimate mags and errs from supplement as a dict of dict # agasc_id : {mag_aca: .., mag_aca_err: ..}. - supplement_mags = get_supplement_table('mags', agasc_dir=default_agasc_dir()) + supplement_mags = get_supplement_table("mags", agasc_dir=default_agasc_dir()) supplement_mags_index = supplement_mags.meta["index"] # Get bad stars as {agasc_id: {source: ..}} - bad_stars = get_supplement_table('bad', agasc_dir=default_agasc_dir()) + bad_stars = get_supplement_table("bad", agasc_dir=default_agasc_dir()) bad_stars_index = bad_stars.meta["index"] for star in stars: - agasc_id = int(star['AGASC_ID']) + agasc_id = int(star["AGASC_ID"]) if agasc_id in supplement_mags_index: idx = supplement_mags_index[agasc_id] - mag_est = supplement_mags['mag_aca'][idx] - mag_est_err = supplement_mags['mag_aca_err'][idx] + mag_est = supplement_mags["mag_aca"][idx] + mag_est_err = supplement_mags["mag_aca_err"][idx] - set_star(star, 'MAG_ACA', mag_est) + set_star(star, "MAG_ACA", mag_est) # Mag err is stored as int16 in units of 0.01 mag. Use same convention here. - set_star(star, 'MAG_ACA_ERR', round(mag_est_err * 100)) - set_star(star, 'MAG_CATID', MAG_CATID_SUPPLEMENT) - if 'COLOR1' in stars.colnames: - color1 = star['COLOR1'] + set_star(star, "MAG_ACA_ERR", round(mag_est_err * 100)) + set_star(star, "MAG_CATID", MAG_CATID_SUPPLEMENT) + if "COLOR1" in stars.colnames: + color1 = star["COLOR1"] if np.isclose(color1, 0.7) or np.isclose(color1, 1.5): - star['COLOR1'] = color1 - 0.01 + star["COLOR1"] = color1 - 0.01 if agasc_id in bad_stars_index: - set_star(star, 'CLASS', BAD_CLASS_SUPPLEMENT) + set_star(star, "CLASS", BAD_CLASS_SUPPLEMENT) diff --git a/agasc/healpix.py b/agasc/healpix.py index cae876a4..68a5722f 100644 --- a/agasc/healpix.py +++ b/agasc/healpix.py @@ -19,7 +19,6 @@ import tables from astropy.table import Table - __all__ = ["is_healpix", "get_stars_from_healpix_h5"] @@ -106,7 +105,7 @@ def get_stars_from_healpix_h5( stars : astropy.table.Table Table of stars within the search cone, with columns from the AGASC data table. """ - from agasc import sphere_dist, read_h5_table + from agasc import read_h5_table, sphere_dist # Table of healpix, idx0, idx1 where idx is the index into main AGASC data table healpix_index_map, nside = get_healpix_info(agasc_file) diff --git a/agasc/paths.py b/agasc/paths.py index c44d4027..c2a4744e 100644 --- a/agasc/paths.py +++ b/agasc/paths.py @@ -2,10 +2,10 @@ import os from pathlib import Path -SUPPLEMENT_FILENAME = 'agasc_supplement.h5' +SUPPLEMENT_FILENAME = "agasc_supplement.h5" -__all__ = ['default_agasc_dir', 'default_agasc_file', 'SUPPLEMENT_FILENAME'] +__all__ = ["default_agasc_dir", "default_agasc_file", "SUPPLEMENT_FILENAME"] def default_agasc_dir(): @@ -16,10 +16,10 @@ def default_agasc_dir(): :returns: Path """ - if 'AGASC_DIR' in os.environ: - out = Path(os.environ['AGASC_DIR']) + if "AGASC_DIR" in os.environ: + out = Path(os.environ["AGASC_DIR"]) else: - out = Path(os.environ['SKA'], 'data', 'agasc') + out = Path(os.environ["SKA"], "data", "agasc") return out @@ -29,4 +29,5 @@ def default_agasc_file(): :returns: str """ from agasc import get_agasc_filename + return get_agasc_filename() diff --git a/agasc/scripts/mag_estimate_report.py b/agasc/scripts/mag_estimate_report.py index 167bd14b..070a862a 100755 --- a/agasc/scripts/mag_estimate_report.py +++ b/agasc/scripts/mag_estimate_report.py @@ -4,11 +4,13 @@ Produce reports of the magnitude supplement. """ -import os import argparse +import os from pathlib import Path + import numpy as np -from astropy import table, time, units as u +from astropy import table, time +from astropy import units as u from cxotime import CxoTime, units from agasc.supplement.magnitudes import mag_estimate_report @@ -16,35 +18,67 @@ def get_parser(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--start', help='Time to start processing new observations.' - ' CxoTime-compatible time stamp.' - ' Default: now - 90 days') - parser.add_argument('--stop', help='Time to stop processing new observations.' - ' CxoTime-compatible time stamp.' - ' Default: now') - parser.add_argument('--input-dir', default='$SKA/data/agasc', - help='Directory containing mag-stats files. Default: $SKA/data/agasc') - parser.add_argument('--output-dir', default='supplement_reports/suspect', - help='Output directory.' - ' Default: supplement_reports/suspect') - parser.add_argument('--obs-stats', default='mag_stats_obsid.fits', - help='FITS file with mag-stats for all observations.' - ' Default: mag_stats_obsid.fits') - parser.add_argument('--agasc-stats', default='mag_stats_agasc.fits', - help='FITS file with mag-stats for all observed AGASC stars.' - ' Default: mag_stats_agasc.fits') - parser.add_argument('--weekly-report', - help="Add links to navigate weekly reports.", - action='store_true', default=False) - parser.add_argument('--all-stars', - help="Include all stars in the report, not just suspect.", - action='store_true', default=False) + parser.add_argument( + "--start", + help=( + "Time to start processing new observations." + " CxoTime-compatible time stamp." + " Default: now - 90 days" + ), + ) + parser.add_argument( + "--stop", + help=( + "Time to stop processing new observations." + " CxoTime-compatible time stamp." + " Default: now" + ), + ) + parser.add_argument( + "--input-dir", + default="$SKA/data/agasc", + help="Directory containing mag-stats files. Default: $SKA/data/agasc", + ) + parser.add_argument( + "--output-dir", + default="supplement_reports/suspect", + help="Output directory. Default: supplement_reports/suspect", + ) + parser.add_argument( + "--obs-stats", + default="mag_stats_obsid.fits", + help=( + "FITS file with mag-stats for all observations." + " Default: mag_stats_obsid.fits" + ), + ) + parser.add_argument( + "--agasc-stats", + default="mag_stats_agasc.fits", + help=( + "FITS file with mag-stats for all observed AGASC stars." + " Default: mag_stats_agasc.fits" + ), + ) + parser.add_argument( + "--weekly-report", + help="Add links to navigate weekly reports.", + action="store_true", + default=False, + ) + parser.add_argument( + "--all-stars", + help="Include all stars in the report, not just suspect.", + action="store_true", + default=False, + ) return parser def main(): import kadi.commands - kadi.commands.conf.commands_version = '1' + + kadi.commands.conf.commands_version = "1" args = get_parser().parse_args() @@ -64,39 +98,39 @@ def main(): else: args.start = CxoTime(args.start) - t = (obs_stats['mp_starcat_time']) + t = obs_stats["mp_starcat_time"] ok = (t < args.stop) & (t > args.start) if not args.all_stars: - ok &= ~obs_stats['obs_ok'] - stars = np.unique(obs_stats[ok]['agasc_id']) - sections = [{ - 'id': 'stars', - 'title': 'Stars', - 'stars': stars - }] + ok &= ~obs_stats["obs_ok"] + stars = np.unique(obs_stats[ok]["agasc_id"]) + sections = [{"id": "stars", "title": "Stars", "stars": stars}] - agasc_stats = agasc_stats[np.in1d(agasc_stats['agasc_id'], stars)] + agasc_stats = agasc_stats[np.in1d(agasc_stats["agasc_id"], stars)] if args.weekly_report: t = CxoTime(args.stop) directory = args.output_dir / t.date[:8] week = time.TimeDelta(7 * u.day) nav_links = { - 'previous': f'../{(t - week).date[:8]}', - 'up': '..', - 'next': f'../{(t + week).date[:8]}' + "previous": f"../{(t - week).date[:8]}", + "up": "..", + "next": f"../{(t + week).date[:8]}", } else: directory = args.output_dir nav_links = None - msr = mag_estimate_report.MagEstimateReport(agasc_stats, - obs_stats, - directory=directory) - msr.multi_star_html(sections=sections, tstart=args.start, tstop=args.stop, - filename='index.html', - nav_links=nav_links) + msr = mag_estimate_report.MagEstimateReport( + agasc_stats, obs_stats, directory=directory + ) + msr.multi_star_html( + sections=sections, + tstart=args.start, + tstop=args.stop, + filename="index.html", + nav_links=nav_links, + ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/agasc/scripts/supplement_diff.py b/agasc/scripts/supplement_diff.py index 5180023b..ec21969f 100755 --- a/agasc/scripts/supplement_diff.py +++ b/agasc/scripts/supplement_diff.py @@ -4,32 +4,28 @@ Generate diff between to supplement files and output in HTML format. """ -import os -import tables -from pathlib import Path -import difflib -import pygments.lexers -import pygments.formatters import argparse import datetime - +import difflib +import os from io import StringIO +from pathlib import Path -from astropy.io import ascii +import pygments.formatters +import pygments.lexers +import tables from astropy import table +from astropy.io import ascii def read_file(filename, exclude=[]): formats = { - 'agasc_versions': {k: '{:>21s}' for k in ['mags', 'obs', 'bad', 'supplement']}, - 'last_updated': {k: '{:>21s}' for k in ['mags', 'obs', 'bad', 'supplement']}, - 'obs': { - 'comments': '{:>80s}', - 'agasc_id': '{:10d}' - }, + "agasc_versions": {k: "{:>21s}" for k in ["mags", "obs", "bad", "supplement"]}, + "last_updated": {k: "{:>21s}" for k in ["mags", "obs", "bad", "supplement"]}, + "obs": {"comments": "{:>80s}", "agasc_id": "{:10d}"}, } - node_names = ['agasc_versions', 'last_updated', 'obs', 'bad'] + node_names = ["agasc_versions", "last_updated", "obs", "bad"] with tables.open_file(filename) as h5: for node in h5.root: if node.name not in node_names: @@ -38,48 +34,51 @@ def read_file(filename, exclude=[]): for name in node_names: if name in exclude: continue - node = h5.get_node(f'/{name}') + node = h5.get_node(f"/{name}") t = table.Table(node[:]) t.convert_bytestring_to_unicode s = StringIO() - ascii.write(t, s, format='fixed_width', formats=formats.get(name, {})) + ascii.write(t, s, format="fixed_width", formats=formats.get(name, {})) s.seek(0) lines = s.readlines() dashes = "-" * (len(lines[0]) - 1) + "\n" - all_lines += ([dashes, f'| {node.name}\n', dashes] + lines + [dashes]) + all_lines += [dashes, f"| {node.name}\n", dashes] + lines + [dashes] return all_lines def diff_files(fromfile, tofile, exclude_mags=False): - exclude = ['mags'] if exclude_mags else [] + exclude = ["mags"] if exclude_mags else [] fromlines = read_file(fromfile, exclude) tolines = read_file(tofile, exclude) diff = difflib.unified_diff(fromlines, tolines, str(fromfile), str(tofile)) - diff = ''.join(diff) + diff = "".join(diff) return diff def diff_to_html(diff): - date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M') + date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") lexer = pygments.lexers.DiffLexer() formatter = pygments.formatters.HtmlFormatter( - full=True, linenos='table', - title=f'AGASC supplement diff - {date}') + full=True, linenos="table", title=f"AGASC supplement diff - {date}" + ) return pygments.highlight(diff, lexer, formatter) def get_parser(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--from', dest='fromfile', type=Path, help='The original supplement file.') - parser.add_argument('--to', dest='tofile', type=Path, help='The modified supplement file.') parser.add_argument( - '-o', - help='Output HTML file', - type=Path, - default='agasc_supplement_diff.html' + "--from", dest="fromfile", type=Path, help="The original supplement file." + ) + parser.add_argument( + "--to", dest="tofile", type=Path, help="The modified supplement file." + ) + parser.add_argument( + "-o", help="Output HTML file", type=Path, default="agasc_supplement_diff.html" ) parser.add_argument( - '--exclude-mags', default=False, action='store_true', + "--exclude-mags", + default=False, + action="store_true", help='Exclude changes in the "mags" table. There can be many of these.', ) return parser @@ -87,20 +86,26 @@ def get_parser(): def main(): args = get_parser().parse_args() - if args.fromfile is None and 'SKA' in os.environ: - args.fromfile = Path(os.environ['SKA']) / 'data' / 'agasc' / 'agasc_supplement.h5' - if args.tofile is None and 'SKA' in os.environ: - args.tofile = Path(os.environ['SKA']) / 'data' / 'agasc' / 'rc' / 'agasc_supplement.h5' + if args.fromfile is None and "SKA" in os.environ: + args.fromfile = ( + Path(os.environ["SKA"]) / "data" / "agasc" / "agasc_supplement.h5" + ) + if args.tofile is None and "SKA" in os.environ: + args.tofile = ( + Path(os.environ["SKA"]) / "data" / "agasc" / "rc" / "agasc_supplement.h5" + ) assert args.tofile, 'Option "--to" was not given and SKA is not defined' assert args.fromfile, 'Option "--from" was not given and SKA is not defined' - assert args.fromfile.exists(), f'File {args.fromfile} does not exist' - assert args.tofile.exists(), f'File {args.tofile} does not exist' + assert args.fromfile.exists(), f"File {args.fromfile} does not exist" + assert args.tofile.exists(), f"File {args.tofile} does not exist" if not args.o.parent.exists(): args.o.parent.mkdir() - with open(args.o, 'w') as fh: - diff = diff_to_html(diff_files(args.fromfile, args.tofile, exclude_mags=args.exclude_mags)) + with open(args.o, "w") as fh: + diff = diff_to_html( + diff_files(args.fromfile, args.tofile, exclude_mags=args.exclude_mags) + ) fh.write(diff) diff --git a/agasc/scripts/supplement_tasks.py b/agasc/scripts/supplement_tasks.py index ab755aa9..b4f4e227 100644 --- a/agasc/scripts/supplement_tasks.py +++ b/agasc/scripts/supplement_tasks.py @@ -9,31 +9,29 @@ * schedule-promotion: schedule supplement promotion """ -import os -import subprocess import argparse -from pathlib import Path -import shutil import getpass +import os import platform +import shutil +import subprocess from email.mime.text import MIMEText +from pathlib import Path from cxotime import CxoTime - -AGASC_DATA = Path(os.environ['SKA']) / 'data' / 'agasc' +AGASC_DATA = Path(os.environ["SKA"]) / "data" / "agasc" def email_promotion_report( - filenames, - destdir, - to, - sender=f"{getpass.getuser()}@{platform.uname()[1]}" + filenames, destdir, to, sender=f"{getpass.getuser()}@{platform.uname()[1]}" ): date = CxoTime().date[:14] filenames = "- " + "\n- ".join([str(f) for f in filenames]) - msg = MIMEText(f"The following files were promoted to {destdir} on {date}:\n{filenames}") + msg = MIMEText( + f"The following files were promoted to {destdir} on {date}:\n{filenames}" + ) msg["From"] = sender msg["To"] = to msg["Subject"] = "AGASC RC supplement promoted" @@ -45,21 +43,20 @@ def update_rc(): """ Update the supplement in $SKA/data/agasc/rc """ - filenames = list((AGASC_DATA / 'rc' / 'promote').glob('*')) + filenames = list((AGASC_DATA / "rc" / "promote").glob("*")) if filenames: for file in filenames: file.rename(AGASC_DATA / file.name) - email_promotion_report( - filenames, - destdir=AGASC_DATA, - to='aca@cfa.harvard.edu' - ) - - subprocess.run([ - 'task_schedule3.pl', - '-config', - 'agasc/task_schedule_update_supplement_rc.cfg' - ]) + email_promotion_report(filenames, destdir=AGASC_DATA, to="aca@cfa.harvard.edu") + + subprocess.run( + [ + "task_schedule3.pl", + "-config", + "agasc/task_schedule_update_supplement_rc.cfg", + ], + check=False, + ) def disposition(): @@ -68,11 +65,14 @@ def disposition(): This actually schedules a task to run. """ - subprocess.run([ - 'task_schedule3.pl', - '-config', - 'agasc/task_schedule_supplement_dispositions.cfg' - ]) + subprocess.run( + [ + "task_schedule3.pl", + "-config", + "agasc/task_schedule_supplement_dispositions.cfg", + ], + check=False, + ) def stage_promotion(): @@ -82,26 +82,29 @@ def stage_promotion(): It just copies the files into $SKA/data/agasc/rc/promote. The promotion task_schedule will move them to $SKA/data/agasc. """ - promote_dir = AGASC_DATA / 'rc' / 'promote' - rc_dir = AGASC_DATA / 'rc' + promote_dir = AGASC_DATA / "rc" / "promote" + rc_dir = AGASC_DATA / "rc" promote_dir.mkdir(exist_ok=True) - for filename in ['agasc_supplement.h5', 'mag_stats_agasc.fits', 'mag_stats_obsid.fits']: + for filename in [ + "agasc_supplement.h5", + "mag_stats_agasc.fits", + "mag_stats_obsid.fits", + ]: shutil.copy(rc_dir / filename, promote_dir / filename) TASKS = { - 'update-rc': update_rc, - 'disposition': disposition, - 'schedule-promotion': stage_promotion, + "update-rc": update_rc, + "disposition": disposition, + "schedule-promotion": stage_promotion, } def get_parser(): parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument('task', choices=TASKS) + parser.add_argument("task", choices=TASKS) return parser @@ -110,5 +113,5 @@ def main(): TASKS[args.task]() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/agasc/scripts/update_mag_supplement.py b/agasc/scripts/update_mag_supplement.py index 8e59bf89..6e4a2df8 100755 --- a/agasc/scripts/update_mag_supplement.py +++ b/agasc/scripts/update_mag_supplement.py @@ -5,116 +5,163 @@ """ -import os -from pathlib import Path import argparse import logging -import yaml +import os +from pathlib import Path + import pyyaks.logger +import yaml +from cxotime import CxoTime +from cxotime import units as u -from agasc.supplement.magnitudes import star_obs_catalogs -from agasc.supplement.magnitudes import update_mag_supplement from agasc.scripts import update_supplement -from cxotime import CxoTime, units as u +from agasc.supplement.magnitudes import star_obs_catalogs, update_mag_supplement def get_parser(): parser = argparse.ArgumentParser( - description=__doc__, - parents=[update_supplement.get_obs_status_parser()] + description=__doc__, parents=[update_supplement.get_obs_status_parser()] + ) + parser.add_argument( + "--start", + help=( + "Include only stars observed after this time." + " CxoTime-compatible time stamp." + " Default: now - 30 days." + ), + ) + parser.add_argument( + "--stop", + help=( + "Include only stars observed before this time." + " CxoTime-compatible time stamp." + " Default: now." + ), + ) + parser.add_argument( + "--whole-history", + help="Include all star observations and ignore --start/stop.", + action="store_true", + default=False, + ) + parser.add_argument( + "--agasc-id-file", + help=( + "Include only observations of stars whose AGASC IDs are specified " + "in this file, one per line." + ), + ) + parser.add_argument( + "--output-dir", + help=( + "Directory where agasc_supplement.h5 is located." + "Other output is placed here as well. Default: ." + ), + default=".", + ) + parser.add_argument( + "--include-bad", + help='Do not exclude "bad" stars from magnitude estimates. Default: False', + action="store_true", + default=False, + ) + report = parser.add_argument_group("Reporting") + report.add_argument( + "--report", + help="Generate HTML report for the period covered. Default: False", + action="store_true", + default=False, + ) + report.add_argument( + "--reports-dir", + help=( + "Directory where to place reports." + " Default: /supplement_reports/weekly." + ), + ) + + other = parser.add_argument_group("Other") + other.add_argument( + "--multi-process", + help="Use multi-processing to accelerate run.", + action="store_true", + default=False, + ) + other.add_argument( + "--log-level", default="info", choices=["debug", "info", "warning", "error"] + ) + other.add_argument( + "--no-progress", + dest="no_progress", + help="Do not show a progress bar", + action="store_true", + ) # this has no default, it will be None. + other.add_argument( + "--args-file", + help=( + "YAML file with arguments to " + "agasc.supplement.magnitudes.update_mag_supplement.do" + ), + ) + other.add_argument( + "--dry-run", + action="store_true", + help="Dry run (no actual file or database updates)", ) - parser.add_argument('--start', - help='Include only stars observed after this time.' - ' CxoTime-compatible time stamp.' - ' Default: now - 30 days.') - parser.add_argument('--stop', - help='Include only stars observed before this time.' - ' CxoTime-compatible time stamp.' - ' Default: now.') - parser.add_argument('--whole-history', - help='Include all star observations and ignore --start/stop.', - action='store_true', default=False) - parser.add_argument('--agasc-id-file', - help='Include only observations of stars whose AGASC IDs are specified ' - 'in this file, one per line.') - parser.add_argument('--output-dir', - help='Directory where agasc_supplement.h5 is located.' - 'Other output is placed here as well. Default: .', - default='.') - parser.add_argument('--include-bad', - help='Do not exclude "bad" stars from magnitude estimates. Default: False', - action='store_true', default=False) - report = parser.add_argument_group('Reporting') - report.add_argument('--report', - help='Generate HTML report for the period covered. Default: False', - action='store_true', default=False) - report.add_argument('--reports-dir', - help='Directory where to place reports.' - ' Default: /supplement_reports/weekly.') - - other = parser.add_argument_group('Other') - other.add_argument('--multi-process', - help="Use multi-processing to accelerate run.", - action='store_true', default=False) - other.add_argument('--log-level', - default='info', - choices=['debug', 'info', 'warning', 'error']) - other.add_argument('--no-progress', dest='no_progress', - help='Do not show a progress bar', - action='store_true') # this has no default, it will be None. - other.add_argument('--args-file', - help='YAML file with arguments to ' - 'agasc.supplement.magnitudes.update_mag_supplement.do') - other.add_argument("--dry-run", - action="store_true", - help="Dry run (no actual file or database updates)") return parser def main(): import kadi.commands - kadi.commands.conf.commands_version = '1' - logger = logging.getLogger('agasc.supplement') + kadi.commands.conf.commands_version = "1" + + logger = logging.getLogger("agasc.supplement") the_parser = get_parser() args = the_parser.parse_args() if args.args_file: args.args_file = Path(args.args_file) if not args.args_file.exists(): - logger.error(f'File does not exist: {args.args_file}') + logger.error(f"File does not exist: {args.args_file}") the_parser.exit(1) with open(args.args_file) as fh: file_args = yaml.load(fh, Loader=yaml.SafeLoader) the_parser.set_defaults(**file_args) args = the_parser.parse_args() - status_to_int = {'true': 1, 'false': 0, 'ok': 1, 'good': 1, 'bad': 0} + status_to_int = {"true": 1, "false": 0, "ok": 1, "good": 1, "bad": 0} if args.status and args.status.lower() in status_to_int: args.status = status_to_int[args.status.lower()] args.output_dir = Path(os.path.expandvars(args.output_dir)) if args.reports_dir is None: - args.reports_dir = args.output_dir / 'supplement_reports' / 'weekly' + args.reports_dir = args.output_dir / "supplement_reports" / "weekly" else: args.reports_dir = Path(os.path.expandvars(args.reports_dir)) if args.whole_history: if args.start or args.stop: - logger.error('--whole-history argument is incompatible with --start/--stop arguments') + logger.error( + "--whole-history argument is incompatible with --start/--stop arguments" + ) the_parser.exit(1) args.start = None args.stop = None pyyaks.logger.get_logger( - name='agasc.supplement', + name="agasc.supplement", level=args.log_level.upper(), - format="%(asctime)s %(message)s" + format="%(asctime)s %(message)s", ) - if (((args.obsid or args.mp_starcat_time) and not args.status) - or (not (args.obsid or args.mp_starcat_time) and args.status)): + if ((args.obsid or args.mp_starcat_time) and not args.status) or ( + not (args.obsid or args.mp_starcat_time) and args.status + ): logger.error( - 'To override OBS status, both --obs/mp-starcat-time and --status options are needed.') + "To override OBS status, both --obs/mp-starcat-time and --status options" + " are needed." + ) the_parser.exit(1) star_obs_catalogs.load(args.stop) @@ -122,7 +169,7 @@ def main(): # set the list of AGASC IDs from file if specified. If not, it will include all. agasc_ids = [] if args.agasc_id_file: - with open(args.agasc_id_file, 'r') as f: + with open(args.agasc_id_file, "r") as f: agasc_ids = [int(line.strip()) for line in f.readlines()] # update 'bad' and 'obs' tables in supplement @@ -131,12 +178,16 @@ def main(): # set start/stop times if args.whole_history: if args.start or args.stop: - raise ValueError('incompatible arguments: whole_history and start/stop') - args.start = CxoTime(star_obs_catalogs.STARS_OBS['mp_starcat_time']).min().date - args.stop = CxoTime(star_obs_catalogs.STARS_OBS['mp_starcat_time']).max().date + raise ValueError("incompatible arguments: whole_history and start/stop") + args.start = CxoTime(star_obs_catalogs.STARS_OBS["mp_starcat_time"]).min().date + args.stop = CxoTime(star_obs_catalogs.STARS_OBS["mp_starcat_time"]).max().date else: args.stop = CxoTime(args.stop).date if args.stop else CxoTime.now().date - args.start = CxoTime(args.start).date if args.start else (CxoTime.now() - 30 * u.day).date + args.start = ( + CxoTime(args.start).date + if args.start + else (CxoTime.now() - 30 * u.day).date + ) report_date = None if args.report: @@ -146,13 +197,14 @@ def main(): report_date += ((7 - report_date.datetime.weekday()) % 7) * u.day report_date = CxoTime(report_date.date[:8]) - args_log_file = args.output_dir / 'call_args.yml' + args_log_file = args.output_dir / "call_args.yml" if not args.output_dir.exists(): args.output_dir.mkdir(parents=True) - with open(args_log_file, 'w') as fh: + with open(args_log_file, "w") as fh: # there must be a better way to do this... - yaml_args = {k: str(v) if issubclass(type(v), Path) else v - for k, v in vars(args).items()} + yaml_args = { + k: str(v) if issubclass(type(v), Path) else v for k, v in vars(args).items() + } yaml.dump(yaml_args, fh) update_mag_supplement.do( @@ -168,9 +220,11 @@ def main(): dry_run=args.dry_run, no_progress=args.no_progress, ) - if args.report and (args.reports_dir / f'{report_date.date[:8]}').exists(): - args_log_file.replace(args.reports_dir / f'{report_date.date[:8]}' / args_log_file.name) + if args.report and (args.reports_dir / f"{report_date.date[:8]}").exists(): + args_log_file.replace( + args.reports_dir / f"{report_date.date[:8]}" / args_log_file.name + ) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/agasc/scripts/update_supplement.py b/agasc/scripts/update_supplement.py index 856da197..12d8d02b 100755 --- a/agasc/scripts/update_supplement.py +++ b/agasc/scripts/update_supplement.py @@ -10,18 +10,18 @@ """ import argparse -from pathlib import Path import logging -from cxotime.cxotime import CxoTime -import yaml +from pathlib import Path + import numpy as np import pyyaks.logger +import yaml +from cxotime.cxotime import CxoTime from agasc.supplement.magnitudes import star_obs_catalogs as cat -from agasc.supplement.utils import update_mags_table, update_obs_table, add_bad_star - +from agasc.supplement.utils import add_bad_star, update_mags_table, update_obs_table -logger = logging.getLogger('agasc.supplement') +logger = logging.getLogger("agasc.supplement") def _parse_obs_status_file(filename): @@ -35,79 +35,90 @@ def _parse_obs_status_file(filename): """ with open(filename) as fh: status = yaml.load(fh, Loader=yaml.SafeLoader) - if 'obs' not in status: - status['obs'] = [] - if 'bad' not in status: - status['bad'] = [] - if 'mags' not in status: - status['mags'] = [] + if "obs" not in status: + status["obs"] = [] + if "bad" not in status: + status["bad"] = [] + if "mags" not in status: + status["mags"] = [] - if hasattr(status['bad'], 'items'): - status['bad'] = list(status['bad'].items()) + if hasattr(status["bad"], "items"): + status["bad"] = list(status["bad"].items()) return status def _sanitize_args(status): - for star in status['mags']: - if 'last_obs_time' in star: - star['last_obs_time'] = CxoTime(star['last_obs_time']).cxcsec + for star in status["mags"]: + if "last_obs_time" in star: + star["last_obs_time"] = CxoTime(star["last_obs_time"]).cxcsec else: - obs = cat.STARS_OBS[cat.STARS_OBS['agasc_id'] == star['agasc_id']] + obs = cat.STARS_OBS[cat.STARS_OBS["agasc_id"] == star["agasc_id"]] if len(obs) == 0: - raise Exception(f"Can not guess last_obs_time for agasc_id={star['agasc_id']}") - star['last_obs_time'] = CxoTime(obs['mp_starcat_time']).max().cxcsec + raise Exception( + f"Can not guess last_obs_time for agasc_id={star['agasc_id']}" + ) + star["last_obs_time"] = CxoTime(obs["mp_starcat_time"]).max().cxcsec - for value in status['obs']: + for value in status["obs"]: if cat.STARS_OBS is None: - raise RuntimeError('Observation catalog is not initialized') - if 'mp_starcat_time' in value: - key = 'mp_starcat_time' - rows = cat.STARS_OBS[cat.STARS_OBS['mp_starcat_time'] == value['mp_starcat_time']] - elif 'obsid' in value: - key = 'obsid' - rows = cat.STARS_OBS[cat.STARS_OBS['obsid'] == value['obsid']] + raise RuntimeError("Observation catalog is not initialized") + if "mp_starcat_time" in value: + key = "mp_starcat_time" + rows = cat.STARS_OBS[ + cat.STARS_OBS["mp_starcat_time"] == value["mp_starcat_time"] + ] + elif "obsid" in value: + key = "obsid" + rows = cat.STARS_OBS[cat.STARS_OBS["obsid"] == value["obsid"]] else: - raise Exception('Need to specify mp_starcat_time or OBSID') + raise Exception("Need to specify mp_starcat_time or OBSID") if len(rows) == 0: raise Exception( - f'Observation catalog has no observation with {key}={value[key]}' + f"Observation catalog has no observation with {key}={value[key]}" ) - if 'agasc_id' not in value: - value['agasc_id'] = list(sorted(rows['agasc_id'])) + if "agasc_id" not in value: + value["agasc_id"] = sorted(rows["agasc_id"]) else: - value['agasc_id'] = (list(np.atleast_1d(value['agasc_id']))) - if 'comments' not in value: - value['comments'] = '' - if 'mp_starcat_time' not in value: - value['mp_starcat_time'] = rows['mp_starcat_time'][0] - if 'obsid' not in value: - value['obsid'] = rows['obsid'][0] + value["agasc_id"] = list(np.atleast_1d(value["agasc_id"])) + if "comments" not in value: + value["comments"] = "" + if "mp_starcat_time" not in value: + value["mp_starcat_time"] = rows["mp_starcat_time"][0] + if "obsid" not in value: + value["obsid"] = rows["obsid"][0] # sanity checks: # AGASC IDs are not checked to exist because a star could be observed without it being # in the starcheck catalog (a spoiler star), but the observation must exist and have # matching mp_starcat_time and OBSID - if rows['obsid'][0] != value['obsid']: - raise Exception(f'inconsistent observation spec {value}') - if rows['mp_starcat_time'][0] != value['mp_starcat_time']: - raise Exception(f'inconsistent observation spec {value}') + if rows["obsid"][0] != value["obsid"]: + raise Exception(f"inconsistent observation spec {value}") + if rows["mp_starcat_time"][0] != value["mp_starcat_time"]: + raise Exception(f"inconsistent observation spec {value}") rows = [] - for value in status['obs']: - for agasc_id in value['agasc_id']: + for value in status["obs"]: + for agasc_id in value["agasc_id"]: row = value.copy() - row['agasc_id'] = agasc_id + row["agasc_id"] = agasc_id rows.append(row) - status['obs'] = rows + status["obs"] = rows return status -def parse_args(filename=None, bad_star_id=None, bad_star_source=None, - obsid=None, status=None, comments='', agasc_id=None, mp_starcat_time=None, - **_ - ): +def parse_args( + filename=None, + bad_star_id=None, + bad_star_source=None, + obsid=None, + status=None, + comments="", + agasc_id=None, + mp_starcat_time=None, + **_, +): """ Combine obs/bad-star status from file and from arguments. @@ -131,35 +142,41 @@ def parse_args(filename=None, bad_star_id=None, bad_star_source=None, mags = [] if bad_star_id and bad_star_source is None: - raise RuntimeError('If you specify bad_star, you must specify bad_star_source') + raise RuntimeError("If you specify bad_star, you must specify bad_star_source") if filename is not None: status_file = _parse_obs_status_file(filename) - bad = status_file['bad'] - obs_status_override = status_file['obs'] - mags = status_file['mags'] + bad = status_file["bad"] + obs_status_override = status_file["obs"] + mags = status_file["mags"] if (obsid is not None or mp_starcat_time is not None) and status is not None: row = { - 'mp_starcat_time': mp_starcat_time, - 'agasc_id': agasc_id, - 'obsid': obsid, - 'status': status, - 'comments': comments + "mp_starcat_time": mp_starcat_time, + "agasc_id": agasc_id, + "obsid": obsid, + "status": status, + "comments": comments, } - optional = ['obsid', 'mp_starcat_time', 'agasc_id', 'comments'] + optional = ["obsid", "mp_starcat_time", "agasc_id", "comments"] obs_status_override.append( - {key: row[key] for key in row if key not in optional or row[key] is not None} + { + key: row[key] + for key in row + if key not in optional or row[key] is not None + } ) for bs in bad_star_id: bad_dict = dict(bad) if bs in bad_dict and bad_dict[bs] != bad_star_source: - raise RuntimeError('name collision: conflicting bad_star in file and in args') + raise RuntimeError( + "name collision: conflicting bad_star in file and in args" + ) if (bs, bad_star_source) not in bad: bad.append((bs, bad_star_source)) - status = {'obs': obs_status_override, 'bad': bad, 'mags': mags} + status = {"obs": obs_status_override, "bad": bad, "mags": mags} status = _sanitize_args(status) return status @@ -172,24 +189,41 @@ def get_obs_status_parser(): """ parser = argparse.ArgumentParser(add_help=False) status = parser.add_argument_group( - 'OBS/star status', - 'options to modify the "bads" and "obs" tables in AGASC supplement. ' - 'Modifications to supplement happen before all magnitude estimates are made.' + "OBS/star status", + ( + 'options to modify the "bads" and "obs" tables in AGASC supplement.' + " Modifications to supplement happen before all magnitude estimates are" + " made." + ), + ) + status.add_argument( + "--obs-status-file", + help=( + "YAML file with star/observation status. " + "More info at https://sot.github.io/agasc/supplement.html" + ), + ) + status.add_argument( + "--mp-starcat-time", + help=( + "Observation starcat time for status override. " + "Usually the mission planning catalog time" + ), + ) + status.add_argument("--obsid", help="OBSID for status override.", type=int) + status.add_argument("--agasc-id", help="AGASC ID for status override.", type=int) + status.add_argument("--status", help="Status to override.") + status.add_argument("--comments", help="Comments for status override.", default="") + status.add_argument( + "--bad-star-id", + help="AGASC ID of bad star.", + default=[], + action="append", + type=int, + ) + status.add_argument( + "--bad-star-source", type=int, help="Source identifier indicating provenance." ) - status.add_argument('--obs-status-file', - help='YAML file with star/observation status. ' - 'More info at https://sot.github.io/agasc/supplement.html') - status.add_argument('--mp-starcat-time', - help='Observation starcat time for status override. ' - 'Usually the mission planning catalog time') - status.add_argument('--obsid', help='OBSID for status override.', type=int) - status.add_argument('--agasc-id', help='AGASC ID for status override.', type=int) - status.add_argument('--status', help='Status to override.') - status.add_argument('--comments', help='Comments for status override.', default='') - status.add_argument('--bad-star-id', help='AGASC ID of bad star.', - default=[], action='append', type=int) - status.add_argument("--bad-star-source", type=int, - help="Source identifier indicating provenance.") return parser @@ -201,19 +235,22 @@ def get_parser(): :return: argparse.ArgumentParser """ parse = argparse.ArgumentParser( - description=__doc__, - parents=[get_obs_status_parser()] + description=__doc__, parents=[get_obs_status_parser()] + ) + parse.add_argument( + "--output-dir", + default=".", + type=Path, + help="Directory containing agasc_supplement.h5 (default='.')", + ) + parse.add_argument( + "--log-level", default="info", choices=["debug", "info", "warning", "error"] + ) + parse.add_argument( + "--dry-run", + action="store_true", + help="Dry run (no actual file or database updates)", ) - parse.add_argument("--output-dir", - default='.', - type=Path, - help="Directory containing agasc_supplement.h5 (default='.')") - parse.add_argument('--log-level', - default='info', - choices=['debug', 'info', 'warning', 'error']) - parse.add_argument("--dry-run", - action="store_true", - help="Dry run (no actual file or database updates)") return parse @@ -225,34 +262,28 @@ def update(args): Returns a list of AGASC IDs whose records were updated in the supplement """ - status = parse_args( - filename=args.obs_status_file, - **vars(args) - ) + status = parse_args(filename=args.obs_status_file, **vars(args)) - if status['mags']: + if status["mags"]: update_mags_table( - args.output_dir / 'agasc_supplement.h5', - status['mags'], - dry_run=args.dry_run + args.output_dir / "agasc_supplement.h5", + status["mags"], + dry_run=args.dry_run, ) - if status['obs']: + if status["obs"]: update_obs_table( - args.output_dir / 'agasc_supplement.h5', - status['obs'], - dry_run=args.dry_run + args.output_dir / "agasc_supplement.h5", status["obs"], dry_run=args.dry_run ) - if status['bad']: + if status["bad"]: add_bad_star( - args.output_dir / 'agasc_supplement.h5', - status['bad'], - dry_run=args.dry_run + args.output_dir / "agasc_supplement.h5", status["bad"], dry_run=args.dry_run ) - agasc_ids = sorted(set([o['agasc_id'] for o in status['obs']] - + [o[0] for o in status['bad']])) + agasc_ids = sorted( + set([o["agasc_id"] for o in status["obs"]] + [o[0] for o in status["bad"]]) + ) return agasc_ids @@ -261,23 +292,24 @@ def main(): The main function for the update_obs_status script. """ import kadi.commands - kadi.commands.conf.commands_version = '1' + + kadi.commands.conf.commands_version = "1" args = get_parser().parse_args() - status_to_int = {'ok': 0, 'good': 0, 'bad': 1} + status_to_int = {"ok": 0, "good": 0, "bad": 1} if args.status and args.status.lower() in status_to_int: args.status = status_to_int[args.status.lower()] pyyaks.logger.get_logger( - name='agasc.supplement', + name="agasc.supplement", level=args.log_level.upper(), - format="%(asctime)s %(message)s" + format="%(asctime)s %(message)s", ) cat.load() update(args) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/agasc/supplement/magnitudes/mag_estimate.py b/agasc/supplement/magnitudes/mag_estimate.py index 3ea6887b..076689a1 100644 --- a/agasc/supplement/magnitudes/mag_estimate.py +++ b/agasc/supplement/magnitudes/mag_estimate.py @@ -2,107 +2,110 @@ Functions to estimate observed ACA magnitudes """ +import collections +import logging import sys import traceback -import logging -import collections -import scipy.stats -import scipy.special -import numpy as np import numba +import numpy as np +import scipy.special +import scipy.stats +import Ska.quatutil from astropy.table import Table, vstack - from Chandra.Time import DateTime -from cheta import fetch -from Quaternion import Quat -import Ska.quatutil -from mica.archive import aca_l0 -from mica.archive.aca_dark.dark_cal import get_dark_cal_image from chandra_aca.transform import count_rate_to_mag, pixels_to_yagzag +from cheta import fetch from cxotime import CxoTime from kadi import events +from mica.archive import aca_l0 +from mica.archive.aca_dark.dark_cal import get_dark_cal_image +from Quaternion import Quat -from . import star_obs_catalogs from agasc import get_star +from . import star_obs_catalogs -logger = logging.getLogger('agasc.supplement') +logger = logging.getLogger("agasc.supplement") MAX_MAG = 15 MASK = { - 'mouse_bit': np.array([[True, True, True, True, True, True, True, True], - [True, True, False, False, False, False, True, True], - [True, False, False, False, False, False, False, True], - [True, False, False, False, False, False, False, True], - [True, False, False, False, False, False, False, True], - [True, False, False, False, False, False, False, True], - [True, True, False, False, False, False, True, True], - [True, True, True, True, True, True, True, True]]) + "mouse_bit": np.array( + [ + [True, True, True, True, True, True, True, True], + [True, True, False, False, False, False, True, True], + [True, False, False, False, False, False, False, True], + [True, False, False, False, False, False, False, True], + [True, False, False, False, False, False, False, True], + [True, False, False, False, False, False, False, True], + [True, True, False, False, False, False, True, True], + [True, True, True, True, True, True, True, True], + ] + ) } EXCEPTION_MSG = { - -1: 'Unknown', - 0: 'OK', - 1: 'No level 0 data', - 2: 'No telemetry data', - 3: 'Mismatch in telemetry between aca_l0 and cheta', - 4: 'Time mismatch between cheta and level0', - 5: 'Failed job', - 6: 'Suspect observation' + -1: "Unknown", + 0: "OK", + 1: "No level 0 data", + 2: "No telemetry data", + 3: "Mismatch in telemetry between aca_l0 and cheta", + 4: "Time mismatch between cheta and level0", + 5: "Failed job", + 6: "Suspect observation", } EXCEPTION_CODES = collections.defaultdict(lambda: -1) EXCEPTION_CODES.update({msg: code for code, msg in EXCEPTION_MSG.items() if code > 0}) class MagStatsException(Exception): - def __init__(self, msg='', agasc_id=None, obsid=None, mp_starcat_time=None, - **kwargs): + def __init__( + self, msg="", agasc_id=None, obsid=None, mp_starcat_time=None, **kwargs + ): super().__init__(msg) self.error_code = EXCEPTION_CODES[msg] self.msg = msg self.agasc_id = agasc_id - self.obsid = obsid[0] if type(obsid) is list and len(obsid) == 1 else obsid - self.mp_starcat_time = (mp_starcat_time[0] if type(mp_starcat_time) is list - and len(mp_starcat_time) == 1 else mp_starcat_time) + self.obsid = obsid[0] if isinstance(obsid, list) and len(obsid) == 1 else obsid + self.mp_starcat_time = ( + mp_starcat_time[0] + if isinstance(mp_starcat_time, list) and len(mp_starcat_time) == 1 + else mp_starcat_time + ) for k in kwargs: setattr(self, k, kwargs[k]) exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type is None: - self.exception = { - 'type': '', - 'value': '', - 'traceback': '' - } + self.exception = {"type": "", "value": "", "traceback": ""} else: stack_trace = [] for step in traceback.extract_tb(exc_traceback): - stack_trace.append( - f" in {step.filename}:{step.lineno}/{step.name}:" - ) + stack_trace.append(f" in {step.filename}:{step.lineno}/{step.name}:") stack_trace.append(f" {step.line}") self.exception = { - 'type': exc_type.__name__, - 'value': str(exc_value), - 'traceback': '\n'.join(stack_trace) + "type": exc_type.__name__, + "value": str(exc_value), + "traceback": "\n".join(stack_trace), } def __str__(self): - return f'MagStatsException: {self.msg} (agasc_id: {self.agasc_id}, ' \ - f'obsid: {self.obsid}, mp_starcat_time: {self.mp_starcat_time})' + return ( + f"MagStatsException: {self.msg} (agasc_id: {self.agasc_id}, " + f"obsid: {self.obsid}, mp_starcat_time: {self.mp_starcat_time})" + ) def __iter__(self): - yield 'error_code', self.error_code - yield 'msg', self.msg - yield 'agasc_id', self.agasc_id - yield 'obsid', self.obsid - yield 'mp_starcat_time', self.mp_starcat_time - yield 'exception_type', self.exception['type'] - yield 'traceback', self.exception['traceback'] + yield "error_code", self.error_code + yield "msg", self.msg + yield "agasc_id", self.agasc_id + yield "obsid", self.obsid + yield "mp_starcat_time", self.mp_starcat_time + yield "exception_type", self.exception["type"] + yield "traceback", self.exception["traceback"] def _magnitude_correction(time, mag_aca): @@ -113,17 +116,22 @@ def _magnitude_correction(time, mag_aca): :param mag_aca: np.array :return: np.array """ - params = {"t_ref": "2011-01-01 12:00:00.000", - "p": [0.005899340720522751, - 0.12029019332761458, - -2.99386247406073e-10, - -6.9534637950633265, - 0.7916261423307238]} - - q = params['p'] - t_ref = DateTime(params['t_ref']) - dmag = (q[0] + (q[1] + q[2] * np.atleast_1d(time)) - * np.exp(q[3] + q[4] * np.atleast_1d(mag_aca))) + params = { + "t_ref": "2011-01-01 12:00:00.000", + "p": [ + 0.005899340720522751, + 0.12029019332761458, + -2.99386247406073e-10, + -6.9534637950633265, + 0.7916261423307238, + ], + } + + q = params["p"] + t_ref = DateTime(params["t_ref"]) + dmag = q[0] + (q[1] + q[2] * np.atleast_1d(time)) * np.exp( + q[3] + q[4] * np.atleast_1d(mag_aca) + ) dmag[np.atleast_1d(time) < t_ref.secs] = 0 return np.squeeze(dmag) @@ -135,14 +143,14 @@ def get_responsivity(time): This was estimated with bright stars that were observed more than a hundred times during the mission. More details in the `responsivity notebook`_: - .. _responsivity notebook: https://nbviewer.jupyter.org/urls/cxc.cfa.harvard.edu/mta/ASPECT/jgonzalez/mag_stats/notebooks/03-high_mag_responsivity-fit.ipynb # noqa + .. _responsivity notebook: https://nbviewer.jupyter.org/urls/cxc.cfa.harvard.edu/mta/ASPECT/jgonzalez/mag_stats/notebooks/03-high_mag_responsivity-fit.ipynb :param time: float Time in CXC seconds :return: - """ - a, b, c = [3.19776750e-02, 5.35201479e+08, 8.49670756e+07] - return - a * (1 + scipy.special.erf((time - b) / c)) / 2 + """ # noqa: E501 + a, b, c = [3.19776750e-02, 5.35201479e08, 8.49670756e07] + return -a * (1 + scipy.special.erf((time - b) / c)) / 2 def get_droop_systematic_shift(magnitude): @@ -153,12 +161,12 @@ def get_droop_systematic_shift(magnitude): The magnitude shift is time-independent. It depends only on the catalog magnitude and is zero for bright stars. More details in the `droop notebook`_: - .. _droop notebook: https://nbviewer.jupyter.org/urls/cxc.cfa.harvard.edu/mta/ASPECT/jgonzalez/mag_stats/notebooks/04-DroopAfterSubtractionAndResponsivity-fit.ipynb # noqa + .. _droop notebook: https://nbviewer.jupyter.org/urls/cxc.cfa.harvard.edu/mta/ASPECT/jgonzalez/mag_stats/notebooks/04-DroopAfterSubtractionAndResponsivity-fit.ipynb :param magnitude: float Catalog ACA magnitude :return: - """ + """ # noqa: E501 a, b = [11.25572, 0.59486369] return np.exp((magnitude - a) / b) @@ -226,58 +234,64 @@ def get_star_position(star, telem): aca_misalign = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) rad_to_arcsec = 206264.81 - q = np.array([telem['AOATTQT1'], - telem['AOATTQT2'], - telem['AOATTQT3'], - telem['AOATTQT4']]).transpose() + q = np.array( + [telem["AOATTQT1"], telem["AOATTQT2"], telem["AOATTQT3"], telem["AOATTQT4"]] + ).transpose() norm = np.sum(q**2, axis=1, keepdims=True) # I am just normalizing q, just in case. n = np.squeeze(np.sqrt(norm)) - q[n != 0] /= np.sqrt(norm)[n != 0] # prevent warning when dividing by zero (it happens) + q[n != 0] /= np.sqrt(norm)[ + n != 0 + ] # prevent warning when dividing by zero (it happens) q_att = Quat(q=q) ts = q_att.transform - star_pos_eci = Ska.quatutil.radec2eci(star['RA_PMCORR'], star['DEC_PMCORR']) - d_aca = np.dot(np.dot(aca_misalign, ts.transpose(0, 2, 1)), - star_pos_eci).transpose() + star_pos_eci = Ska.quatutil.radec2eci(star["RA_PMCORR"], star["DEC_PMCORR"]) + d_aca = np.dot( + np.dot(aca_misalign, ts.transpose(0, 2, 1)), star_pos_eci + ).transpose() yag = np.arctan2(d_aca[:, 1], d_aca[:, 0]) * rad_to_arcsec zag = np.arctan2(d_aca[:, 2], d_aca[:, 0]) * rad_to_arcsec - logger.debug(f' star position. AGASC_ID={star["AGASC_ID"]}, ' - f'{len(yag)} samples, ({yag[0]}, {zag[0]})...') + logger.debug( + f' star position. AGASC_ID={star["AGASC_ID"]}, ' + f"{len(yag)} samples, ({yag[0]}, {zag[0]})..." + ) return { - 'yang_star': yag, - 'zang_star': zag, + "yang_star": yag, + "zang_star": zag, } # this is in case one has to return empty telemetry -_telem_dtype = [('times', 'float64'), - ('IMGSIZE', 'int32'), - ('IMGROW0', 'int16'), - ('IMGCOL0', 'int16'), - ('IMGRAW', 'float32'), - ('AOACASEQ', '= excluded_range[0]) & (times <= excluded_range[1])) + excluded |= (times >= excluded_range[0]) & (times <= excluded_range[1]) telem.update({k: telem[k][~excluded] for k in telem}) slot_data = slot_data[~excluded] if len(slot_data) == 0: # the intersection was null. - raise MagStatsException('Nothing left after removing excluded ranges', - agasc_id=obs["agasc_id"], - obsid=obs["obsid"], - mp_starcat_time=obs["mp_starcat_time"]) - - for name in ['AOACIIR', 'AOACISP', 'AOACYAN', 'AOACZAN', 'AOACMAG', 'AOACFCT']: - telem[name] = telem[f'{name}{slot}'] - del telem[f'{name}{slot}'] - for name in ['AOACIIR', 'AOACISP']: + raise MagStatsException( + "Nothing left after removing excluded ranges", + agasc_id=obs["agasc_id"], + obsid=obs["obsid"], + mp_starcat_time=obs["mp_starcat_time"], + ) + + for name in ["AOACIIR", "AOACISP", "AOACYAN", "AOACZAN", "AOACMAG", "AOACFCT"]: + telem[name] = telem[f"{name}{slot}"] + del telem[f"{name}{slot}"] + for name in ["AOACIIR", "AOACISP"]: telem[name] = np.char.rstrip(telem[name]) - mag_est_ok = \ - (telem['AOACASEQ'] == 'KALM') & (telem['AOACIIR'] == 'OK') & \ - (telem['AOPCADMD'] == 'NPNT') & (telem['AOACFCT'] == 'TRAK') + mag_est_ok = ( + (telem["AOACASEQ"] == "KALM") + & (telem["AOACIIR"] == "OK") + & (telem["AOPCADMD"] == "NPNT") + & (telem["AOACFCT"] == "TRAK") + ) - assert len(slot_data) == len(mag_est_ok), \ - f'len(slot_data) != len(ok) ({len(slot_data)} != {len(mag_est_ok)})' + assert len(slot_data) == len( + mag_est_ok + ), f"len(slot_data) != len(ok) ({len(slot_data)} != {len(mag_est_ok)})" # etc... - logger.debug(' Adding magnitude estimates') + logger.debug(" Adding magnitude estimates") telem.update(get_mag_from_img(slot_data, start, mag_est_ok)) - logger.debug(' Adding star position') + logger.debug(" Adding star position") telem.update(get_star_position(star=star, telem=telem)) - logger.debug(' Correcting for droop') - droop_shift = get_droop_systematic_shift(star['MAG_ACA']) + logger.debug(" Correcting for droop") + droop_shift = get_droop_systematic_shift(star["MAG_ACA"]) - logger.debug(' Correcting for responsivity') + logger.debug(" Correcting for responsivity") responsivity = get_responsivity(start) - telem['mags'] = telem['mags_img'] - responsivity - droop_shift - telem['mags'][~mag_est_ok] = 0. - telem['mag_est_ok'] = mag_est_ok - - telem['dy'] = np.ones(len(mag_est_ok)) * np.inf - telem['dz'] = np.ones(len(mag_est_ok)) * np.inf - telem['dr'] = np.ones(len(mag_est_ok)) * np.inf - yang = telem['yang_img'] - telem['yang_star'] - zang = telem['zang_img'] - telem['zang_star'] + telem["mags"] = telem["mags_img"] - responsivity - droop_shift + telem["mags"][~mag_est_ok] = 0.0 + telem["mag_est_ok"] = mag_est_ok + + telem["dy"] = np.ones(len(mag_est_ok)) * np.inf + telem["dz"] = np.ones(len(mag_est_ok)) * np.inf + telem["dr"] = np.ones(len(mag_est_ok)) * np.inf + yang = telem["yang_img"] - telem["yang_star"] + zang = telem["zang_img"] - telem["zang_star"] rang = np.sqrt(yang**2 + zang**2) if np.any(mag_est_ok & (rang < 10)): y25, y50, y75 = np.quantile(yang[mag_est_ok & (rang < 10)], [0.25, 0.5, 0.75]) z25, z50, z75 = np.quantile(zang[mag_est_ok & (rang < 10)], [0.25, 0.5, 0.75]) - centroid_outlier = ((yang > y75 + 3 * (y75 - y25)) - | (yang < y25 - 3 * (y75 - y25)) - | (zang > z75 + 3 * (z75 - z25)) - | (zang < z25 - 3 * (z75 - z25))) + centroid_outlier = ( + (yang > y75 + 3 * (y75 - y25)) + | (yang < y25 - 3 * (y75 - y25)) + | (zang > z75 + 3 * (z75 - z25)) + | (zang < z25 - 3 * (z75 - z25)) + ) - telem['dy'] = yang - np.mean(yang[mag_est_ok & ~centroid_outlier]) - telem['dz'] = zang - np.mean(zang[mag_est_ok & ~centroid_outlier]) - telem['dr'] = (telem['dy'] ** 2 + telem['dz'] ** 2) ** .5 + telem["dy"] = yang - np.mean(yang[mag_est_ok & ~centroid_outlier]) + telem["dz"] = zang - np.mean(zang[mag_est_ok & ~centroid_outlier]) + telem["dr"] = (telem["dy"] ** 2 + telem["dz"] ** 2) ** 0.5 return telem @@ -433,32 +477,35 @@ def get_telemetry_by_agasc_id(agasc_id, obsid=None, ignore_exceptions=False): if True, any exception is ignored. Useful in some cases. Default is False. :return: dict """ - logger.debug(f' Getting telemetry for AGASC ID={agasc_id}') + logger.debug(f" Getting telemetry for AGASC ID={agasc_id}") star_obs_catalogs.load() if obsid is None: obs = star_obs_catalogs.STARS_OBS[ - (star_obs_catalogs.STARS_OBS['agasc_id'] == agasc_id)] + (star_obs_catalogs.STARS_OBS["agasc_id"] == agasc_id) + ] else: - obs = star_obs_catalogs.STARS_OBS[(star_obs_catalogs.STARS_OBS['agasc_id'] == agasc_id) - & (star_obs_catalogs.STARS_OBS['obsid'] == obsid)] + obs = star_obs_catalogs.STARS_OBS[ + (star_obs_catalogs.STARS_OBS["agasc_id"] == agasc_id) + & (star_obs_catalogs.STARS_OBS["obsid"] == obsid) + ] if len(obs) > 1: - obs = obs.loc['mp_starcat_time', sorted(obs['mp_starcat_time'])] + obs = obs.loc["mp_starcat_time", sorted(obs["mp_starcat_time"])] telem = [] - for i, o in enumerate(obs): + for _i, o in enumerate(obs): try: t = Table(get_telemetry(o)) - t['obsid'] = o['obsid'] - t['agasc_id'] = agasc_id + t["obsid"] = o["obsid"] + t["agasc_id"] = agasc_id telem.append(t) except Exception: if not ignore_exceptions: logger.info(f'{agasc_id=}, obsid={o["obsid"]} failed') exc_type, exc_value, exc_traceback = sys.exc_info() trace = traceback.extract_tb(exc_traceback) - logger.info(f'{exc_type.__name__} {exc_value}') + logger.info(f"{exc_type.__name__} {exc_value}") for step in trace: - logger.info(f' in {step.filename}:{step.lineno}/{step.name}:') - logger.info(f' {step.line}') + logger.info(f" in {step.filename}:{step.lineno}/{step.name}:") + logger.info(f" {step.line}") raise return vstack(telem) @@ -475,31 +522,38 @@ def add_obs_info(telem, obs_stats): The result of calc_obs_stats. :return: """ - logger.debug(' Adding observation info to telemetry...') - obs_stats['obs_ok'] = ( - (obs_stats['n'] > 10) - & (obs_stats['f_mag_est_ok'] > 0.3) - & (obs_stats['lf_variability_100s'] < 1) + logger.debug(" Adding observation info to telemetry...") + obs_stats["obs_ok"] = ( + (obs_stats["n"] > 10) + & (obs_stats["f_mag_est_ok"] > 0.3) + & (obs_stats["lf_variability_100s"] < 1) ) - obs_stats['comments'] = np.zeros(len(obs_stats), dtype=' 0 - and np.isfinite(s['q75']) and np.isfinite(s['q25'])): - iqr = s['q75'] - s['q25'] - telem['obs_outlier'][o] = ( - telem[o]['mag_est_ok'] & (iqr > 0) - & ((telem[o]['mags'] < s['q25'] - 1.5 * iqr) - | (telem[o]['mags'] > s['q75'] + 1.5 * iqr)) + obsid = s["obsid"] + o = telem["obsid"] == obsid + telem["obs_ok"][o] = np.ones(np.count_nonzero(o), dtype=bool) * s["obs_ok"] + if ( + np.any(telem["mag_est_ok"][o]) + and s["f_mag_est_ok"] > 0 + and np.isfinite(s["q75"]) + and np.isfinite(s["q25"]) + ): + iqr = s["q75"] - s["q25"] + telem["obs_outlier"][o] = ( + telem[o]["mag_est_ok"] + & (iqr > 0) + & ( + (telem[o]["mags"] < s["q25"] - 1.5 * iqr) + | (telem[o]["mags"] > s["q75"] + 1.5 * iqr) + ) ) - logger.debug(f' Adding observation info to telemetry {obsid=}') + logger.debug(f" Adding observation info to telemetry {obsid=}") return telem @@ -507,7 +561,7 @@ def add_obs_info(telem, obs_stats): def staggered_aca_slice(array_in, array_out, row, col): for i in np.arange(len(row)): if row[i] + 8 < 1024 and col[i] + 8 < 1024: - array_out[i] = array_in[row[i]:row[i] + 8, col[i]:col[i] + 8] + array_out[i] = array_in[row[i] : row[i] + 8, col[i] : col[i] + 8] def get_mag_from_img(slot_data, t_start, ok=True): @@ -523,26 +577,29 @@ def get_mag_from_img(slot_data, t_start, ok=True): Only magnitudes for entries with ok=True are calculated. The rest are set to MAX_MAG. :return: """ - logger.debug(' magnitude from images...') - dark_cal = get_dark_cal_image(t_start, 'nearest', - t_ccd_ref=np.mean(slot_data['TEMPCCD'] - 273.16), - aca_image=False) + logger.debug(" magnitude from images...") + dark_cal = get_dark_cal_image( + t_start, + "nearest", + t_ccd_ref=np.mean(slot_data["TEMPCCD"] - 273.16), + aca_image=False, + ) # all images will be 8x8, with a centered mask, imgrow will always be the one of the 8x8 corner. - imgrow_8x8 = np.where(slot_data['IMGSIZE'] == 8, - slot_data['IMGROW0'], - slot_data['IMGROW0'] - 1 - ) - imgcol_8x8 = np.where(slot_data['IMGSIZE'] == 8, - slot_data['IMGCOL0'], - slot_data['IMGCOL0'] - 1 - ) + imgrow_8x8 = np.where( + slot_data["IMGSIZE"] == 8, slot_data["IMGROW0"], slot_data["IMGROW0"] - 1 + ) + imgcol_8x8 = np.where( + slot_data["IMGSIZE"] == 8, slot_data["IMGCOL0"], slot_data["IMGCOL0"] - 1 + ) # subtract closest dark cal dark = np.zeros([len(slot_data), 8, 8], dtype=np.float64) - staggered_aca_slice(dark_cal.astype(float), dark, 512 + imgrow_8x8, 512 + imgcol_8x8) - img_sub = slot_data['IMGRAW'] - dark * 1.696 / 5 - img_sub.mask |= MASK['mouse_bit'] + staggered_aca_slice( + dark_cal.astype(float), dark, 512 + imgrow_8x8, 512 + imgcol_8x8 + ) + img_sub = slot_data["IMGRAW"] - dark * 1.696 / 5 + img_sub.mask |= MASK["mouse_bit"] # calculate magnitude mag = np.ones(len(slot_data)) * MAX_MAG @@ -552,7 +609,7 @@ def get_mag_from_img(slot_data, t_start, ok=True): mag[mag > MAX_MAG] = MAX_MAG # this extra step is to investigate the background scale dark = np.ma.array(dark * 1.696 / 5, mask=img_sub.mask) - img_raw = np.ma.array(slot_data['IMGRAW'], mask=img_sub.mask) + img_raw = np.ma.array(slot_data["IMGRAW"], mask=img_sub.mask) dark_count = np.ma.sum(np.ma.sum(dark, axis=1), axis=1) img_count = np.ma.sum(np.ma.sum(img_raw, axis=1), axis=1) @@ -560,102 +617,127 @@ def get_mag_from_img(slot_data, t_start, ok=True): yag = np.zeros(len(slot_data)) zag = np.zeros(len(slot_data)) pixel_center = np.arange(8) + 0.5 - projected_image = np.ma.sum(slot_data['IMGRAW'], axis=1) - col = np.ma.sum(pixel_center * projected_image, axis=1) / np.ma.sum(projected_image, axis=1) - projected_image = np.ma.sum(slot_data['IMGRAW'], axis=2) - row = np.ma.sum(pixel_center * projected_image, axis=1) / np.ma.sum(projected_image, axis=1) + projected_image = np.ma.sum(slot_data["IMGRAW"], axis=1) + col = np.ma.sum(pixel_center * projected_image, axis=1) / np.ma.sum( + projected_image, axis=1 + ) + projected_image = np.ma.sum(slot_data["IMGRAW"], axis=2) + row = np.ma.sum(pixel_center * projected_image, axis=1) / np.ma.sum( + projected_image, axis=1 + ) y_pixel = row + imgrow_8x8 z_pixel = col + imgcol_8x8 yag[m], zag[m] = pixels_to_yagzag(y_pixel[m], z_pixel[m]) - logger.debug(f' magnitude from images... {len(mag)} samples: {mag[0]:.2f}...') + logger.debug(f" magnitude from images... {len(mag)} samples: {mag[0]:.2f}...") return { - 'mags_img': mag, - 'yang_img': yag, - 'zang_img': zag, - 'counts_img': img_count, - 'counts_dark': dark_count + "mags_img": mag, + "yang_img": yag, + "zang_img": zag, + "counts_img": img_count, + "counts_dark": dark_count, } OBS_STATS_INFO = { - 'agasc_id': 'AGASC ID of the star', - 'obsid': 'OBSID at the time the star catalog is commanded (from kadi.commands)', - 'slot': 'Slot number', - 'type': 'GUI/ACQ/BOT', - 'mp_starcat_time': 'The time at which the star catalog is commanded (from kadi.commands)', - 'tstart': 'Start time of the observation according to kadi.commands (in cxc seconds)', - 'tstop': 'Start time of the observation according to kadi.commands (in cxc seconds)', - 'mag_correction': 'Overall correction applied to the magnitude estimate', - 'responsivity': 'Responsivity correction applied to the magnitude estimate', - 'droop_shift': 'Droop shift correction applied to the magnitude estimate', - 'mag_aca': 'ACA star magnitude from the AGASC catalog', - 'mag_aca_err': 'ACA star magnitude uncertainty from the AGASC catalog', - 'row': - 'Expected row number, based on star location and yanf/zang from mica.archive.starcheck DB', - 'col': - 'Expected col number, based on star location and yanf/zang from mica.archive.starcheck DB', - 'mag_img': 'Magnitude estimate from image telemetry (uncorrected)', - 'mag_obs': 'Estimated ACA star magnitude', - 'mag_obs_err': 'Estimated ACA star magnitude uncertainty', - 'aoacmag_mean': 'Mean of AOACMAG from telemetry', - 'aoacmag_err': 'Standard deviation of AOACMAG from telemetry', - 'aoacmag_q25': '1st quartile of AOACMAG from telemetry', - 'aoacmag_median': 'Median of AOACMAG from telemetry', - 'aoacmag_q75': '3rd quartile of AOACMAG from telemetry', - 'counts_img': 'Raw counts from image telemetry, summed over the mouse-bit window', - 'counts_dark': 'Expected counts from background, summed over the mouse-bit window', - 'f_kalman': - 'Fraction of all samples where AOACASEQ == "KALM" and AOPCADMD == "NPNT" (n_kalman/n)', - 'f_track': 'Fraction of kalman samples with AOACFCT == "TRAK" (n_track/n_kalman)', - 'f_mag_est_ok': ( - 'Fraction of all samples included in magnitude estimate regardless of centroid residual' - ' (n_mag_est_ok/n_kalman)' + "agasc_id": "AGASC ID of the star", + "obsid": "OBSID at the time the star catalog is commanded (from kadi.commands)", + "slot": "Slot number", + "type": "GUI/ACQ/BOT", + "mp_starcat_time": ( + "The time at which the star catalog is commanded (from kadi.commands)" + ), + "tstart": ( + "Start time of the observation according to kadi.commands (in cxc seconds)" + ), + "tstop": ( + "Start time of the observation according to kadi.commands (in cxc seconds)" + ), + "mag_correction": "Overall correction applied to the magnitude estimate", + "responsivity": "Responsivity correction applied to the magnitude estimate", + "droop_shift": "Droop shift correction applied to the magnitude estimate", + "mag_aca": "ACA star magnitude from the AGASC catalog", + "mag_aca_err": "ACA star magnitude uncertainty from the AGASC catalog", + "row": ( + "Expected row number, based on star location and yanf/zang from" + " mica.archive.starcheck DB" + ), + "col": ( + "Expected col number, based on star location and yanf/zang from" + " mica.archive.starcheck DB" + ), + "mag_img": "Magnitude estimate from image telemetry (uncorrected)", + "mag_obs": "Estimated ACA star magnitude", + "mag_obs_err": "Estimated ACA star magnitude uncertainty", + "aoacmag_mean": "Mean of AOACMAG from telemetry", + "aoacmag_err": "Standard deviation of AOACMAG from telemetry", + "aoacmag_q25": "1st quartile of AOACMAG from telemetry", + "aoacmag_median": "Median of AOACMAG from telemetry", + "aoacmag_q75": "3rd quartile of AOACMAG from telemetry", + "counts_img": "Raw counts from image telemetry, summed over the mouse-bit window", + "counts_dark": "Expected counts from background, summed over the mouse-bit window", + "f_kalman": ( + 'Fraction of all samples where AOACASEQ == "KALM" and AOPCADMD == "NPNT"' + " (n_kalman/n)" + ), + "f_track": 'Fraction of kalman samples with AOACFCT == "TRAK" (n_track/n_kalman)', + "f_mag_est_ok": ( + "Fraction of all samples included in magnitude estimate regardless of centroid" + " residual (n_mag_est_ok/n_kalman)" ), - 'f_mag_est_ok_3': - 'Fraction of kalman samples with (mag_est_ok & dr3) == True (n_ok_3/n_kalman)', - 'f_mag_est_ok_5': - 'Fraction of kalman samples with (mag_est_ok & dbox5) == True (n_ok_5/n_kalman)', - 'f_ok': 'n_ok_5 / n_kalman. Same as f_ok_5.', # fix this - 'f_ok_3': """n_ok_3 / n_kalman. This is a measure of the fraction of time during an + "f_mag_est_ok_3": ( + "Fraction of kalman samples with (mag_est_ok & dr3) == True (n_ok_3/n_kalman)" + ), + "f_mag_est_ok_5": ( + "Fraction of kalman samples with (mag_est_ok & dbox5) == True (n_ok_5/n_kalman)" + ), + "f_ok": "n_ok_5 / n_kalman. Same as f_ok_5.", # fix this + "f_ok_3": """n_ok_3 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting high-quality star centroids.""", - 'f_ok_5': """ + "f_ok_5": """ n_ok_5 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting any star centroid at all.""", - 'f_dr3': - 'Fraction of mag-est-ok samples with centroid residual < 3 arcsec (n_dr3/n_mag_est_ok)', - 'f_dbox5': ( - 'Fraction of mag-est-ok samples with centroid residual within a 5 arcsec box ' - '(n_dbox5/n_mag_est_ok)' + "f_dr3": ( + "Fraction of mag-est-ok samples with centroid residual < 3 arcsec" + " (n_dr3/n_mag_est_ok)" + ), + "f_dbox5": ( + "Fraction of mag-est-ok samples with centroid residual within a 5 arcsec box " + "(n_dbox5/n_mag_est_ok)" + ), + "q25": "1st quartile of estimated magnitude", + "median": "Median of estimated magnitude", + "q75": "1st quartile of estimated magnitude", + "mean": "Mean of estimated magnitude", + "mean_err": "Uncertainty in the mean of estimated magnitude", + "std": "Standard deviation of estimated magnitude", + "skew": "Skewness of estimated magnitude", + "kurt": "Kurtosis of estimated magnitude", + "t_mean": "Mean of estimated magnitude after removing outliers", + "t_mean_err": ( + "Uncertainty in the mean of estimated magnitude after removing outliers" ), - 'q25': '1st quartile of estimated magnitude', - 'median': 'Median of estimated magnitude', - 'q75': '1st quartile of estimated magnitude', - 'mean': 'Mean of estimated magnitude', - 'mean_err': 'Uncertainty in the mean of estimated magnitude', - 'std': 'Standard deviation of estimated magnitude', - 'skew': 'Skewness of estimated magnitude', - 'kurt': 'Kurtosis of estimated magnitude', - 't_mean': 'Mean of estimated magnitude after removing outliers', - 't_mean_err': 'Uncertainty in the mean of estimated magnitude after removing outliers', - 't_std': 'Standard deviation of estimated magnitude after removing outliers', - 't_skew': 'Skewness of estimated magnitude after removing outliers', - 't_kurt': 'Kurtosis of estimated magnitude after removing outliers', - 'n': 'Number of samples', - 'n_ok': 'Number of samples with (kalman & mag_est_ok & dbox5) == True', - 'outliers': 'Number of outliers (+- 3 IQR)', - 'lf_variability_100s': 'Rolling mean of OK magnitudes with a 100 second window', - 'lf_variability_500s': 'Rolling mean of OK magnitudes with a 500 second window', - 'lf_variability_1000s': 'Rolling mean of OK magnitudes with a 1000 second window', - 'tempccd': 'CCD temperature', - 'dr_star': 'Angle residual', - 'obs_ok': 'Boolean flag: everything OK with this observation', - 'obs_suspect': 'Boolean flag: this observation is "suspect"', - 'obs_fail': 'Boolean flag: a processing error when estimating magnitude for this observation', - 'comments': '', - 'w': 'Weight to be used on a weighted mean (1/std)', - 'mean_corrected': 'Corrected mean used in weighted mean (t_mean + mag_correction)', - 'weighted_mean': 'Mean weighted by inverse of standard deviation (mean/std)', + "t_std": "Standard deviation of estimated magnitude after removing outliers", + "t_skew": "Skewness of estimated magnitude after removing outliers", + "t_kurt": "Kurtosis of estimated magnitude after removing outliers", + "n": "Number of samples", + "n_ok": "Number of samples with (kalman & mag_est_ok & dbox5) == True", + "outliers": "Number of outliers (+- 3 IQR)", + "lf_variability_100s": "Rolling mean of OK magnitudes with a 100 second window", + "lf_variability_500s": "Rolling mean of OK magnitudes with a 500 second window", + "lf_variability_1000s": "Rolling mean of OK magnitudes with a 1000 second window", + "tempccd": "CCD temperature", + "dr_star": "Angle residual", + "obs_ok": "Boolean flag: everything OK with this observation", + "obs_suspect": 'Boolean flag: this observation is "suspect"', + "obs_fail": ( + "Boolean flag: a processing error when estimating magnitude for this" + " observation" + ), + "comments": "", + "w": "Weight to be used on a weighted mean (1/std)", + "mean_corrected": "Corrected mean used in weighted mean (t_mean + mag_correction)", + "weighted_mean": "Mean weighted by inverse of standard deviation (mean/std)", } @@ -671,89 +753,99 @@ def get_obs_stats(obs, telem=None): :return: dict dictionary with stats """ - logger.debug(f' Getting OBS stats for AGASC ID {obs["agasc_id"]},' - f' OBSID {obs["agasc_id"]} at {obs["mp_starcat_time"]}') + logger.debug( + f' Getting OBS stats for AGASC ID {obs["agasc_id"]},' + f' OBSID {obs["agasc_id"]} at {obs["mp_starcat_time"]}' + ) star_obs_catalogs.load() - star = get_star(obs['agasc_id'], use_supplement=False) - start = CxoTime(obs['obs_start']).cxcsec - stop = CxoTime(obs['obs_stop']).cxcsec + star = get_star(obs["agasc_id"], use_supplement=False) + start = CxoTime(obs["obs_start"]).cxcsec + stop = CxoTime(obs["obs_stop"]).cxcsec - stats = {k: obs[k] for k in - ['agasc_id', 'obsid', 'slot', 'type', 'mp_starcat_time']} - stats['mp_starcat_time'] = stats['mp_starcat_time'] - droop_shift = get_droop_systematic_shift(star['MAG_ACA']) + stats = { + k: obs[k] for k in ["agasc_id", "obsid", "slot", "type", "mp_starcat_time"] + } + stats["mp_starcat_time"] = stats["mp_starcat_time"] + droop_shift = get_droop_systematic_shift(star["MAG_ACA"]) responsivity = get_responsivity(start) - stats.update({'tstart': start, - 'tstop': stop, - 'mag_correction': - responsivity - droop_shift, - 'responsivity': responsivity, - 'droop_shift': droop_shift, - 'mag_aca': star['MAG_ACA'], - 'mag_aca_err': star['MAG_ACA_ERR'] / 100, - 'row': obs['row'], - 'col': obs['col'], - }) + stats.update( + { + "tstart": start, + "tstop": stop, + "mag_correction": -responsivity - droop_shift, + "responsivity": responsivity, + "droop_shift": droop_shift, + "mag_aca": star["MAG_ACA"], + "mag_aca_err": star["MAG_ACA_ERR"] / 100, + "row": obs["row"], + "col": obs["col"], + } + ) # other default values - stats.update({ - 'mag_img': np.inf, - 'mag_obs': np.inf, - 'mag_obs_err': np.inf, - 'aoacmag_mean': np.inf, - 'aoacmag_err': np.inf, - 'aoacmag_q25': np.inf, - 'aoacmag_median': np.inf, - 'aoacmag_q75': np.inf, - 'counts_img': np.inf, - 'counts_dark': np.inf, - 'f_kalman': 0., - 'f_track': 0., - 'f_dbox5': 0., - 'f_dr3': 0., - 'f_mag_est_ok': 0., - 'f_mag_est_ok_3': 0., - 'f_mag_est_ok_5': 0., - 'f_ok': 0., - 'f_ok_3': 0., - 'f_ok_5': 0., - 'q25': np.inf, - 'median': np.inf, - 'q75': np.inf, - 'mean': np.inf, - 'mean_err': np.inf, - 'std': np.inf, - 'skew': np.inf, - 'kurt': np.inf, - 't_mean': np.inf, - 't_mean_err': np.inf, - 't_std': np.inf, - 't_skew': np.inf, - 't_kurt': np.inf, - 'n': 0, - 'n_ok': 0, - 'n_ok_3': 0, - 'n_ok_5': 0, - 'n_mag_est_ok': 0, - 'n_mag_est_ok_3': 0, - 'n_mag_est_ok_5': 0, - 'outliers': -1, - 'lf_variability_100s': np.inf, - 'lf_variability_500s': np.inf, - 'lf_variability_1000s': np.inf, - 'tempccd': np.nan, - 'dr_star': np.inf, - }) + stats.update( + { + "mag_img": np.inf, + "mag_obs": np.inf, + "mag_obs_err": np.inf, + "aoacmag_mean": np.inf, + "aoacmag_err": np.inf, + "aoacmag_q25": np.inf, + "aoacmag_median": np.inf, + "aoacmag_q75": np.inf, + "counts_img": np.inf, + "counts_dark": np.inf, + "f_kalman": 0.0, + "f_track": 0.0, + "f_dbox5": 0.0, + "f_dr3": 0.0, + "f_mag_est_ok": 0.0, + "f_mag_est_ok_3": 0.0, + "f_mag_est_ok_5": 0.0, + "f_ok": 0.0, + "f_ok_3": 0.0, + "f_ok_5": 0.0, + "q25": np.inf, + "median": np.inf, + "q75": np.inf, + "mean": np.inf, + "mean_err": np.inf, + "std": np.inf, + "skew": np.inf, + "kurt": np.inf, + "t_mean": np.inf, + "t_mean_err": np.inf, + "t_std": np.inf, + "t_skew": np.inf, + "t_kurt": np.inf, + "n": 0, + "n_ok": 0, + "n_ok_3": 0, + "n_ok_5": 0, + "n_mag_est_ok": 0, + "n_mag_est_ok_3": 0, + "n_mag_est_ok_5": 0, + "outliers": -1, + "lf_variability_100s": np.inf, + "lf_variability_500s": np.inf, + "lf_variability_1000s": np.inf, + "tempccd": np.nan, + "dr_star": np.inf, + } + ) if telem is None: telem = get_telemetry(obs) if len(telem) > 0: stats.update(calc_obs_stats(telem)) - logger.debug(f' slot={stats["slot"]}, f_ok={stats["f_ok"]:.3f}, ' - f'f_mag_est_ok={stats["f_mag_est_ok"]:.3f}, f_dr3={stats["f_dr3"]:.3f}, ' - f'mag={stats["mag_obs"]:.2f}') + logger.debug( + f' slot={stats["slot"]}, f_ok={stats["f_ok"]:.3f}, ' + f'f_mag_est_ok={stats["f_mag_est_ok"]:.3f}, f_dr3={stats["f_dr3"]:.3f}, ' + f'mag={stats["mag_obs"]:.2f}' + ) return stats @@ -775,18 +867,18 @@ def calc_obs_stats(telem): # From here on, we only consider the telemetry in NPNT in Kalman mode. All # subsequent counts and fractions are calculated from this subset. ########################################################################### - telem = telem[(telem['AOACASEQ'] == 'KALM') & (telem['AOPCADMD'] == 'NPNT')] - times = telem['times'] + telem = telem[(telem["AOACASEQ"] == "KALM") & (telem["AOPCADMD"] == "NPNT")] + times = telem["times"] - dr3 = (telem['dr'] < 3) - dbox5 = (np.abs(telem['dy']) < 5) & (np.abs(telem['dz']) < 5) + dr3 = telem["dr"] < 3 + dbox5 = (np.abs(telem["dy"]) < 5) & (np.abs(telem["dz"]) < 5) # Samples used in the magnitude estimation processing # note that SP flag is not included - mag_est_ok = (telem['AOACIIR'] == 'OK') & (telem['AOACFCT'] == 'TRAK') - aca_trak = (telem['AOACFCT'] == 'TRAK') - sat_pix = (telem['AOACISP'] == 'OK') - ion_rad = (telem['AOACIIR'] == 'OK') + mag_est_ok = (telem["AOACIIR"] == "OK") & (telem["AOACFCT"] == "TRAK") + aca_trak = telem["AOACFCT"] == "TRAK" + sat_pix = telem["AOACISP"] == "OK" + ion_rad = telem["AOACIIR"] == "OK" n_kalman = len(telem) f_kalman = n_kalman / n_total @@ -803,38 +895,40 @@ def calc_obs_stats(telem): # Select readouts OK for OBC Kalman filter and compute star mean offset ok = mag_est_ok & dbox5 if np.any(ok): - yang_mean = np.mean(telem['yang_img'][ok] - telem['yang_star'][ok]) - zang_mean = np.mean(telem['zang_img'][ok] - telem['zang_star'][ok]) + yang_mean = np.mean(telem["yang_img"][ok] - telem["yang_star"][ok]) + zang_mean = np.mean(telem["zang_img"][ok] - telem["zang_star"][ok]) dr_star = np.sqrt(yang_mean**2 + zang_mean**2) else: dr_star = np.inf stats = { - 'f_kalman': f_kalman, - 'f_track': (np.count_nonzero(aca_trak) / n_kalman) if n_kalman else 0, - 'f_dbox5': f_5, - 'f_dr3': f_3, - 'f_mag_est_ok': (n_mag_est_ok / n_kalman) if n_kalman else 0, - 'f_mag_est_ok_3': (n_mag_est_ok_3 / n_kalman) if n_kalman else 0, - 'f_mag_est_ok_5': (n_mag_est_ok_5 / n_kalman) if n_kalman else 0, - 'f_ok': (n_ok_5 / n_kalman) if n_kalman else 0, - 'f_ok_3': (n_ok_3 / n_kalman) if n_kalman else 0, - 'f_ok_5': (n_ok_5 / n_kalman) if n_kalman else 0, - 'n': n_total, - 'n_ok': n_ok_5, - 'n_ok_5': n_ok_5, - 'n_ok_3': n_ok_3, - 'n_mag_est_ok': n_mag_est_ok, - 'n_mag_est_ok_3': n_mag_est_ok_3, - 'n_mag_est_ok_5': n_mag_est_ok_5, - 'dr_star': dr_star, + "f_kalman": f_kalman, + "f_track": (np.count_nonzero(aca_trak) / n_kalman) if n_kalman else 0, + "f_dbox5": f_5, + "f_dr3": f_3, + "f_mag_est_ok": (n_mag_est_ok / n_kalman) if n_kalman else 0, + "f_mag_est_ok_3": (n_mag_est_ok_3 / n_kalman) if n_kalman else 0, + "f_mag_est_ok_5": (n_mag_est_ok_5 / n_kalman) if n_kalman else 0, + "f_ok": (n_ok_5 / n_kalman) if n_kalman else 0, + "f_ok_3": (n_ok_3 / n_kalman) if n_kalman else 0, + "f_ok_5": (n_ok_5 / n_kalman) if n_kalman else 0, + "n": n_total, + "n_ok": n_ok_5, + "n_ok_5": n_ok_5, + "n_ok_3": n_ok_3, + "n_mag_est_ok": n_mag_est_ok, + "n_mag_est_ok_3": n_mag_est_ok_3, + "n_mag_est_ok_5": n_mag_est_ok_5, + "dr_star": dr_star, } - if stats['n_mag_est_ok_3'] < 10: + if stats["n_mag_est_ok_3"] < 10: return stats - aoacmag_q25, aoacmag_q50, aoacmag_q75 = np.quantile(telem['AOACMAG'][ok], [0.25, 0.5, 0.75]) + aoacmag_q25, aoacmag_q50, aoacmag_q75 = np.quantile( + telem["AOACMAG"][ok], [0.25, 0.5, 0.75] + ) - mags = telem['mags'] + mags = telem["mags"] q25, q50, q75 = np.quantile(mags[ok], [0.25, 0.5, 0.75]) iqr = q75 - q25 outlier = ok & ((mags > q75 + 3 * iqr) | (mags < q25 - 3 * iqr)) @@ -847,152 +941,188 @@ def calc_obs_stats(telem): s_500s = s_500s[~np.isnan(s_500s)] s_1000s = s_1000s[~np.isnan(s_1000s)] - stats.update({ - 'aoacmag_mean': np.mean(telem['AOACMAG'][ok]), - 'aoacmag_err': np.std(telem['AOACMAG'][ok]), - 'aoacmag_q25': aoacmag_q25, - 'aoacmag_median': aoacmag_q50, - 'aoacmag_q75': aoacmag_q75, - 'q25': q25, - 'median': q50, - 'q75': q75, - 'counts_img': np.mean(telem['counts_img'][ok]), - 'counts_dark': np.mean(telem['counts_dark'][ok]), - 'mean': np.mean(mags[ok]), - 'mean_err': scipy.stats.sem(mags[ok]), - 'std': np.std(mags[ok]), - 'skew': scipy.stats.skew(mags), - 'kurt': scipy.stats.kurtosis(mags), - 't_mean': np.mean(mags[ok & (~outlier)]), - 't_mean_err': scipy.stats.sem(mags[ok & (~outlier)]), - 't_std': np.std(mags[ok & (~outlier)]), - 't_skew': scipy.stats.skew(mags[ok & (~outlier)]), - 't_kurt': scipy.stats.kurtosis(mags[ok & (~outlier)]), - 'outliers': np.count_nonzero(outlier), - 'lf_variability_100s': np.max(s_100s) - np.min(s_100s), - 'lf_variability_500s': np.max(s_500s) - np.min(s_500s), - 'lf_variability_1000s': np.max(s_1000s) - np.min(s_1000s), - 'tempccd': np.mean(telem['TEMPCCD'][ok]) - 273.16, - }) - - stats.update({ - 'mag_img': np.mean(telem['mags_img'][ok & (~outlier)]), - 'mag_obs': stats['t_mean'], - 'mag_obs_err': stats['t_mean_err'] - }) + stats.update( + { + "aoacmag_mean": np.mean(telem["AOACMAG"][ok]), + "aoacmag_err": np.std(telem["AOACMAG"][ok]), + "aoacmag_q25": aoacmag_q25, + "aoacmag_median": aoacmag_q50, + "aoacmag_q75": aoacmag_q75, + "q25": q25, + "median": q50, + "q75": q75, + "counts_img": np.mean(telem["counts_img"][ok]), + "counts_dark": np.mean(telem["counts_dark"][ok]), + "mean": np.mean(mags[ok]), + "mean_err": scipy.stats.sem(mags[ok]), + "std": np.std(mags[ok]), + "skew": scipy.stats.skew(mags), + "kurt": scipy.stats.kurtosis(mags), + "t_mean": np.mean(mags[ok & (~outlier)]), + "t_mean_err": scipy.stats.sem(mags[ok & (~outlier)]), + "t_std": np.std(mags[ok & (~outlier)]), + "t_skew": scipy.stats.skew(mags[ok & (~outlier)]), + "t_kurt": scipy.stats.kurtosis(mags[ok & (~outlier)]), + "outliers": np.count_nonzero(outlier), + "lf_variability_100s": np.max(s_100s) - np.min(s_100s), + "lf_variability_500s": np.max(s_500s) - np.min(s_500s), + "lf_variability_1000s": np.max(s_1000s) - np.min(s_1000s), + "tempccd": np.mean(telem["TEMPCCD"][ok]) - 273.16, + } + ) + + stats.update( + { + "mag_img": np.mean(telem["mags_img"][ok & (~outlier)]), + "mag_obs": stats["t_mean"], + "mag_obs_err": stats["t_mean_err"], + } + ) return stats AGASC_ID_STATS_INFO = { - 'last_obs_time': 'CXC seconds corresponding to the last mp_starcat_time for the star', - 'agasc_id': 'AGASC ID of the star', - 'mag_aca': 'ACA star magnitude from the AGASC catalog', - 'mag_aca_err': 'ACA star magnitude uncertainty from the AGASC catalog', - 'mag_obs': 'Estimated ACA star magnitude', - 'mag_obs_err': 'Estimated ACA star magnitude uncertainty', - 'mag_obs_std': 'Estimated ACA star magnitude standard deviation', - 'color': 'Star color from the AGASC catalog', - 'n_obsids': 'Number of observations for the star', - 'n_obsids_fail': 'Number of observations which give an unexpected error', - 'n_obsids_suspect': - 'Number of observations deemed "suspect" and ignored in the magnitude estimate', - 'n_obsids_ok': 'Number of observations considered in the magnitude estimate', - 'n_no_mag': 'Number of observations where the star magnitude was not estimated', - 'n': 'Total number of image samples for the star', - 'n_mag_est_ok': 'Total number of image samples included in magnitude estimate for the star', - 'f_mag_est_ok': ( - 'Fraction of all samples included in magnitude estimate regardless of centroid residual' - ' (n_mag_est_ok / n_kalman)' + "last_obs_time": ( + "CXC seconds corresponding to the last mp_starcat_time for the star" + ), + "agasc_id": "AGASC ID of the star", + "mag_aca": "ACA star magnitude from the AGASC catalog", + "mag_aca_err": "ACA star magnitude uncertainty from the AGASC catalog", + "mag_obs": "Estimated ACA star magnitude", + "mag_obs_err": "Estimated ACA star magnitude uncertainty", + "mag_obs_std": "Estimated ACA star magnitude standard deviation", + "color": "Star color from the AGASC catalog", + "n_obsids": "Number of observations for the star", + "n_obsids_fail": "Number of observations which give an unexpected error", + "n_obsids_suspect": ( + 'Number of observations deemed "suspect" and ignored in the magnitude estimate' ), - 'f_mag_est_ok_3': ( - 'Fraction of kalman samples that are included in magnitude estimate and within 3 arcsec' - ' (n_mag_est_ok_3 / n_kalman)'), - 'f_mag_est_ok_5': ( - 'Fraction of kalman samples that are included in magnitude estimate and within a 5 arcsec' - ' box (n_mag_est_ok_5 / n_kalman)'), - 'f_ok': 'n_ok_5 / n_kalman. Same as f_ok_5.', - 'f_ok_3': """n_ok_3 / n_kalman. This is a measure of the fraction of time during an + "n_obsids_ok": "Number of observations considered in the magnitude estimate", + "n_no_mag": "Number of observations where the star magnitude was not estimated", + "n": "Total number of image samples for the star", + "n_mag_est_ok": ( + "Total number of image samples included in magnitude estimate for the star" + ), + "f_mag_est_ok": ( + "Fraction of all samples included in magnitude estimate regardless of centroid" + " residual (n_mag_est_ok / n_kalman)" + ), + "f_mag_est_ok_3": ( + "Fraction of kalman samples that are included in magnitude estimate and within" + " 3 arcsec (n_mag_est_ok_3 / n_kalman)" + ), + "f_mag_est_ok_5": ( + "Fraction of kalman samples that are included in magnitude estimate and within" + " a 5 arcsec box (n_mag_est_ok_5 / n_kalman)" + ), + "f_ok": "n_ok_5 / n_kalman. Same as f_ok_5.", + "f_ok_3": """n_ok_3 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting high-quality star centroids.""", - 'f_ok_5': """ + "f_ok_5": """ n_ok_5 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting any star centroid at all.""", - 'median': 'Median magnitude over mag-est-ok samples', - 'sigma_minus': '15.8% quantile of magnitude over mag-est-ok samples', - 'sigma_plus': '84.2% quantile of magnitude over mag-est-ok samples', - 'mean': 'Mean of magnitude over mag-est-ok samples', - 'std': 'Standard deviation of magnitude over mag-est-ok samples', - 'mag_weighted_mean': - 'Average of magnitudes over observations, weighed by the inverse of its standard deviation', - 'mag_weighted_std': - 'Uncertainty in the weighted magnitude mean', - 't_mean': 'Mean magnitude after removing outliers on a per-observation basis', - 't_std': 'Magnitude standard deviation after removing outliers on a per-observation basis', - 'n_outlier': 'Number of outliers, removed on a per-observation basis', - 't_mean_1': 'Mean magnitude after removing 1.5*IQR outliers', - 't_std_1': 'Magnitude standard deviation after removing 1.5*IQR outliers', - 'n_outlier_1': 'Number of 1.5*IQR outliers', - 't_mean_2': 'Mean magnitude after removing 3*IQR outliers', - 't_std_2': 'Magnitude standard deviation after removing 3*IQR outliers', - 'n_outlier_2': 'Number of 3*IQR outliers', - 'selected_atol': 'abs(mag_obs - mag_aca) > 0.3', - 'selected_rtol': 'abs(mag_obs - mag_aca) > 3 * mag_aca_err', - 'selected_mag_aca_err': 'mag_aca_err > 0.2', - 'selected_color': '(color == 1.5) | (color == 0.7)', - 't_mean_dr3': - 'Truncated mean magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 't_std_dr3': - 'Truncated magnitude standard deviation after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 'mean_dr3': - 'Mean magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 'std_dr3': - 'Magnitude standard deviation after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 'f_dr3': - 'Fraction of mag-est-ok samples with centroid residual < 3 arcsec (n_dr3/n_mag_est_ok)', - 'n_dr3': 'Number of mag-est-ok samples with centroid residual < 3 arcsec', - 'n_dr3_outliers': - 'Number of magnitude outliers after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 'median_dr3': - 'Median magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 'sigma_minus_dr3': - '15.8% quantile of magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - 'sigma_plus_dr3': - '84.2% quantile of magnitude after removing outliers and samples with ' - 'centroid residual > 3 arcsec on a per-observation basis', - - 't_mean_dbox5': - 'Truncated mean magnitude after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis', - 't_std_dbox5': - 'Truncated magnitude standard deviation after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis', - 'mean_dbox5': - 'Mean magnitude after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis', - 'std_dbox5': - 'Magnitude standard deviation after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis', - 'f_dbox5': 'Fraction of mag-est-ok samples with centroid residual within a 5 arcsec box', - 'n_dbox5': 'Number of mag-est-ok samples with centroid residual within a 5 arcsec box', - 'n_dbox5_outliers': - 'Number of magnitude outliers after removing outliers and samples with ' - 'centroid residual out of a 5 arcsec box, on a per-observation basis', - 'median_dbox5': - 'Median magnitude after removing outliers and samples with ' - 'angular residual out of a 5 arcsec box, on a per-observation basis', - 'sigma_minus_dbox5': - '15.8% quantile of magnitude after removing outliers and samples with ' - 'angular residual out of a 5 arcsec box, on a per-observation basis', - 'sigma_plus_dbox5': - '84.2% quantile of magnitude after removing outliers and samples with ' - 'angular residual out of a 5 arcsec box, on a per-observation basis', + "median": "Median magnitude over mag-est-ok samples", + "sigma_minus": "15.8% quantile of magnitude over mag-est-ok samples", + "sigma_plus": "84.2% quantile of magnitude over mag-est-ok samples", + "mean": "Mean of magnitude over mag-est-ok samples", + "std": "Standard deviation of magnitude over mag-est-ok samples", + "mag_weighted_mean": ( + "Average of magnitudes over observations, weighed by the inverse of its" + " standard deviation" + ), + "mag_weighted_std": "Uncertainty in the weighted magnitude mean", + "t_mean": "Mean magnitude after removing outliers on a per-observation basis", + "t_std": ( + "Magnitude standard deviation after removing outliers on a per-observation" + " basis" + ), + "n_outlier": "Number of outliers, removed on a per-observation basis", + "t_mean_1": "Mean magnitude after removing 1.5*IQR outliers", + "t_std_1": "Magnitude standard deviation after removing 1.5*IQR outliers", + "n_outlier_1": "Number of 1.5*IQR outliers", + "t_mean_2": "Mean magnitude after removing 3*IQR outliers", + "t_std_2": "Magnitude standard deviation after removing 3*IQR outliers", + "n_outlier_2": "Number of 3*IQR outliers", + "selected_atol": "abs(mag_obs - mag_aca) > 0.3", + "selected_rtol": "abs(mag_obs - mag_aca) > 3 * mag_aca_err", + "selected_mag_aca_err": "mag_aca_err > 0.2", + "selected_color": "(color == 1.5) | (color == 0.7)", + "t_mean_dr3": ( + "Truncated mean magnitude after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" + ), + "t_std_dr3": ( + "Truncated magnitude standard deviation after removing outliers and samples" + " with centroid residual > 3 arcsec on a per-observation basis" + ), + "mean_dr3": ( + "Mean magnitude after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" + ), + "std_dr3": ( + "Magnitude standard deviation after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" + ), + "f_dr3": ( + "Fraction of mag-est-ok samples with centroid residual < 3 arcsec" + " (n_dr3/n_mag_est_ok)" + ), + "n_dr3": "Number of mag-est-ok samples with centroid residual < 3 arcsec", + "n_dr3_outliers": ( + "Number of magnitude outliers after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" + ), + "median_dr3": ( + "Median magnitude after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" + ), + "sigma_minus_dr3": ( + "15.8% quantile of magnitude after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" + ), + "sigma_plus_dr3": ( + "84.2% quantile of magnitude after removing outliers and samples with " + "centroid residual > 3 arcsec on a per-observation basis" + ), + "t_mean_dbox5": ( + "Truncated mean magnitude after removing outliers and samples with " + "centroid residual out of a 5 arcsec box, on a per-observation basis" + ), + "t_std_dbox5": ( + "Truncated magnitude standard deviation after removing outliers and samples" + " with centroid residual out of a 5 arcsec box, on a per-observation basis" + ), + "mean_dbox5": ( + "Mean magnitude after removing outliers and samples with " + "centroid residual out of a 5 arcsec box, on a per-observation basis" + ), + "std_dbox5": ( + "Magnitude standard deviation after removing outliers and samples with " + "centroid residual out of a 5 arcsec box, on a per-observation basis" + ), + "f_dbox5": ( + "Fraction of mag-est-ok samples with centroid residual within a 5 arcsec box" + ), + "n_dbox5": ( + "Number of mag-est-ok samples with centroid residual within a 5 arcsec box" + ), + "n_dbox5_outliers": ( + "Number of magnitude outliers after removing outliers and samples with " + "centroid residual out of a 5 arcsec box, on a per-observation basis" + ), + "median_dbox5": ( + "Median magnitude after removing outliers and samples with " + "angular residual out of a 5 arcsec box, on a per-observation basis" + ), + "sigma_minus_dbox5": ( + "15.8% quantile of magnitude after removing outliers and samples with " + "angular residual out of a 5 arcsec box, on a per-observation basis" + ), + "sigma_plus_dbox5": ( + "84.2% quantile of magnitude after removing outliers and samples with " + "angular residual out of a 5 arcsec box, on a per-observation basis" + ), } @@ -1010,282 +1140,333 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): :return: dict dictionary with stats """ - logger.debug(f'Getting stats for AGASC ID {agasc_id}...') + logger.debug(f"Getting stats for AGASC ID {agasc_id}...") min_mag_obs_err = 0.03 if not obs_status_override: obs_status_override = {} star_obs_catalogs.load(tstop=tstop) # Get a table of every time the star has been observed - star_obs = star_obs_catalogs.STARS_OBS[star_obs_catalogs.STARS_OBS['agasc_id'] == agasc_id] + star_obs = star_obs_catalogs.STARS_OBS[ + star_obs_catalogs.STARS_OBS["agasc_id"] == agasc_id + ] if len(star_obs) > 1: - star_obs = star_obs.loc['mp_starcat_time', sorted(star_obs['mp_starcat_time'])] + star_obs = star_obs.loc["mp_starcat_time", sorted(star_obs["mp_starcat_time"])] # this is the default result, if nothing gets calculated result = { - 'last_obs_time': 0, - 'agasc_id': agasc_id, - 'mag_aca': np.nan, - 'mag_aca_err': np.nan, - 'mag_obs': 0., - 'mag_obs_err': np.nan, - 'mag_obs_std': 0., - 'color': np.nan, - 'n_obsids': 0, - 'n_obsids_fail': 0, - 'n_obsids_suspect': 0, - 'n_obsids_ok': 0, - 'n_no_mag': 0, - 'n': 0, - 'n_ok': 0, - 'n_ok_3': 0, - 'n_ok_5': 0, - 'n_mag_est_ok': 0, - 'n_mag_est_ok_3': 0, - 'n_mag_est_ok_5': 0, - 'f_ok': 0., - 'median': 0, - 'sigma_minus': 0, - 'sigma_plus': 0, - 'mean': 0, - 'std': 0, - 'mag_weighted_mean': 0, - 'mag_weighted_std': 0, - 't_mean': 0, - 't_std': 0, - 'n_outlier': 0, - 't_mean_1': 0, - 't_std_1': 0, - 'n_outlier_1': 0, - 't_mean_2': 0, - 't_std_2': 0, - 'n_outlier_2': 0, + "last_obs_time": 0, + "agasc_id": agasc_id, + "mag_aca": np.nan, + "mag_aca_err": np.nan, + "mag_obs": 0.0, + "mag_obs_err": np.nan, + "mag_obs_std": 0.0, + "color": np.nan, + "n_obsids": 0, + "n_obsids_fail": 0, + "n_obsids_suspect": 0, + "n_obsids_ok": 0, + "n_no_mag": 0, + "n": 0, + "n_ok": 0, + "n_ok_3": 0, + "n_ok_5": 0, + "n_mag_est_ok": 0, + "n_mag_est_ok_3": 0, + "n_mag_est_ok_5": 0, + "f_ok": 0.0, + "median": 0, + "sigma_minus": 0, + "sigma_plus": 0, + "mean": 0, + "std": 0, + "mag_weighted_mean": 0, + "mag_weighted_std": 0, + "t_mean": 0, + "t_std": 0, + "n_outlier": 0, + "t_mean_1": 0, + "t_std_1": 0, + "n_outlier_1": 0, + "t_mean_2": 0, + "t_std_2": 0, + "n_outlier_2": 0, # these are the criteria for including in supplement - 'selected_atol': False, - 'selected_rtol': False, - 'selected_mag_aca_err': False, - 'selected_color': False, - 'f_mag_est_ok': 0, - 'f_mag_est_ok_3': 0, - 'f_mag_est_ok_5': 0, - 'f_ok_3': 0, - 'f_ok_5': 0, + "selected_atol": False, + "selected_rtol": False, + "selected_mag_aca_err": False, + "selected_color": False, + "f_mag_est_ok": 0, + "f_mag_est_ok_3": 0, + "f_mag_est_ok_5": 0, + "f_ok_3": 0, + "f_ok_5": 0, } - for tag in ['dr3', 'dbox5']: - result.update({ - f't_mean_{tag}': 0, - f't_std_{tag}': 0, - f't_mean_{tag}_not': 0, - f't_std_{tag}_not': 0, - f'mean_{tag}': 0, - f'std_{tag}': 0, - f'f_{tag}': 0, - f'n_{tag}': 0, - f'n_{tag}_outliers': 0, - f'median_{tag}': 0, - f'sigma_minus_{tag}': 0, - f'sigma_plus_{tag}': 0, - }) + for tag in ["dr3", "dbox5"]: + result.update( + { + f"t_mean_{tag}": 0, + f"t_std_{tag}": 0, + f"t_mean_{tag}_not": 0, + f"t_std_{tag}_not": 0, + f"mean_{tag}": 0, + f"std_{tag}": 0, + f"f_{tag}": 0, + f"n_{tag}": 0, + f"n_{tag}_outliers": 0, + f"median_{tag}": 0, + f"sigma_minus_{tag}": 0, + f"sigma_plus_{tag}": 0, + } + ) n_obsids = len(star_obs) # exclude star_obs that are in obs_status_override with status != 0 - excluded_obs = np.array([((oi, ai) in obs_status_override - and obs_status_override[(oi, ai)]['status'] != 0) - for oi, ai in star_obs[['mp_starcat_time', 'agasc_id']]]) + excluded_obs = np.array( + [ + ( + (oi, ai) in obs_status_override + and obs_status_override[(oi, ai)]["status"] != 0 + ) + for oi, ai in star_obs[["mp_starcat_time", "agasc_id"]] + ] + ) if np.any(excluded_obs): - logger.debug(' Excluding observations flagged in obs-status table: ' - f'{list(star_obs[excluded_obs]["obsid"])}') - - included_obs = np.array([((oi, ai) in obs_status_override - and obs_status_override[(oi, ai)]['status'] == 0) - for oi, ai in star_obs[['mp_starcat_time', 'agasc_id']]]) + logger.debug( + " Excluding observations flagged in obs-status table: " + f'{list(star_obs[excluded_obs]["obsid"])}' + ) + + included_obs = np.array( + [ + ( + (oi, ai) in obs_status_override + and obs_status_override[(oi, ai)]["status"] == 0 + ) + for oi, ai in star_obs[["mp_starcat_time", "agasc_id"]] + ] + ) if np.any(included_obs): - logger.debug(' Including observations marked OK in obs-status table: ' - f'{list(star_obs[included_obs]["obsid"])}') + logger.debug( + " Including observations marked OK in obs-status table: " + f'{list(star_obs[included_obs]["obsid"])}' + ) failures = [] all_telem = [] stats = [] last_obs_time = 0 for i, obs in enumerate(star_obs): - oi, ai = obs['mp_starcat_time', 'agasc_id'] - comment = '' + oi, ai = obs["mp_starcat_time", "agasc_id"] + comment = "" if (oi, ai) in obs_status_override: status = obs_status_override[(oi, ai)] - logger.debug(f' overriding status for (AGASC ID {ai}, starcat time {oi}): ' - f'{status["status"]}, {status["comments"]}') - comment = status['comments'] + logger.debug( + f" overriding status for (AGASC ID {ai}, starcat time {oi}): " + f'{status["status"]}, {status["comments"]}' + ) + comment = status["comments"] try: - last_obs_time = CxoTime(obs['mp_starcat_time']).cxcsec + last_obs_time = CxoTime(obs["mp_starcat_time"]).cxcsec telem = Table(get_telemetry(obs)) obs_stat = get_obs_stats(obs, telem={k: telem[k] for k in telem.colnames}) - obs_stat.update({ - 'obs_ok': ( - included_obs[i] | ( + obs_stat.update( + { + "obs_ok": included_obs[i] + | ( ~excluded_obs[i] - & (obs_stat['n'] > 10) - & (obs_stat['f_mag_est_ok'] > 0.3) - & (obs_stat['lf_variability_100s'] < 1) - ) - ), - 'obs_suspect': False, - 'obs_fail': False, - 'comments': comment - }) + & (obs_stat["n"] > 10) + & (obs_stat["f_mag_est_ok"] > 0.3) + & (obs_stat["lf_variability_100s"] < 1) + ), + "obs_suspect": False, + "obs_fail": False, + "comments": comment, + } + ) all_telem.append(telem) stats.append(obs_stat) - if not obs_stat['obs_ok'] and not excluded_obs[i]: - obs_stat['obs_suspect'] = True + if not obs_stat["obs_ok"] and not excluded_obs[i]: + obs_stat["obs_suspect"] = True failures.append( - dict(MagStatsException(msg='Suspect observation', - agasc_id=obs['agasc_id'], - obsid=obs['obsid'], - mp_starcat_time=obs["mp_starcat_time"],))) + dict( + MagStatsException( + msg="Suspect observation", + agasc_id=obs["agasc_id"], + obsid=obs["obsid"], + mp_starcat_time=obs["mp_starcat_time"], + ) + ) + ) except MagStatsException as e: # this except branch deals with exceptions thrown by get_telemetry all_telem.append(None) # length-zero telemetry short-circuits any new call to get_telemetry obs_stat = get_obs_stats(obs, telem=[]) - obs_stat.update({ - 'obs_ok': False, - 'obs_suspect': False, - 'obs_fail': True, - 'comments': comment if excluded_obs[i] else f'Error: {e.msg}.' - }) + obs_stat.update( + { + "obs_ok": False, + "obs_suspect": False, + "obs_fail": True, + "comments": comment if excluded_obs[i] else f"Error: {e.msg}.", + } + ) stats.append(obs_stat) if not excluded_obs[i]: logger.debug( - f' Error in get_agasc_id_stats({agasc_id=}, obsid={obs["obsid"]}): {e}') + f" Error in get_agasc_id_stats({agasc_id=}," + f' obsid={obs["obsid"]}): {e}' + ) failures.append(dict(e)) stats = Table(stats) - stats['w'] = np.nan - stats['mean_corrected'] = np.nan - stats['weighted_mean'] = np.nan + stats["w"] = np.nan + stats["mean_corrected"] = np.nan + stats["weighted_mean"] = np.nan star = get_star(agasc_id, use_supplement=False) - result.update({ - 'last_obs_time': last_obs_time, - 'mag_aca': star['MAG_ACA'], - 'mag_aca_err': star['MAG_ACA_ERR'] / 100, - 'color': star['COLOR1'], - 'n_obsids_fail': len(failures), - 'n_obsids_suspect': np.count_nonzero(stats['obs_suspect']), - 'n_obsids': n_obsids, - }) + result.update( + { + "last_obs_time": last_obs_time, + "mag_aca": star["MAG_ACA"], + "mag_aca_err": star["MAG_ACA_ERR"] / 100, + "color": star["COLOR1"], + "n_obsids_fail": len(failures), + "n_obsids_suspect": np.count_nonzero(stats["obs_suspect"]), + "n_obsids": n_obsids, + } + ) if not np.any(~excluded_obs): # this happens when all observations have been flagged as not OK a priory (obs-status). - logger.debug(f' Skipping star in get_agasc_id_stats({agasc_id=}).' - ' All observations are flagged as not good.') + logger.debug( + f" Skipping star in get_agasc_id_stats({agasc_id=})." + " All observations are flagged as not good." + ) return result, stats, failures if len(all_telem) - len(failures) <= 0: # and we reach here if some observations were not flagged as bad, but all failed. - logger.debug(f' Error in get_agasc_id_stats({agasc_id=}):' - ' There is no OK observation.') + logger.debug( + f" Error in get_agasc_id_stats({agasc_id=}): There is no OK observation." + ) return result, stats, failures excluded_obs += np.array([t is None for t in all_telem]) - logger.debug(' identifying outlying observations...') + logger.debug(" identifying outlying observations...") for i, (s, t) in enumerate(zip(stats, all_telem)): if excluded_obs[i]: continue - t['obs_ok'] = np.ones_like(t['mag_est_ok'], dtype=bool) * s['obs_ok'] - logger.debug(' identifying outlying observations ' - f'(OBSID={s["obsid"]}, mp_starcat_time={s["mp_starcat_time"]})') - t['obs_outlier'] = np.zeros_like(t['mag_est_ok']) - if np.any(t['mag_est_ok']) and s['f_mag_est_ok'] > 0 and s['obs_ok']: - iqr = s['q75'] - s['q25'] - t['obs_outlier'] = ( - t['mag_est_ok'] + t["obs_ok"] = np.ones_like(t["mag_est_ok"], dtype=bool) * s["obs_ok"] + logger.debug( + " identifying outlying observations " + f'(OBSID={s["obsid"]}, mp_starcat_time={s["mp_starcat_time"]})' + ) + t["obs_outlier"] = np.zeros_like(t["mag_est_ok"]) + if np.any(t["mag_est_ok"]) and s["f_mag_est_ok"] > 0 and s["obs_ok"]: + iqr = s["q75"] - s["q25"] + t["obs_outlier"] = ( + t["mag_est_ok"] & (iqr > 0) - & ((t['mags'] < s['q25'] - 1.5 * iqr) | (t['mags'] > s['q75'] + 1.5 * iqr)) + & ( + (t["mags"] < s["q25"] - 1.5 * iqr) + | (t["mags"] > s["q75"] + 1.5 * iqr) + ) ) - all_telem = vstack([Table(t) for i, t in enumerate(all_telem) if not excluded_obs[i]]) + all_telem = vstack( + [Table(t) for i, t in enumerate(all_telem) if not excluded_obs[i]] + ) n_total = len(all_telem) - kalman = (all_telem['AOACASEQ'] == 'KALM') & (all_telem['AOPCADMD'] == 'NPNT') + kalman = (all_telem["AOACASEQ"] == "KALM") & (all_telem["AOPCADMD"] == "NPNT") all_telem = all_telem[kalman] # non-npm/non-kalman are excluded n_kalman = len(all_telem) - mags = all_telem['mags'] - mag_est_ok = all_telem['mag_est_ok'] & all_telem['obs_ok'] - aca_trak = (all_telem['AOACFCT'] == 'TRAK') & all_telem['obs_ok'] - sat_pix = (all_telem['AOACISP'] == 'OK') & all_telem['obs_ok'] - ion_rad = (all_telem['AOACIIR'] == 'OK') & all_telem['obs_ok'] + mags = all_telem["mags"] + mag_est_ok = all_telem["mag_est_ok"] & all_telem["obs_ok"] + aca_trak = (all_telem["AOACFCT"] == "TRAK") & all_telem["obs_ok"] + sat_pix = (all_telem["AOACISP"] == "OK") & all_telem["obs_ok"] + ion_rad = (all_telem["AOACIIR"] == "OK") & all_telem["obs_ok"] f_mag_est_ok = np.count_nonzero(mag_est_ok) / len(mag_est_ok) - result.update({ - 'mag_obs_err': min_mag_obs_err, - 'n_obsids_ok': np.count_nonzero(stats['obs_ok']), - 'n_no_mag': ( - np.count_nonzero((~stats['obs_ok'])) - + np.count_nonzero(stats['f_mag_est_ok'][stats['obs_ok']] < 0.3) - ), - 'n': n_total, - 'n_mag_est_ok': np.count_nonzero(mag_est_ok), - 'f_mag_est_ok': f_mag_est_ok, - }) - - if result['n_mag_est_ok'] < 10: + result.update( + { + "mag_obs_err": min_mag_obs_err, + "n_obsids_ok": np.count_nonzero(stats["obs_ok"]), + "n_no_mag": np.count_nonzero((~stats["obs_ok"])) + + np.count_nonzero(stats["f_mag_est_ok"][stats["obs_ok"]] < 0.3), + "n": n_total, + "n_mag_est_ok": np.count_nonzero(mag_est_ok), + "f_mag_est_ok": f_mag_est_ok, + } + ) + + if result["n_mag_est_ok"] < 10: return result, stats, failures - sigma_minus, q25, median, q75, sigma_plus = np.quantile(mags[mag_est_ok], - [0.158, 0.25, 0.5, 0.75, 0.842]) + sigma_minus, q25, median, q75, sigma_plus = np.quantile( + mags[mag_est_ok], [0.158, 0.25, 0.5, 0.75, 0.842] + ) iqr = q75 - q25 outlier_1 = mag_est_ok & ((mags > q75 + 1.5 * iqr) | (mags < q25 - 1.5 * iqr)) outlier_2 = mag_est_ok & ((mags > q75 + 3 * iqr) | (mags < q25 - 3 * iqr)) - outlier = all_telem['obs_outlier'] + outlier = all_telem["obs_outlier"] # combine measurements using a weighted mean - obs_ok = stats['obs_ok'] - min_std = max(0.1, stats[obs_ok]['std'].min()) - stats['w'][obs_ok] = np.where(stats['std'][obs_ok] != 0, - 1. / stats['std'][obs_ok], - 1. / min_std) - stats['mean_corrected'][obs_ok] = stats['t_mean'][obs_ok] + stats['mag_correction'][obs_ok] - stats['weighted_mean'][obs_ok] = stats['mean_corrected'][obs_ok] * stats['w'][obs_ok] - - mag_weighted_mean = (stats[obs_ok]['weighted_mean'].sum() / stats[obs_ok]['w'].sum()) - mag_weighted_std = ( - np.sqrt(((stats[obs_ok]['mean'] - mag_weighted_mean)**2 * stats[obs_ok]['w']).sum() - / stats[obs_ok]['w'].sum()) + obs_ok = stats["obs_ok"] + min_std = max(0.1, stats[obs_ok]["std"].min()) + stats["w"][obs_ok] = np.where( + stats["std"][obs_ok] != 0, 1.0 / stats["std"][obs_ok], 1.0 / min_std + ) + stats["mean_corrected"][obs_ok] = ( + stats["t_mean"][obs_ok] + stats["mag_correction"][obs_ok] + ) + stats["weighted_mean"][obs_ok] = ( + stats["mean_corrected"][obs_ok] * stats["w"][obs_ok] ) - result.update({ - 'agasc_id': agasc_id, - 'n_mag_est_ok': np.count_nonzero(mag_est_ok), - 'f_mag_est_ok': f_mag_est_ok, - 'median': median, - 'sigma_minus': sigma_minus, - 'sigma_plus': sigma_plus, - 'mean': np.mean(mags[mag_est_ok]), - 'std': np.std(mags[mag_est_ok]), - 'mag_weighted_mean': mag_weighted_mean, - 'mag_weighted_std': mag_weighted_std, - 't_mean': np.mean(mags[mag_est_ok & (~outlier)]), - 't_std': np.std(mags[mag_est_ok & (~outlier)]), - 'n_outlier': np.count_nonzero(mag_est_ok & outlier), - 't_mean_1': np.mean(mags[mag_est_ok & (~outlier_1)]), - 't_std_1': np.std(mags[mag_est_ok & (~outlier_1)]), - 'n_outlier_1': np.count_nonzero(mag_est_ok & outlier_1), - 't_mean_2': np.mean(mags[mag_est_ok & (~outlier_2)]), - 't_std_2': np.std(mags[mag_est_ok & (~outlier_2)]), - 'n_outlier_2': np.count_nonzero(mag_est_ok & outlier_2), - }) + mag_weighted_mean = stats[obs_ok]["weighted_mean"].sum() / stats[obs_ok]["w"].sum() + mag_weighted_std = np.sqrt( + ((stats[obs_ok]["mean"] - mag_weighted_mean) ** 2 * stats[obs_ok]["w"]).sum() + / stats[obs_ok]["w"].sum() + ) + + result.update( + { + "agasc_id": agasc_id, + "n_mag_est_ok": np.count_nonzero(mag_est_ok), + "f_mag_est_ok": f_mag_est_ok, + "median": median, + "sigma_minus": sigma_minus, + "sigma_plus": sigma_plus, + "mean": np.mean(mags[mag_est_ok]), + "std": np.std(mags[mag_est_ok]), + "mag_weighted_mean": mag_weighted_mean, + "mag_weighted_std": mag_weighted_std, + "t_mean": np.mean(mags[mag_est_ok & (~outlier)]), + "t_std": np.std(mags[mag_est_ok & (~outlier)]), + "n_outlier": np.count_nonzero(mag_est_ok & outlier), + "t_mean_1": np.mean(mags[mag_est_ok & (~outlier_1)]), + "t_std_1": np.std(mags[mag_est_ok & (~outlier_1)]), + "n_outlier_1": np.count_nonzero(mag_est_ok & outlier_1), + "t_mean_2": np.mean(mags[mag_est_ok & (~outlier_2)]), + "t_std_2": np.std(mags[mag_est_ok & (~outlier_2)]), + "n_outlier_2": np.count_nonzero(mag_est_ok & outlier_2), + } + ) residual_ok = { - 3: all_telem['dr'] < 3, - 5: (np.abs(all_telem['dy']) < 5) & (np.abs(all_telem['dz']) < 5) + 3: all_telem["dr"] < 3, + 5: (np.abs(all_telem["dy"]) < 5) & (np.abs(all_telem["dz"]) < 5), } - dr_tag = {3: 'dr3', 5: 'dbox5'} + dr_tag = {3: "dr3", 5: "dbox5"} for dr in [3, 5]: tag = dr_tag[dr] k = mag_est_ok & residual_ok[dr] @@ -1293,46 +1474,64 @@ def get_agasc_id_stats(agasc_id, obs_status_override=None, tstop=None): n_ok = np.count_nonzero(aca_trak & sat_pix & ion_rad & residual_ok[dr]) if not np.any(k): continue - sigma_minus, q25, median, q75, sigma_plus = np.quantile(mags[k], - [0.158, 0.25, 0.5, 0.75, 0.842]) - outlier = mag_est_ok & all_telem['obs_outlier'] - mag_not = np.nanmean(mags[k2 & (~outlier)]) if np.count_nonzero(k2 & (~outlier)) else np.nan - std_not = np.nanstd(mags[k2 & (~outlier)]) if np.count_nonzero(k2 & (~outlier)) else np.nan - result.update({ - f't_mean_{tag}': np.mean(mags[k & (~outlier)]), - f't_std_{tag}': np.std(mags[k & (~outlier)]), - f't_mean_{tag}_not': mag_not, - f't_std_{tag}_not': std_not, - f'mean_{tag}': np.mean(mags[k]), - f'std_{tag}': np.std(mags[k]), - f'f_{tag}': np.count_nonzero(k) / np.count_nonzero(mag_est_ok), - f'n_{tag}': np.count_nonzero(k), - f'n_{tag}_outliers': np.count_nonzero(k & outlier), - f'f_mag_est_ok_{dr}': np.count_nonzero(k) / len(k), - f'n_mag_est_ok_{dr}': np.count_nonzero(k), - f'median_{tag}': median, - f'sigma_minus_{tag}': sigma_minus, - f'sigma_plus_{tag}': sigma_plus, - f'f_ok_{dr}': (n_ok / n_kalman) if n_kalman else 0, - f'n_ok_{dr}': n_ok, - }) - - result.update({ - 'mag_obs': result['t_mean_dbox5'], - 'mag_obs_err': np.sqrt(result['t_std_dbox5']**2 + min_mag_obs_err**2), - 'mag_obs_std': result['t_std_dbox5'], - 'n_ok': result['n_ok_5'], - 'f_ok': result['f_ok_5'], - }) + sigma_minus, q25, median, q75, sigma_plus = np.quantile( + mags[k], [0.158, 0.25, 0.5, 0.75, 0.842] + ) + outlier = mag_est_ok & all_telem["obs_outlier"] + mag_not = ( + np.nanmean(mags[k2 & (~outlier)]) + if np.count_nonzero(k2 & (~outlier)) + else np.nan + ) + std_not = ( + np.nanstd(mags[k2 & (~outlier)]) + if np.count_nonzero(k2 & (~outlier)) + else np.nan + ) + result.update( + { + f"t_mean_{tag}": np.mean(mags[k & (~outlier)]), + f"t_std_{tag}": np.std(mags[k & (~outlier)]), + f"t_mean_{tag}_not": mag_not, + f"t_std_{tag}_not": std_not, + f"mean_{tag}": np.mean(mags[k]), + f"std_{tag}": np.std(mags[k]), + f"f_{tag}": np.count_nonzero(k) / np.count_nonzero(mag_est_ok), + f"n_{tag}": np.count_nonzero(k), + f"n_{tag}_outliers": np.count_nonzero(k & outlier), + f"f_mag_est_ok_{dr}": np.count_nonzero(k) / len(k), + f"n_mag_est_ok_{dr}": np.count_nonzero(k), + f"median_{tag}": median, + f"sigma_minus_{tag}": sigma_minus, + f"sigma_plus_{tag}": sigma_plus, + f"f_ok_{dr}": (n_ok / n_kalman) if n_kalman else 0, + f"n_ok_{dr}": n_ok, + } + ) + + result.update( + { + "mag_obs": result["t_mean_dbox5"], + "mag_obs_err": np.sqrt(result["t_std_dbox5"] ** 2 + min_mag_obs_err**2), + "mag_obs_std": result["t_std_dbox5"], + "n_ok": result["n_ok_5"], + "f_ok": result["f_ok_5"], + } + ) # these are the criteria for including in supplement - result.update({ - 'selected_atol': np.abs(result['mag_obs'] - result['mag_aca']) > 0.3, - 'selected_rtol': np.abs(result['mag_obs'] - result['mag_aca']) > 3 * result['mag_aca_err'], - 'selected_mag_aca_err': result['mag_aca_err'] > 0.2, - 'selected_color': (result['color'] == 1.5) | (np.isclose(result['color'], 0.7)) - }) - - logger.debug(f' stats for AGASC ID {agasc_id}: ' - f' {stats["mag_obs"][0]}') + result.update( + { + "selected_atol": np.abs(result["mag_obs"] - result["mag_aca"]) > 0.3, + "selected_rtol": ( + np.abs(result["mag_obs"] - result["mag_aca"]) + > 3 * result["mag_aca_err"] + ), + "selected_mag_aca_err": result["mag_aca_err"] > 0.2, + "selected_color": (result["color"] == 1.5) + | (np.isclose(result["color"], 0.7)), + } + ) + + logger.debug(f' stats for AGASC ID {agasc_id}: {stats["mag_obs"][0]}') return result, stats, failures diff --git a/agasc/supplement/magnitudes/mag_estimate_report.py b/agasc/supplement/magnitudes/mag_estimate_report.py index 56ca1e47..a55f2f30 100644 --- a/agasc/supplement/magnitudes/mag_estimate_report.py +++ b/agasc/supplement/magnitudes/mag_estimate_report.py @@ -1,32 +1,32 @@ -import platform +import copy +import errno import getpass +import json import logging -import errno import os -import copy -import json +import platform import warnings -from subprocess import Popen, PIPE -from pathlib import Path from email.mime.text import MIMEText +from pathlib import Path +from subprocess import PIPE, Popen + import jinja2 -import numpy as np import matplotlib.pyplot as plt +import numpy as np +from astropy import table +from cxotime import CxoTime from matplotlib.ticker import FixedLocator from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable from tqdm import tqdm -from astropy import table -from cxotime import CxoTime from agasc.supplement.magnitudes import mag_estimate - JINJA2 = jinja2.Environment( - loader=jinja2.PackageLoader('agasc.supplement.magnitudes', 'templates'), - autoescape=jinja2.select_autoescape(['html', 'xml']) + loader=jinja2.PackageLoader("agasc.supplement.magnitudes", "templates"), + autoescape=jinja2.select_autoescape(["html", "xml"]), ) -logger = logging.getLogger('agasc.supplement') +logger = logging.getLogger("agasc.supplement") class TableEncoder(json.JSONEncoder): @@ -52,13 +52,14 @@ class TableEncoder(json.JSONEncoder): ] } """ + def default(self, obj): if isinstance(obj, table.Table): return {name: val.tolist() for name, val in obj.columns.items()} if isinstance(obj, CxoTime): return obj.isot if isinstance(obj, np.ma.core.MaskedArray): - return {'columns': obj.dtype.names, 'data': obj.tolist()} + return {"columns": obj.dtype.names, "data": obj.tolist()} if np.isscalar(obj): return int(obj) if np.isreal(obj): @@ -67,16 +68,19 @@ def default(self, obj): class MagEstimateReport: - def __init__(self, - agasc_stats='mag_stats_agasc.fits', - obs_stats='mag_stats_obsid.fits', - directory='./mag_estimates_reports'): - + def __init__( + self, + agasc_stats="mag_stats_agasc.fits", + obs_stats="mag_stats_obsid.fits", + directory="./mag_estimates_reports", + ): if type(agasc_stats) is table.Table: self.agasc_stats = agasc_stats else: if not Path(agasc_stats).exists: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), agasc_stats) + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), agasc_stats + ) self.agasc_stats = table.Table.read(agasc_stats) self.agasc_stats.convert_bytestring_to_unicode() @@ -84,113 +88,151 @@ def __init__(self, self.obs_stats = obs_stats else: if not Path(obs_stats).exists: - raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), obs_stats) + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), obs_stats + ) self.obs_stats = table.Table.read(obs_stats) self.obs_stats.convert_bytestring_to_unicode() self.directory = Path(directory) - def single_star_html(self, agasc_id, directory, - static_dir='https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources', - highlight_obs=lambda o: ~o['obs_ok']): - if np.sum(self.agasc_stats['agasc_id'] == agasc_id) == 0: + def single_star_html( + self, + agasc_id, + directory, + static_dir="https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources", + highlight_obs=lambda o: ~o["obs_ok"], + ): + if np.sum(self.agasc_stats["agasc_id"] == agasc_id) == 0: return - star_template = JINJA2.get_template('star_report.html') + star_template = JINJA2.get_template("star_report.html") directory = Path(directory) if not directory.exists(): - logger.debug(f'making report directory {directory}') + logger.debug(f"making report directory {directory}") directory.mkdir(parents=True) - obs_stat = self.obs_stats[self.obs_stats['agasc_id'] == agasc_id] + obs_stat = self.obs_stats[self.obs_stats["agasc_id"] == agasc_id] if len(obs_stat) == 0: - raise Exception(f'agasc_id {agasc_id} has not observations') - obs_stat.sort(keys=['mp_starcat_time']) - agasc_stat = dict(self.agasc_stats[self.agasc_stats['agasc_id'] == agasc_id][0]) - agasc_stat['n_obs_bad'] = \ - agasc_stat['n_obsids'] - agasc_stat['n_obsids_ok'] - agasc_stat['last_obs'] = ':'.join(obs_stat[-1]['mp_starcat_time'].split(':')[:4]) + raise Exception(f"agasc_id {agasc_id} has not observations") + obs_stat.sort(keys=["mp_starcat_time"]) + agasc_stat = dict(self.agasc_stats[self.agasc_stats["agasc_id"] == agasc_id][0]) + agasc_stat["n_obs_bad"] = agasc_stat["n_obsids"] - agasc_stat["n_obsids_ok"] + agasc_stat["last_obs"] = ":".join( + obs_stat[-1]["mp_starcat_time"].split(":")[:4] + ) # OBSIDs can be repeated - obsids = list(np.unique(obs_stat[highlight_obs(obs_stat)]['obsid'])) - - args = [{'only_ok': False, 'draw_agasc_mag': True, 'draw_legend': True, 'ylim': 'max'}, - {'title': 'Magnitude Estimates', - 'only_ok': True, - 'ylim': 'stats', - 'highlight_obsid': obsids, - 'draw_obs_mag_stats': True, - 'draw_agasc_mag_stats': True, - 'draw_legend': True, - 'outside_markers': True - }, - {'type': 'flags'}] + obsids = list(np.unique(obs_stat[highlight_obs(obs_stat)]["obsid"])) + + args = [ + { + "only_ok": False, + "draw_agasc_mag": True, + "draw_legend": True, + "ylim": "max", + }, + { + "title": "Magnitude Estimates", + "only_ok": True, + "ylim": "stats", + "highlight_obsid": obsids, + "draw_obs_mag_stats": True, + "draw_agasc_mag_stats": True, + "draw_legend": True, + "outside_markers": True, + }, + {"type": "flags"}, + ] for obsid in obsids: - args.append({'obsid': obsid, - 'ylim': 'fit', - 'only_ok': False, - 'draw_obs_mag_stats': True, - 'draw_agasc_mag_stats': True, - 'draw_legend': True, - 'draw_roll_mean': True, - 'outside_markers': True - }) - args.append({'type': 'flags', 'obsid': obsid}) - fig = self.plot_set(agasc_id, args=args, filename=directory / 'mag_stats.png') + args.append( + { + "obsid": obsid, + "ylim": "fit", + "only_ok": False, + "draw_obs_mag_stats": True, + "draw_agasc_mag_stats": True, + "draw_legend": True, + "draw_roll_mean": True, + "outside_markers": True, + } + ) + args.append({"type": "flags", "obsid": obsid}) + fig = self.plot_set(agasc_id, args=args, filename=directory / "mag_stats.png") plt.close(fig) - with open(directory / 'index.html', 'w') as out: + with open(directory / "index.html", "w") as out: with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message="Warning: converting a masked element to nan.", ) - out.write(star_template.render( - agasc_stats=agasc_stat, - obs_stats=obs_stat.as_array(), - static_dir=static_dir, - glossary=GLOSSARY) + out.write( + star_template.render( + agasc_stats=agasc_stat, + obs_stats=obs_stat.as_array(), + static_dir=static_dir, + glossary=GLOSSARY, + ) ) - with open(directory / 'data.json', 'w') as json_out: + with open(directory / "data.json", "w") as json_out: json.dump( { - 'agasc_stats': agasc_stat, - 'obs_stats': obs_stat, - 'static_dir': static_dir, - 'glossary': GLOSSARY, + "agasc_stats": agasc_stat, + "obs_stats": obs_stat, + "static_dir": static_dir, + "glossary": GLOSSARY, }, json_out, cls=TableEncoder, ) - return directory / 'index.html' - - def multi_star_html(self, sections=None, updated_stars=None, fails=(), - tstart=None, tstop=None, report_date=None, - filename=None, - include_all_stars=False, - make_single_reports=True, - nav_links=None, - highlight_obs=lambda o: ~o['obs_ok'], - static_dir='https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources', - no_progress=None): + return directory / "index.html" + + def multi_star_html( + self, + sections=None, + updated_stars=None, + fails=(), + tstart=None, + tstop=None, + report_date=None, + filename=None, + include_all_stars=False, + make_single_reports=True, + nav_links=None, + highlight_obs=lambda o: ~o["obs_ok"], + static_dir="https://cxc.cfa.harvard.edu/mta/ASPECT/www_resources", + no_progress=None, + ): if sections is None: sections = [] else: # Copying this because it is a dict that will get modified sections = copy.deepcopy(sections) - run_template = JINJA2.get_template('run_report.html') + run_template = JINJA2.get_template("run_report.html") - updated_star_ids = \ - updated_stars['agasc_id'] if updated_stars is not None and len(updated_stars) else [] + updated_star_ids = ( + updated_stars["agasc_id"] + if updated_stars is not None and len(updated_stars) + else [] + ) if updated_stars is None: updated_stars = [] info = { - 'tstart': tstart if tstart else CxoTime(self.obs_stats['mp_starcat_time']).min().date, - 'tstop': tstop if tstop else CxoTime(self.obs_stats['mp_starcat_time']).max().date, - 'report_date': report_date if report_date else CxoTime.now().date + "tstart": ( + tstart + if tstart + else CxoTime(self.obs_stats["mp_starcat_time"]).min().date + ), + "tstop": ( + tstop + if tstop + else CxoTime(self.obs_stats["mp_starcat_time"]).max().date + ), + "report_date": report_date if report_date else CxoTime.now().date, } if filename is None: @@ -198,148 +240,183 @@ def multi_star_html(self, sections=None, updated_stars=None, fails=(), # this is the list of agasc_id for which we will generate individual reports (if possible) if sections: - agasc_ids = np.concatenate([np.array(s['stars'], dtype=int) for s in sections]) + agasc_ids = np.concatenate( + [np.array(s["stars"], dtype=int) for s in sections] + ) else: agasc_ids = [] if include_all_stars: - sections.append({ - 'id': 'other_stars', - 'title': 'Other Stars', - 'stars': self.agasc_stats['agasc_id'][ - ~np.in1d(self.agasc_stats['agasc_id'], agasc_ids)] - }) - agasc_ids = self.agasc_stats['agasc_id'] - failed_agasc_ids = [f['agasc_id'] for f in fails - if f['agasc_id'] and int(f['agasc_id']) in self.obs_stats['agasc_id']] + sections.append( + { + "id": "other_stars", + "title": "Other Stars", + "stars": self.agasc_stats["agasc_id"][ + ~np.in1d(self.agasc_stats["agasc_id"], agasc_ids) + ], + } + ) + agasc_ids = self.agasc_stats["agasc_id"] + failed_agasc_ids = [ + f["agasc_id"] + for f in fails + if f["agasc_id"] and int(f["agasc_id"]) in self.obs_stats["agasc_id"] + ] agasc_ids = np.unique(np.concatenate([agasc_ids, failed_agasc_ids])) # this turns all None into '' in a new list of failures - fails = [{k: '' if v is None else v for k, v in f.items()} for i, f in enumerate(fails)] + fails = [ + {k: "" if v is None else v for k, v in f.items()} + for i, f in enumerate(fails) + ] agasc_stats = self.agasc_stats.copy() # add some extra fields if len(agasc_stats): - agasc_stats['n_obs_bad_fail'] = agasc_stats['n_obsids_fail'] - agasc_stats['n_obs_bad'] = agasc_stats['n_obsids'] - agasc_stats['n_obsids_ok'] - agasc_stats['flag'] = ' ' - agasc_stats['flag'][:] = '' - agasc_stats['flag'][(agasc_stats['n_obs_bad'] > 0) - | (agasc_stats['n_obsids'] == 0)] = 'warning' - agasc_stats['flag'][agasc_stats['n_obs_bad_fail'] > 0] = 'danger' - agasc_stats['delta'] = (agasc_stats['t_mean_dr3'] - agasc_stats['mag_aca']) - agasc_stats['sigma'] = ((agasc_stats['t_mean_dr3'] - agasc_stats['mag_aca']) - / agasc_stats['mag_aca_err']) - agasc_stats['new'] = True - agasc_stats['new'][np.in1d(agasc_stats['agasc_id'], updated_star_ids)] = False - agasc_stats['update_mag_aca'] = np.nan - agasc_stats['update_mag_aca_err'] = np.nan - agasc_stats['last_obs'] = CxoTime(agasc_stats['last_obs_time']).date + agasc_stats["n_obs_bad_fail"] = agasc_stats["n_obsids_fail"] + agasc_stats["n_obs_bad"] = ( + agasc_stats["n_obsids"] - agasc_stats["n_obsids_ok"] + ) + agasc_stats["flag"] = " " + agasc_stats["flag"][:] = "" + agasc_stats["flag"][ + (agasc_stats["n_obs_bad"] > 0) | (agasc_stats["n_obsids"] == 0) + ] = "warning" + agasc_stats["flag"][agasc_stats["n_obs_bad_fail"] > 0] = "danger" + agasc_stats["delta"] = agasc_stats["t_mean_dr3"] - agasc_stats["mag_aca"] + agasc_stats["sigma"] = ( + agasc_stats["t_mean_dr3"] - agasc_stats["mag_aca"] + ) / agasc_stats["mag_aca_err"] + agasc_stats["new"] = True + agasc_stats["new"][ + np.in1d(agasc_stats["agasc_id"], updated_star_ids) + ] = False + agasc_stats["update_mag_aca"] = np.nan + agasc_stats["update_mag_aca_err"] = np.nan + agasc_stats["last_obs"] = CxoTime(agasc_stats["last_obs_time"]).date if len(updated_stars): - agasc_stats['update_mag_aca'][np.in1d(agasc_stats['agasc_id'], updated_star_ids)] = \ - updated_stars['mag_aca'] - agasc_stats['update_mag_aca_err'][np.in1d(agasc_stats['agasc_id'], updated_star_ids)] =\ - updated_stars['mag_aca_err'] + agasc_stats["update_mag_aca"][ + np.in1d(agasc_stats["agasc_id"], updated_star_ids) + ] = updated_stars["mag_aca"] + agasc_stats["update_mag_aca_err"][ + np.in1d(agasc_stats["agasc_id"], updated_star_ids) + ] = updated_stars["mag_aca_err"] tooltips = { - 'warning': 'At least one bad observation', - 'danger': 'At least failed observation' + "warning": "At least one bad observation", + "danger": "At least failed observation", } # make all individual star reports star_reports = {} - logger.debug('-' * 80) + logger.debug("-" * 80) logger.info("Generating star reports") - for agasc_id in tqdm(np.atleast_1d(agasc_ids).tolist(), - desc='progress', disable=no_progress, unit='star'): + for agasc_id in tqdm( + np.atleast_1d(agasc_ids).tolist(), + desc="progress", + disable=no_progress, + unit="star", + ): try: - logger.debug('-' * 80) - logger.debug(f'{agasc_id=}') - dirname = self.directory / 'stars' / f'{agasc_id//1e7:03.0f}' / f'{agasc_id:.0f}' + logger.debug("-" * 80) + logger.debug(f"{agasc_id=}") + dirname = ( + self.directory + / "stars" + / f"{agasc_id//1e7:03.0f}" + / f"{agasc_id:.0f}" + ) if make_single_reports: self.single_star_html( - agasc_id, - directory=dirname, - highlight_obs=highlight_obs + agasc_id, directory=dirname, highlight_obs=highlight_obs ) star_reports[agasc_id] = dirname except mag_estimate.MagStatsException: - logger.debug(f' Error generating report for {agasc_id=}') + logger.debug(f" Error generating report for {agasc_id=}") # remove empty sections, and set the star tables for each of the remaining sections sections = sections.copy() - sections = [section for section in sections if len(section['stars'])] + sections = [section for section in sections if len(section["stars"])] for section in sections: - section['stars'] = agasc_stats[np.in1d(agasc_stats['agasc_id'], - section['stars'])].as_array() + section["stars"] = agasc_stats[ + np.in1d(agasc_stats["agasc_id"], section["stars"]) + ].as_array() # this is a hack - star_reports = {i: str(star_reports[i].relative_to(self.directory)) for i in star_reports} + star_reports = { + i: str(star_reports[i].relative_to(self.directory)) for i in star_reports + } # make report if not self.directory.exists(): self.directory.mkdir(parents=True) - with open(self.directory / filename, 'w') as out: + with open(self.directory / filename, "w") as out: with warnings.catch_warnings(): warnings.filterwarnings( "ignore", message="Warning: converting a masked element to nan.", ) - out.write(run_template.render( - info=info, - sections=sections, - failures=fails, - star_reports=star_reports, - nav_links=nav_links, - tooltips=tooltips, - static_dir=static_dir, - glossary=GLOSSARY - )) - - json_filename = filename.replace('.html', '.json') + out.write( + run_template.render( + info=info, + sections=sections, + failures=fails, + star_reports=star_reports, + nav_links=nav_links, + tooltips=tooltips, + static_dir=static_dir, + glossary=GLOSSARY, + ) + ) + + json_filename = filename.replace(".html", ".json") if json_filename == filename: - json_filename = filename + '.json' - with open(self.directory / json_filename, 'w') as json_out: + json_filename = filename + ".json" + with open(self.directory / json_filename, "w") as json_out: json.dump( { - 'info': info, - 'sections': sections, - 'failures': fails, - 'star_reports': star_reports, - 'tooltips': tooltips, - 'static_dir': static_dir, - 'glossary': GLOSSARY, + "info": info, + "sections": sections, + "failures": fails, + "star_reports": star_reports, + "tooltips": tooltips, + "static_dir": static_dir, + "glossary": GLOSSARY, }, json_out, cls=TableEncoder, ) - def plot_agasc_id_single(self, agasc_id, obsid=None, - telem=None, - highlight_obsid=(), - highlight_outliers=True, - only_ok=True, - title=None, - draw_agasc_mag_stats=False, - draw_obs_mag_stats=False, - draw_agasc_mag=False, - draw_roll_mean=False, - draw_legend=False, - ylim='fit', - ax=None, - outside_markers=False): + def plot_agasc_id_single( + self, + agasc_id, + obsid=None, + telem=None, + highlight_obsid=(), + highlight_outliers=True, + only_ok=True, + title=None, + draw_agasc_mag_stats=False, + draw_obs_mag_stats=False, + draw_agasc_mag=False, + draw_roll_mean=False, + draw_legend=False, + ylim="fit", + ax=None, + outside_markers=False, + ): if title is not None: ax.set_title(title) elif obsid is not None: - ax.set_title(f'OBSID {obsid}') - if type(highlight_obsid) is not list and np.isscalar(highlight_obsid): + ax.set_title(f"OBSID {obsid}") + if not isinstance(highlight_obsid, list) and np.isscalar(highlight_obsid): highlight_obsid = [highlight_obsid] - agasc_stat = self.agasc_stats[self.agasc_stats['agasc_id'] == agasc_id][0] - obs_stats = self.obs_stats[self.obs_stats['agasc_id'] == agasc_id] + agasc_stat = self.agasc_stats[self.agasc_stats["agasc_id"] == agasc_id][0] + obs_stats = self.obs_stats[self.obs_stats["agasc_id"] == agasc_id] arg_obsid = obsid if arg_obsid: - obs_stats = obs_stats[obs_stats['obsid'] == arg_obsid] + obs_stats = obs_stats[obs_stats["obsid"] == arg_obsid] previous_axes = plt.gca() if ax is not None: @@ -348,93 +425,109 @@ def plot_agasc_id_single(self, agasc_id, obsid=None, ax = plt.gca() if telem is None: try: - telem = mag_estimate.get_telemetry_by_agasc_id(agasc_id, ignore_exceptions=True) + telem = mag_estimate.get_telemetry_by_agasc_id( + agasc_id, ignore_exceptions=True + ) telem = mag_estimate.add_obs_info(telem, obs_stats) except Exception as e: - logger.debug(f'Error making plot: {e}') + logger.debug(f"Error making plot: {e}") telem = [] - if len(telem) == 0 or (arg_obsid is not None and np.sum(telem['obsid'] == arg_obsid) == 0): - msg = 'No Telemetry' + if len(telem) == 0 or ( + arg_obsid is not None and np.sum(telem["obsid"] == arg_obsid) == 0 + ): + msg = "No Telemetry" if arg_obsid is not None: - msg += f' for OBSID {arg_obsid}' + msg += f" for OBSID {arg_obsid}" ax.text( np.mean(ax.get_xlim()), np.mean(ax.get_ylim()), msg, - horizontalalignment='center', - verticalalignment='center') + horizontalalignment="center", + verticalalignment="center", + ) return - obsids = [arg_obsid] if arg_obsid else np.unique(telem['obsid']) + obsids = [arg_obsid] if arg_obsid else np.unique(telem["obsid"]) - timeline = telem[['times', 'mags', 'obsid', 'obs_outlier']].copy() - timeline['index'] = np.arange(len(timeline)) - timeline['mean'] = np.nan - timeline['std'] = np.nan - timeline['mag_mean'] = np.nan - timeline['mag_std'] = np.nan - for i, obsid in enumerate(np.unique(timeline['obsid'])): - sel = (obs_stats['obsid'] == obsid) + timeline = telem[["times", "mags", "obsid", "obs_outlier"]].copy() + timeline["index"] = np.arange(len(timeline)) + timeline["mean"] = np.nan + timeline["std"] = np.nan + timeline["mag_mean"] = np.nan + timeline["mag_std"] = np.nan + for obsid in np.unique(timeline["obsid"]): + sel = obs_stats["obsid"] == obsid if draw_obs_mag_stats and np.any(sel): - timeline['mag_mean'][timeline['obsid'] == obsid] = obs_stats[sel]['mean'][0] - timeline['mag_std'][timeline['obsid'] == obsid] = obs_stats[sel]['std'][0] + timeline["mag_mean"][timeline["obsid"] == obsid] = obs_stats[sel][ + "mean" + ][0] + timeline["mag_std"][timeline["obsid"] == obsid] = obs_stats[sel]["std"][ + 0 + ] timeline = timeline.as_array() - ok = (telem['AOACASEQ'] == 'KALM') & (telem['AOACIIR'] == 'OK') & \ - (telem['AOPCADMD'] == 'NPNT') - ok = ok & (telem['dr'] < 5) + ok = ( + (telem["AOACASEQ"] == "KALM") + & (telem["AOACIIR"] == "OK") + & (telem["AOPCADMD"] == "NPNT") + ) + ok = ok & (telem["dr"] < 5) # set the limits of the plot beforehand - ok_ylim = ok & (telem['mags'] > 0) + ok_ylim = ok & (telem["mags"] > 0) ylims_set = False if arg_obsid is not None: - ok_ylim = ok_ylim & (telem['obsid'] == arg_obsid) - if ylim == 'fit': + ok_ylim = ok_ylim & (telem["obsid"] == arg_obsid) + if ylim == "fit": if np.sum(ok_ylim): - q25, q50, q75 = np.quantile(telem['mags'][ok_ylim], [.25, 0.5, 0.75]) + q25, q50, q75 = np.quantile(telem["mags"][ok_ylim], [0.25, 0.5, 0.75]) else: - q25, q50, q75 = np.quantile(telem['mags'], [.25, 0.5, 0.75]) + q25, q50, q75 = np.quantile(telem["mags"], [0.25, 0.5, 0.75]) iqr = max(q75 - q25, 0.05) ax.set_ylim((q25 - 2 * iqr, q75 + 2 * iqr)) ylims_set = True - elif ylim == 'stats': + elif ylim == "stats": if arg_obsid is not None: - q25, q50, q75 = obs_stats[['q25', 'median', 'q75']][0] + q25, q50, q75 = obs_stats[["q25", "median", "q75"]][0] iqr = max(q75 - q25, 0.05) ax.set_ylim((q25 - 3 * iqr, q75 + 3 * iqr)) ylims_set = True - elif arg_obsid is None and agasc_stat['mag_obs_std'] > 0: - ylim = (agasc_stat['mag_obs'] - 6 * agasc_stat['mag_obs_std'], - agasc_stat['mag_obs'] + 6 * agasc_stat['mag_obs_std']) + elif arg_obsid is None and agasc_stat["mag_obs_std"] > 0: + ylim = ( + agasc_stat["mag_obs"] - 6 * agasc_stat["mag_obs_std"], + agasc_stat["mag_obs"] + 6 * agasc_stat["mag_obs_std"], + ) ax.set_ylim(ylim) ylims_set = True - if ylim == 'max' or not ylims_set: + if ylim == "max" or not ylims_set: if np.any(ok_ylim): - ymin, ymax = np.min(telem['mags'][ok_ylim]), np.max(telem['mags'][ok_ylim]) + ymin, ymax = np.min(telem["mags"][ok_ylim]), np.max( + telem["mags"][ok_ylim] + ) else: - ymin, ymax = np.min(telem['mags']), np.max(telem['mags']) + ymin, ymax = np.min(telem["mags"]), np.max(telem["mags"]) dy = max(0.3, ymax - ymin) ax.set_ylim((ymin - 0.1 * dy, ymax + 0.1 * dy)) # set flags for different categories of markers - highlighted = np.zeros(len(timeline['times']), dtype=bool) + highlighted = np.zeros(len(timeline["times"]), dtype=bool) if highlight_obsid: - highlighted = highlighted | np.in1d(timeline['obsid'], highlight_obsid) + highlighted = highlighted | np.in1d(timeline["obsid"], highlight_obsid) if highlight_outliers: - highlighted = highlighted | timeline['obs_outlier'] + highlighted = highlighted | timeline["obs_outlier"] ymin, ymax = ax.get_ylim() ymin, ymax = (ymin + 0.01 * (ymax - ymin)), (ymax - 0.01 * (ymax - ymin)) - top = np.ones_like(timeline['mags']) * ymax - bottom = np.ones_like(timeline['mags']) * ymin + top = np.ones_like(timeline["mags"]) * ymax + bottom = np.ones_like(timeline["mags"]) * ymin if outside_markers: - outside_up = timeline['mags'] >= ymax - outside_down = timeline['mags'] <= ymin + outside_up = timeline["mags"] >= ymax + outside_down = timeline["mags"] <= ymin else: - outside_up = np.zeros_like(timeline['mags'], dtype=bool) - outside_down = np.zeros_like(timeline['mags'], dtype=bool) + outside_up = np.zeros_like(timeline["mags"], dtype=bool) + outside_down = np.zeros_like(timeline["mags"], dtype=bool) inside = ~outside_up & ~outside_down # loop over obsids, making each plot @@ -442,99 +535,135 @@ def plot_agasc_id_single(self, agasc_id, obsid=None, line_handles = [] limits = {} for i, obsid in enumerate(obsids): - in_obsid = timeline['obsid'] == obsid - if len(timeline[timeline['obsid'] == obsid]) == 0: + in_obsid = timeline["obsid"] == obsid + if len(timeline[timeline["obsid"] == obsid]) == 0: continue - limits[obsid] = (timeline['index'][timeline['obsid'] == obsid].min(), - timeline['index'][timeline['obsid'] == obsid].max()) + limits[obsid] = ( + timeline["index"][timeline["obsid"] == obsid].min(), + timeline["index"][timeline["obsid"] == obsid].max(), + ) if not only_ok and np.any(in_obsid & ~ok): - s = plt.scatter(timeline['index'][in_obsid & ~ok & inside], - timeline[in_obsid & ~ok & inside]['mags'], - s=10, marker='.', color='r', label='not OK') + s = plt.scatter( + timeline["index"][in_obsid & ~ok & inside], + timeline[in_obsid & ~ok & inside]["mags"], + s=10, + marker=".", + color="r", + label="not OK", + ) if i == 0: marker_handles.append(s) _ = plt.scatter( - timeline['index'][in_obsid & ~ok & outside_down], + timeline["index"][in_obsid & ~ok & outside_down], bottom[in_obsid & ~ok & outside_down], - s=10, marker='v', color='r' + s=10, + marker="v", + color="r", ) _ = plt.scatter( - timeline['index'][in_obsid & ~ok & outside_up], + timeline["index"][in_obsid & ~ok & outside_up], top[in_obsid & ~ok & outside_up], - s=10, marker='^', color='r' + s=10, + marker="^", + color="r", ) if np.any(in_obsid & ok & highlighted): s = plt.scatter( - timeline['index'][in_obsid & ok & highlighted & inside], - timeline[in_obsid & ok & highlighted & inside]['mags'], - s=10, marker='.', color='orange', label='Highlighted' + timeline["index"][in_obsid & ok & highlighted & inside], + timeline[in_obsid & ok & highlighted & inside]["mags"], + s=10, + marker=".", + color="orange", + label="Highlighted", ) if i == 0: marker_handles.append(s) _ = plt.scatter( - timeline['index'][in_obsid & ok & highlighted & outside_up], + timeline["index"][in_obsid & ok & highlighted & outside_up], top[in_obsid & ok & highlighted & outside_up], - s=10, marker='^', color='orange', label='Highlighted' + s=10, + marker="^", + color="orange", + label="Highlighted", ) _ = plt.scatter( - timeline['index'][in_obsid & ok & highlighted & outside_down], + timeline["index"][in_obsid & ok & highlighted & outside_down], bottom[in_obsid & ok & highlighted & outside_down], - s=10, marker='v', color='orange', label='Highlighted' + s=10, + marker="v", + color="orange", + label="Highlighted", ) if np.any(in_obsid & ok & ~highlighted): s = plt.scatter( - timeline['index'][in_obsid & ok & ~highlighted & inside], - timeline[in_obsid & ok & ~highlighted & inside]['mags'], - s=10, marker='.', color='k', label='OK' + timeline["index"][in_obsid & ok & ~highlighted & inside], + timeline[in_obsid & ok & ~highlighted & inside]["mags"], + s=10, + marker=".", + color="k", + label="OK", ) if i == 0: marker_handles.append(s) _ = plt.scatter( - timeline['index'][ - in_obsid & ok & (~highlighted) & outside_down], + timeline["index"][in_obsid & ok & (~highlighted) & outside_down], bottom[in_obsid & ok & ~highlighted & outside_down], - s=10, marker='v', color='k', label='OK' + s=10, + marker="v", + color="k", + label="OK", ) _ = plt.scatter( - timeline['index'][in_obsid & ok & ~highlighted & outside_up], + timeline["index"][in_obsid & ok & ~highlighted & outside_up], top[in_obsid & ok & ~highlighted & outside_up], - s=10, marker='^', color='k', label='OK' + s=10, + marker="^", + color="k", + label="OK", ) - sel = (obs_stats['obsid'] == obsid) + sel = obs_stats["obsid"] == obsid if draw_obs_mag_stats and np.sum(sel): - label = '' if i else 'mag$_{OBSID}$' - if (np.isfinite(obs_stats[sel][0]['t_mean']) - and np.isfinite(obs_stats[sel][0]['t_std'])): - mag_mean = obs_stats[sel][0]['t_mean'] - mag_mean_minus = mag_mean - obs_stats[sel][0]['t_std'] - mag_mean_plus = mag_mean + obs_stats[sel][0]['t_std'] + label = "" if i else "mag$_{OBSID}$" + if np.isfinite(obs_stats[sel][0]["t_mean"]) and np.isfinite( + obs_stats[sel][0]["t_std"] + ): + mag_mean = obs_stats[sel][0]["t_mean"] + mag_mean_minus = mag_mean - obs_stats[sel][0]["t_std"] + mag_mean_plus = mag_mean + obs_stats[sel][0]["t_std"] lh = ax.plot( limits[obsid], [mag_mean, mag_mean], - linewidth=2, color='orange', label=label + linewidth=2, + color="orange", + label=label, ) if i == 0: line_handles += lh - ax.fill_between(limits[obsid], - [mag_mean_minus, mag_mean_minus], - [mag_mean_plus, mag_mean_plus], - color='orange', alpha=0.1, zorder=100) - else: - ( - ax.plot([], [], linewidth=2, color='orange', label=label) + ax.fill_between( + limits[obsid], + [mag_mean_minus, mag_mean_minus], + [mag_mean_plus, mag_mean_plus], + color="orange", + alpha=0.1, + zorder=100, ) + else: + (ax.plot([], [], linewidth=2, color="orange", label=label)) if draw_roll_mean: - o = (timeline['obsid'] == obsid) & ok + o = (timeline["obsid"] == obsid) & ok - roll_mean = mag_estimate.rolling_mean(timeline['times'], - timeline['mags'], - window=100, - selection=o) + roll_mean = mag_estimate.rolling_mean( + timeline["times"], timeline["mags"], window=100, selection=o + ) lh = ax.plot( - timeline['index'], roll_mean, '--', - linewidth=1, color='r', label='rolling mean' + timeline["index"], + roll_mean, + "--", + linewidth=1, + color="r", + label="rolling mean", ) if i == 0: line_handles += lh @@ -542,45 +671,67 @@ def plot_agasc_id_single(self, agasc_id, obsid=None, sorted_obsids = sorted(limits.keys(), key=lambda lim: limits[lim][1]) for i, obsid in enumerate(sorted_obsids): (tmin, tmax) = limits[obsid] - ax.plot([tmin, tmin], ax.get_ylim(), ':', color='purple', scaley=False) + ax.plot([tmin, tmin], ax.get_ylim(), ":", color="purple", scaley=False) shift = 0.07 * (ax.get_ylim()[1] - ax.get_ylim()[0]) * (1 + i % 3) - ax.text(np.mean([tmin, tmax]), ax.get_ylim()[0] + shift, f'{obsid}', - verticalalignment='top', horizontalalignment='center') + ax.text( + np.mean([tmin, tmax]), + ax.get_ylim()[0] + shift, + f"{obsid}", + verticalalignment="top", + horizontalalignment="center", + ) if limits: tmax = max([v[1] for v in limits.values()]) - ax.plot([tmax, tmax], ax.get_ylim(), ':', color='purple', scaley=False) + ax.plot([tmax, tmax], ax.get_ylim(), ":", color="purple", scaley=False) xlim = ax.get_xlim() ax.set_xlim(xlim) if draw_agasc_mag: - mag_aca = np.mean(agasc_stat['mag_aca']) - line_handles += ( - ax.plot(xlim, [mag_aca, mag_aca], label='mag$_{AGASC}$', - color='green', scalex=False, scaley=False) + mag_aca = np.mean(agasc_stat["mag_aca"]) + line_handles += ax.plot( + xlim, + [mag_aca, mag_aca], + label="mag$_{AGASC}$", + color="green", + scalex=False, + scaley=False, ) - if (draw_agasc_mag_stats - and np.isfinite(agasc_stat['mag_obs']) and agasc_stat['mag_obs'] > 0): - mag_weighted_mean = agasc_stat['mag_obs'] - mag_weighted_std = agasc_stat['mag_obs_std'] - line_handles += ( - ax.plot(ax.get_xlim(), [mag_weighted_mean, mag_weighted_mean], - label='mag', color='r', scalex=False) + if ( + draw_agasc_mag_stats + and np.isfinite(agasc_stat["mag_obs"]) + and agasc_stat["mag_obs"] > 0 + ): + mag_weighted_mean = agasc_stat["mag_obs"] + mag_weighted_std = agasc_stat["mag_obs_std"] + line_handles += ax.plot( + ax.get_xlim(), + [mag_weighted_mean, mag_weighted_mean], + label="mag", + color="r", + scalex=False, + ) + ax.fill_between( + xlim, + [ + mag_weighted_mean - mag_weighted_std, + mag_weighted_mean - mag_weighted_std, + ], + [ + mag_weighted_mean + mag_weighted_std, + mag_weighted_mean + mag_weighted_std, + ], + color="r", + alpha=0.1, ) - ax.fill_between(xlim, - [mag_weighted_mean - mag_weighted_std, - mag_weighted_mean - mag_weighted_std], - [mag_weighted_mean + mag_weighted_std, - mag_weighted_mean + mag_weighted_std], - color='r', alpha=0.1) if draw_legend: ax.set_xlim((xlim[0], xlim[1] + 0.1 * (xlim[1] - xlim[0]))) if marker_handles: - ax.add_artist(plt.legend(handles=marker_handles, loc='lower right')) + ax.add_artist(plt.legend(handles=marker_handles, loc="lower right")) if line_handles: - plt.legend(handles=line_handles, loc='upper right') + plt.legend(handles=line_handles, loc="upper right") plt.sca(previous_axes) @@ -590,117 +741,162 @@ def plot_flags(telemetry, ax=None, obsid=None): ax = plt.gca() if len(telemetry) > 0: - timeline = telemetry[['times', 'mags', 'obsid', 'obs_ok', 'dr', 'AOACFCT', - 'AOACASEQ', 'AOACIIR', 'AOPCADMD', - ]] - timeline['x'] = np.arange(len(timeline)) - timeline['y'] = np.ones(len(timeline)) + timeline = telemetry[ + [ + "times", + "mags", + "obsid", + "obs_ok", + "dr", + "AOACFCT", + "AOACASEQ", + "AOACIIR", + "AOPCADMD", + ] + ] + timeline["x"] = np.arange(len(timeline)) + timeline["y"] = np.ones(len(timeline)) timeline = timeline.as_array() - all_ok = ((timeline['AOACASEQ'] == 'KALM') - & (timeline['AOPCADMD'] == 'NPNT') - & (timeline['AOACFCT'] == 'TRAK') - & (timeline['AOACIIR'] == 'OK') - & (timeline['dr'] < 3) - ) + all_ok = ( + (timeline["AOACASEQ"] == "KALM") + & (timeline["AOPCADMD"] == "NPNT") + & (timeline["AOACFCT"] == "TRAK") + & (timeline["AOACIIR"] == "OK") + & (timeline["dr"] < 3) + ) flags = [ - ('dr > 5', ((timeline['AOACASEQ'] == 'KALM') - & (timeline['AOPCADMD'] == 'NPNT') - & (timeline['AOACFCT'] == 'TRAK') - & (timeline['dr'] >= 5))), - ('Ion. rad.', (timeline['AOACIIR'] != 'OK')), - ('not track', ((timeline['AOACASEQ'] == 'KALM') - & (timeline['AOPCADMD'] == 'NPNT') - & (timeline['AOACFCT'] != 'TRAK'))), - ('not Kalman', ((timeline['AOACASEQ'] != 'KALM') - | (timeline['AOPCADMD'] != 'NPNT'))), + ( + "dr > 5", + ( + (timeline["AOACASEQ"] == "KALM") + & (timeline["AOPCADMD"] == "NPNT") + & (timeline["AOACFCT"] == "TRAK") + & (timeline["dr"] >= 5) + ), + ), + ("Ion. rad.", (timeline["AOACIIR"] != "OK")), + ( + "not track", + ( + (timeline["AOACASEQ"] == "KALM") + & (timeline["AOPCADMD"] == "NPNT") + & (timeline["AOACFCT"] != "TRAK") + ), + ), + ( + "not Kalman", + ( + (timeline["AOACASEQ"] != "KALM") + | (timeline["AOPCADMD"] != "NPNT") + ), + ), ] else: timeline_dtype = np.dtype( - [(n, float) for n in ['times', 'mags', 'obsid', 'obs_ok', 'dr']] - + [(n, int) for n in ['AOACFCT', 'AOACASEQ', 'AOACIIR', 'AOPCADMD', - 'x', 'y']] + [(n, float) for n in ["times", "mags", "obsid", "obs_ok", "dr"]] + + [ + (n, int) + for n in ["AOACFCT", "AOACASEQ", "AOACIIR", "AOPCADMD", "x", "y"] + ] ) timeline = np.array([], dtype=timeline_dtype) all_ok = np.array([], dtype=bool) flags = [ - ('dr > 5', np.array([], dtype=bool)), - ('Ion. rad.', np.array([], dtype=bool)), - ('not track', np.array([], dtype=bool)), - ('not Kalman', np.array([], dtype=bool)), + ("dr > 5", np.array([], dtype=bool)), + ("Ion. rad.", np.array([], dtype=bool)), + ("not track", np.array([], dtype=bool)), + ("not Kalman", np.array([], dtype=bool)), ] if obsid: - for i, (l, o) in enumerate(flags): - flags[i] = (l, o[timeline['obsid'] == obsid]) - all_ok = all_ok[timeline['obsid'] == obsid] - timeline = timeline[timeline['obsid'] == obsid] + for i, (l, o) in enumerate(flags): # noqa: E741 + flags[i] = (l, o[timeline["obsid"] == obsid]) + all_ok = all_ok[timeline["obsid"] == obsid] + timeline = timeline[timeline["obsid"] == obsid] - obsids = np.unique(timeline['obsid']) + obsids = np.unique(timeline["obsid"]) limits = {} if len(timeline) > 0: if obsid is None: - all_ok = timeline['obs_ok'] & all_ok - flags = [('OBS not OK', ~timeline['obs_ok'])] + flags + all_ok = timeline["obs_ok"] & all_ok + flags = [("OBS not OK", ~timeline["obs_ok"])] + flags - for i, obsid in enumerate(obsids): - limits[obsid] = (timeline['x'][timeline['obsid'] == obsid].min(), - timeline['x'][timeline['obsid'] == obsid].max()) + for obsid in obsids: + limits[obsid] = ( + timeline["x"][timeline["obsid"] == obsid].min(), + timeline["x"][timeline["obsid"] == obsid].max(), + ) ok = [f[1] for f in flags] labels = [f[0] for f in flags] - ticks = [i for i in range(len(flags))] + ticks = list(range(len(flags))) for i in range(len(ok)): - ax.scatter(timeline['x'][ok[i]], ticks[i] * timeline['y'][ok[i]], s=4, marker='.', - color='k') + ax.scatter( + timeline["x"][ok[i]], + ticks[i] * timeline["y"][ok[i]], + s=4, + marker=".", + color="k", + ) ax.yaxis.set_major_locator(FixedLocator(ticks)) ax.set_yticklabels(labels) ax.set_ylim((-1, ticks[-1] + 1)) - ax.grid(True, axis='y', linestyle=':') + ax.grid(True, axis="y", linestyle=":") sorted_obsids = sorted(limits.keys(), key=lambda lim: limits[lim][1]) for i, obsid in enumerate(sorted_obsids): (tmin, tmax) = limits[obsid] - ax.axvline(tmin, linestyle=':', color='purple') + ax.axvline(tmin, linestyle=":", color="purple") if i == len(limits) - 1: - ax.axvline(tmax, linestyle=':', color='purple') + ax.axvline(tmax, linestyle=":", color="purple") divider = make_axes_locatable(ax) - ax_dr = divider.append_axes("bottom", size='25%', pad=0., sharex=ax) + ax_dr = divider.append_axes("bottom", size="25%", pad=0.0, sharex=ax) if len(timeline) > 0: - dr = timeline['dr'].copy() + dr = timeline["dr"].copy() dr[dr > 10] = 10 ax_dr.scatter( - timeline['x'][all_ok & (dr < 10)], dr[all_ok & (dr < 10)], - s=3, marker='.', color='k' + timeline["x"][all_ok & (dr < 10)], + dr[all_ok & (dr < 10)], + s=3, + marker=".", + color="k", ) ax_dr.scatter( - timeline['x'][all_ok & (dr >= 10)], dr[all_ok & (dr >= 10)], - s=3, marker='^', color='k' + timeline["x"][all_ok & (dr >= 10)], + dr[all_ok & (dr >= 10)], + s=3, + marker="^", + color="k", ) ax_dr.scatter( - timeline['x'][~all_ok & (dr < 10)], dr[~all_ok & (dr < 10)], - s=3, marker='.', color='r' + timeline["x"][~all_ok & (dr < 10)], + dr[~all_ok & (dr < 10)], + s=3, + marker=".", + color="r", ) ax_dr.scatter( - timeline['x'][~all_ok & (dr >= 10)], + timeline["x"][~all_ok & (dr >= 10)], dr[~all_ok & (dr >= 10)], - s=3, marker='^', - color='r' + s=3, + marker="^", + color="r", ) - ax_dr.set_ylabel('dr') + ax_dr.set_ylabel("dr") ax_dr.set_ylim((-0.5, 10.5)) - ax_dr.set_yticks([0., 2.5, 5, 7.5, 10], minor=True) - ax_dr.set_yticks([0., 5, 10], minor=False) - ax_dr.grid(True, axis='y', linestyle=':') + ax_dr.set_yticks([0.0, 2.5, 5, 7.5, 10], minor=True) + ax_dr.set_yticks([0.0, 5, 10], minor=False) + ax_dr.grid(True, axis="y", linestyle=":") for i, obsid in enumerate(sorted_obsids): (tmin, tmax) = limits[obsid] - ax_dr.axvline(tmin, linestyle=':', color='purple') + ax_dr.axvline(tmin, linestyle=":", color="purple") if i == len(sorted_obsids) - 1: - ax_dr.axvline(tmax, linestyle=':', color='purple') + ax_dr.axvline(tmax, linestyle=":", color="purple") def plot_set(self, agasc_id, args, telem=None, filename=None): if not args: @@ -708,13 +904,14 @@ def plot_set(self, agasc_id, args, telem=None, filename=None): if telem is None: try: - telem = mag_estimate.get_telemetry_by_agasc_id(agasc_id, ignore_exceptions=True) + telem = mag_estimate.get_telemetry_by_agasc_id( + agasc_id, ignore_exceptions=True + ) telem = mag_estimate.add_obs_info( - telem, - self.obs_stats[self.obs_stats['agasc_id'] == agasc_id] + telem, self.obs_stats[self.obs_stats["agasc_id"] == agasc_id] ) except Exception as e: - logger.debug(f'Error making plot: {e}') + logger.debug(f"Error making plot: {e}") telem = [] if len(telem) == 0: @@ -723,11 +920,11 @@ def plot_set(self, agasc_id, args, telem=None, filename=None): fig, ax = plt.subplots(len(args), 1, figsize=(15, 3.5 * len(args))) if len(args) == 1: ax = [ax] - ax[0].set_title(f'AGASC ID {agasc_id}') + ax[0].set_title(f"AGASC ID {agasc_id}") for i, kwargs in enumerate(args): - if 'type' in kwargs and kwargs['type'] == 'flags': - o = kwargs['obsid'] if 'obsid' in kwargs else None + if "type" in kwargs and kwargs["type"] == "flags": + o = kwargs["obsid"] if "obsid" in kwargs else None self.plot_flags(telem, ax[i], obsid=o) if i: ax[i].set_xlim(ax[i - 1].get_xlim()) @@ -736,74 +933,87 @@ def plot_set(self, agasc_id, args, telem=None, filename=None): plt.tight_layout() if filename is not None: - print(f'saving {filename}') + print(f"saving {filename}") fig.savefig(filename) return fig -def email_bad_obs_report(bad_obs, to, - sender=f"{getpass.getuser()}@{platform.uname()[1]}"): +def email_bad_obs_report( + bad_obs, to, sender=f"{getpass.getuser()}@{platform.uname()[1]}" +): date = CxoTime().date[:14] - message = JINJA2.get_template('email_report.txt') + message = JINJA2.get_template("email_report.txt") msg = MIMEText(message) msg["From"] = sender msg["To"] = to - msg["Subject"] = \ - f"{len(bad_obs)} suspicious observation{'s' if len(bad_obs) else ''}" \ + msg["Subject"] = ( + f"{len(bad_obs)} suspicious observation{'s' if len(bad_obs) else ''}" f" in magnitude estimates at {date}." + ) p = Popen(["/usr/sbin/sendmail", "-t", "-oi"], stdin=PIPE) p.communicate(msg.as_string().encode()) GLOSSARY = { - 'sample': """ + "sample": """ One sample of ACA image telemetry. This could be a real image readout or it could be a null 4x4 image when the ACA is not tracking. Be aware that the sample period is faster for null 4x4 images (1.025 sec) than for typical 6x6 or 8x8 tracked images (2.05 or 4.1 sec respectively).""", - 'Kalman samples': - 'Subset of samples when ACA could be tracking stars (AOPCADMD == NPNT & AOACASEQ == KALM)', - 'dr3': """ + "Kalman samples": ( + "Subset of samples when ACA could be tracking stars (AOPCADMD == NPNT &" + " AOACASEQ == KALM)" + ), + "dr3": """ Subset of tracked Kalman samples with radial centroid residual < 3 arcsec. This corresponds to "high quality" readouts. This effectively includes the track subset (AOACFCT == TRAK) because readouts with no TRAK are assigned an infinite centroid residual.""", - 'dbox5': """Subset of tracked Kalman samples with centroid residual within a 5 arcsec box. + "dbox5": """Subset of tracked Kalman samples with centroid residual within a 5 arcsec box. This corresponds to spatial filter used by the OBC for inclusion in Kalman filter. This effectively includes the track subset (AOACFCT == TRAK) because readouts with no TRAK are assigned an infinite centroid residual.""", - 'track': 'Subset of Kalman samples where the image is being tracked (AOACFCT == TRAK)', - 'sat_pix': 'Subset of Kalman samples with saturated pixel flag OK (AOACISP == OK)', - 'ion_rad': 'Subset of Kalman samples with ionizing radiation flag OK (AOACIIR == OK)', - 'mag_est_ok': - "Subset of Kalman samples that have a magnitude estimate (track & ion_rad)", - 'n_total': 'Total number of sample regardless of OBC PCAD status', - 'n': 'Synonym for n_total', - 'n_kalman': 'Number of Kalman samples', - 'n_dr3': 'Number of dr3 samples.', - 'n_dbox5': 'Number of dbox5 samples.', - 'n_track': 'Number of track samples.', - 'n_ok_3': 'Number of (track & sat_pix & ion_rad & dr3) samples', - 'n_ok_5': 'Number of (track & sat_pix & ion_rad & dbox5) samples', - 'n_mag_est_ok': 'Number of (track & ion_rad) samples', - 'n_mag_est_ok_3': 'Number of (track & ion_rad & dr3) samples', - 'n_mag_est_ok_5': 'Number of (track & ion_rad & dbox5) samples', - 'f_dr3': - 'Fraction of mag-est-ok samples with centroid residual < 3 arcsec' - '((mag_est_ok & n_dr3)/n_mag_est_ok)', - 'f_dbox5': - 'Fraction of mag-est-ok samples with centroid within 5 arcsec box' - '((mag_est_ok & n_dbox5)/n_mag_est_ok)', - 'f_mag_est_ok': """n_mag_est_ok_3/n_kalman. This is a measure of the fraction of time during - an observation that a magnitude estimate is available.""", - 'f_mag_est_ok_3': "n_mag_est_ok_3/n_kalman.", - 'f_mag_est_ok_5': "n_mag_est_ok_5/n_kalman.", - 'f_ok': 'n_ok_5 / n_kalman. Same as f_ok_5', - 'f_ok_3': """n_ok_3 / n_kalman. This is a measure of the fraction of time during an + "track": ( + "Subset of Kalman samples where the image is being tracked (AOACFCT == TRAK)" + ), + "sat_pix": "Subset of Kalman samples with saturated pixel flag OK (AOACISP == OK)", + "ion_rad": ( + "Subset of Kalman samples with ionizing radiation flag OK (AOACIIR == OK)" + ), + "mag_est_ok": ( + "Subset of Kalman samples that have a magnitude estimate (track & ion_rad)" + ), + "n_total": "Total number of sample regardless of OBC PCAD status", + "n": "Synonym for n_total", + "n_kalman": "Number of Kalman samples", + "n_dr3": "Number of dr3 samples.", + "n_dbox5": "Number of dbox5 samples.", + "n_track": "Number of track samples.", + "n_ok_3": "Number of (track & sat_pix & ion_rad & dr3) samples", + "n_ok_5": "Number of (track & sat_pix & ion_rad & dbox5) samples", + "n_mag_est_ok": "Number of (track & ion_rad) samples", + "n_mag_est_ok_3": "Number of (track & ion_rad & dr3) samples", + "n_mag_est_ok_5": "Number of (track & ion_rad & dbox5) samples", + "f_dr3": ( + "Fraction of mag-est-ok samples with centroid residual < 3 arcsec" + "((mag_est_ok & n_dr3)/n_mag_est_ok)" + ), + "f_dbox5": ( + "Fraction of mag-est-ok samples with centroid within 5 arcsec box" + "((mag_est_ok & n_dbox5)/n_mag_est_ok)" + ), + "f_mag_est_ok": ( + """n_mag_est_ok_3/n_kalman. This is a measure of the fraction of time during + an observation that a magnitude estimate is available.""" + ), + "f_mag_est_ok_3": "n_mag_est_ok_3/n_kalman.", + "f_mag_est_ok_5": "n_mag_est_ok_5/n_kalman.", + "f_ok": "n_ok_5 / n_kalman. Same as f_ok_5", + "f_ok_3": """n_ok_3 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting high-quality star centroids.""", - 'f_ok_5': """ + "f_ok_5": """ n_ok_5 / n_kalman. This is a measure of the fraction of time during an observation that the Kalman filter is getting any star centroid at all. This includes measurements out to 5 arcsec box halfwidth, so potentially 7 arcsec diff --git a/agasc/supplement/magnitudes/star_obs_catalogs.py b/agasc/supplement/magnitudes/star_obs_catalogs.py index 2f5245bc..82d89329 100644 --- a/agasc/supplement/magnitudes/star_obs_catalogs.py +++ b/agasc/supplement/magnitudes/star_obs_catalogs.py @@ -1,14 +1,12 @@ import os + import numpy as np import tables - -from astropy.table import Table, join from astropy import table - +from astropy.table import Table, join from chandra_aca.transform import yagzag_to_pixels from kadi import commands, events - STARS_OBS = None """The table of star observations""" @@ -20,31 +18,35 @@ def get_star_observations(start=None, stop=None, obsid=None): This is basically the join of kadi.commands.get_observations and kadi.commands.get_starcats, with some extra information (pixel row/col, magnitude error). """ - join_keys = ['starcat_date', 'obsid'] + join_keys = ["starcat_date", "obsid"] observations = Table(commands.get_observations(start=start, stop=stop, obsid=obsid)) - observations = observations[~observations['starcat_date'].mask] + observations = observations[~observations["starcat_date"].mask] # the following line removes manual commands - observations = observations[observations['source'] != 'CMD_EVT'] - catalogs = commands.get_starcats_as_table(start=start, stop=stop, obsid=obsid, unique=True) - catalogs = catalogs[np.in1d(catalogs['type'], ['BOT', 'GUI'])] + observations = observations[observations["source"] != "CMD_EVT"] + catalogs = commands.get_starcats_as_table( + start=start, stop=stop, obsid=obsid, unique=True + ) + catalogs = catalogs[np.in1d(catalogs["type"], ["BOT", "GUI"])] star_obs = join(observations, catalogs, keys=join_keys) - star_obs.rename_columns(['id', 'starcat_date'], ['agasc_id', 'mp_starcat_time']) - star_obs['row'], star_obs['col'] = yagzag_to_pixels(star_obs['yang'], star_obs['zang']) + star_obs.rename_columns(["id", "starcat_date"], ["agasc_id", "mp_starcat_time"]) + star_obs["row"], star_obs["col"] = yagzag_to_pixels( + star_obs["yang"], star_obs["zang"] + ) # Add mag_aca_err column - filename = os.path.join(os.environ['SKA'], 'data', 'agasc', 'proseco_agasc_1p7.h5') + filename = os.path.join(os.environ["SKA"], "data", "agasc", "proseco_agasc_1p7.h5") with tables.open_file(filename) as h5: - agasc_ids = h5.root.data.col('AGASC_ID') - mag_errs = h5.root.data.col('MAG_ACA_ERR') * 0.01 + agasc_ids = h5.root.data.col("AGASC_ID") + mag_errs = h5.root.data.col("MAG_ACA_ERR") * 0.01 - tt = Table([agasc_ids, mag_errs], names=['agasc_id', 'mag_aca_err']) - star_obs = table.join(star_obs, tt, keys='agasc_id') + tt = Table([agasc_ids, mag_errs], names=["agasc_id", "mag_aca_err"]) + star_obs = table.join(star_obs, tt, keys="agasc_id") - star_obs.add_index(['mp_starcat_time']) + star_obs.add_index(["mp_starcat_time"]) - max_time = events.dwells.all().latest('tstart').stop - star_obs = star_obs[star_obs['obs_start'] <= max_time] + max_time = events.dwells.all().latest("tstart").stop + star_obs = star_obs[star_obs["obs_start"] <= max_time] return star_obs diff --git a/agasc/supplement/magnitudes/update_mag_supplement.py b/agasc/supplement/magnitudes/update_mag_supplement.py index c029a7b9..9d36346b 100755 --- a/agasc/supplement/magnitudes/update_mag_supplement.py +++ b/agasc/supplement/magnitudes/update_mag_supplement.py @@ -1,28 +1,33 @@ #!/usr/bin/env python -import traceback -import sys -import warnings -import os -import pickle import datetime import logging +import os +import pickle +import sys +import traceback +import warnings from functools import partial from multiprocessing import Pool -import jinja2 -from tqdm import tqdm -import tables +import jinja2 import numpy as np -from astropy import table -from astropy import time, units as u - -from mica.starcheck import get_starcheck_catalog -from agasc.supplement.magnitudes import star_obs_catalogs, mag_estimate, mag_estimate_report as msr -from agasc.supplement.utils import save_version, MAGS_DTYPE +import tables +from astropy import table, time +from astropy import units as u from cxotime import CxoTime +from mica.starcheck import get_starcheck_catalog +from tqdm import tqdm +from agasc.supplement.magnitudes import ( + mag_estimate, + star_obs_catalogs, +) +from agasc.supplement.magnitudes import ( + mag_estimate_report as msr, +) +from agasc.supplement.utils import MAGS_DTYPE, save_version -logger = logging.getLogger('agasc.supplement') +logger = logging.getLogger("agasc.supplement") def level0_archive_time_range(): @@ -31,9 +36,10 @@ def level0_archive_time_range(): :return: tuple of CxoTime """ - import sqlite3 import os - db_file = os.path.expandvars('$SKA/data/mica/archive/aca0/archfiles.db3') + import sqlite3 + + db_file = os.path.expandvars("$SKA/data/mica/archive/aca0/archfiles.db3") with sqlite3.connect(db_file) as connection: cursor = connection.cursor() cursor.execute("select tstop from archfiles order by tstop desc limit 1") @@ -57,22 +63,22 @@ def get_agasc_id_stats(agasc_ids, obs_status_override={}, tstop=None, no_progres :return: astropy.table.Table, astropy.table.Table, list obs_stats, agasc_stats, fails """ - from agasc.supplement.magnitudes import mag_estimate from astropy.table import Table, vstack + from agasc.supplement.magnitudes import mag_estimate + fails = [] obs_stats = [] agasc_stats = [] - bar = tqdm(agasc_ids, desc='progress', disable=no_progress, unit='star') + bar = tqdm(agasc_ids, desc="progress", disable=no_progress, unit="star") for agasc_id in agasc_ids: bar.update() try: - logger.debug('-' * 80) - logger.debug(f'{agasc_id=}') - agasc_stat, obs_stat, obs_fail = \ - mag_estimate.get_agasc_id_stats(agasc_id=agasc_id, - obs_status_override=obs_status_override, - tstop=tstop) + logger.debug("-" * 80) + logger.debug(f"{agasc_id=}") + agasc_stat, obs_stat, obs_fail = mag_estimate.get_agasc_id_stats( + agasc_id=agasc_id, obs_status_override=obs_status_override, tstop=tstop + ) agasc_stats.append(agasc_stat) obs_stats.append(obs_stat) fails += obs_fail @@ -82,19 +88,21 @@ def get_agasc_id_stats(agasc_ids, obs_status_override={}, tstop=None, no_progres fails.append(dict(e)) except Exception as e: # transform Exception to MagStatsException for standard book keeping - msg = f'Unexpected Error: {e}' + msg = f"Unexpected Error: {e}" logger.debug(msg) - fails.append(dict(mag_estimate.MagStatsException(agasc_id=agasc_id, msg=msg))) + fails.append( + dict(mag_estimate.MagStatsException(agasc_id=agasc_id, msg=msg)) + ) exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type is not None: trace = traceback.format_exception(exc_type, exc_value, exc_traceback) for level in trace: for line in level.splitlines(): logger.debug(line) - logger.debug('') + logger.debug("") bar.close() - logger.debug('-' * 80) + logger.debug("-" * 80) try: agasc_stats = Table(agasc_stats) if agasc_stats else None @@ -103,13 +111,19 @@ def get_agasc_id_stats(agasc_ids, obs_status_override={}, tstop=None, no_progres agasc_stats = None obs_stats = None # transform Exception to MagStatsException for standard book keeping - fails.append(dict(mag_estimate.MagStatsException( - msg=f'Exception at end of get_agasc_id_stats: {str(e)}'))) + fails.append( + dict( + mag_estimate.MagStatsException( + msg=f"Exception at end of get_agasc_id_stats: {str(e)}" + ) + ) + ) return obs_stats, agasc_stats, fails -def get_agasc_id_stats_pool(agasc_ids, obs_status_override=None, batch_size=100, tstop=None, - no_progress=None): +def get_agasc_id_stats_pool( + agasc_ids, obs_status_override=None, batch_size=100, tstop=None, no_progress=None +): """ Call update_mag_stats.get_agasc_id_stats multiple times using a multiprocessing.Pool @@ -125,7 +139,8 @@ def get_agasc_id_stats_pool(agasc_ids, obs_status_override=None, batch_size=100, obs_stats, agasc_stats, fails, failed_jobs """ import time - from astropy.table import vstack, Table + + from astropy.table import Table, vstack if obs_status_override is None: obs_status_override = {} @@ -133,14 +148,17 @@ def get_agasc_id_stats_pool(agasc_ids, obs_status_override=None, batch_size=100, jobs = [] args = [] finished = 0 - logger.info(f'Processing {batch_size} stars per job') + logger.info(f"Processing {batch_size} stars per job") for i in range(0, len(agasc_ids), batch_size): - args.append(agasc_ids[i:i + batch_size]) + args.append(agasc_ids[i : i + batch_size]) with Pool() as pool: for arg in args: - jobs.append(pool.apply_async(get_agasc_id_stats, - [arg, obs_status_override, tstop, True])) - bar = tqdm(total=len(jobs), desc='progress', disable=no_progress, unit='job') + jobs.append( + pool.apply_async( + get_agasc_id_stats, [arg, obs_status_override, tstop, True] + ) + ) + bar = tqdm(total=len(jobs), desc="progress", disable=no_progress, unit="job") while finished < len(jobs): finished = sum([f.ready() for f in jobs]) if finished - bar.n: @@ -156,9 +174,13 @@ def get_agasc_id_stats_pool(agasc_ids, obs_status_override=None, batch_size=100, job.get() except Exception as e: for agasc_id in arg: - fails.append(dict( - mag_estimate.MagStatsException(agasc_id=agasc_id, msg=f'Failed job: {e}') - )) + fails.append( + dict( + mag_estimate.MagStatsException( + agasc_id=agasc_id, msg=f"Failed job: {e}" + ) + ) + ) results = [job.get() for job in jobs if job.successful()] @@ -175,22 +197,22 @@ def _update_table(table_old, table_new, keys): # checking names, because actual types change upon saving in fits format if set(table_old.colnames) != set(table_new.colnames): raise Exception( - 'Tables have different columns:' - f'\n {table_old.colnames}' - f'\n {table_new.colnames}' + "Tables have different columns:" + f"\n {table_old.colnames}" + f"\n {table_new.colnames}" ) table_old = table_old.copy() new_row = np.ones(len(table_new), dtype=bool) - _, i_new, i_old = np.intersect1d(table_new[keys].as_array(), - table_old[keys].as_array(), - return_indices=True) + _, i_new, i_old = np.intersect1d( + table_new[keys].as_array(), table_old[keys].as_array(), return_indices=True + ) new_row[i_new] = False columns = table_old.as_array().dtype.names table_old[i_old] = table_new[i_new][columns] return table.vstack([table_old, table_new[new_row][columns]]) -def update_mag_stats(obs_stats, agasc_stats, fails, outdir='.'): +def update_mag_stats(obs_stats, agasc_stats, fails, outdir="."): """ Update the mag_stats catalog. @@ -206,32 +228,36 @@ def update_mag_stats(obs_stats, agasc_stats, fails, outdir='.'): :return: """ if agasc_stats is not None and len(agasc_stats): - filename = outdir / 'mag_stats_agasc.fits' - logger.debug(f'Updating {filename}') + filename = outdir / "mag_stats_agasc.fits" + logger.debug(f"Updating {filename}") if filename.exists(): - agasc_stats = _update_table(table.Table.read(filename), agasc_stats, - keys=['agasc_id']) + agasc_stats = _update_table( + table.Table.read(filename), agasc_stats, keys=["agasc_id"] + ) os.remove(filename) for column in agasc_stats.colnames: if column in mag_estimate.AGASC_ID_STATS_INFO: - agasc_stats[column].description = mag_estimate.AGASC_ID_STATS_INFO[column] + agasc_stats[column].description = mag_estimate.AGASC_ID_STATS_INFO[ + column + ] agasc_stats.write(filename) if obs_stats is not None and len(obs_stats): - filename = outdir / 'mag_stats_obsid.fits' - logger.debug(f'Updating {filename}') + filename = outdir / "mag_stats_obsid.fits" + logger.debug(f"Updating {filename}") if filename.exists(): - obs_stats = _update_table(table.Table.read(filename), obs_stats, - keys=['agasc_id', 'obsid']) + obs_stats = _update_table( + table.Table.read(filename), obs_stats, keys=["agasc_id", "obsid"] + ) os.remove(filename) for column in obs_stats.colnames: if column in mag_estimate.OBS_STATS_INFO: obs_stats[column].description = mag_estimate.OBS_STATS_INFO[column] obs_stats.write(filename) if len(fails): - filename = outdir / 'mag_stats_fails.pkl' - logger.info(f'A summary of all failures is saved in {filename}') + filename = outdir / "mag_stats_fails.pkl" + logger.info(f"A summary of all failures is saved in {filename}") # logger.debug(f'Updating {filename}') - with open(filename, 'wb') as out: + with open(filename, "wb") as out: pickle.dump(fails, out) @@ -250,19 +276,19 @@ def update_supplement(agasc_stats, filename, include_all=True): return [], [] if include_all: - outliers_new = agasc_stats[ - (agasc_stats['n_obsids_ok'] > 0) - ] + outliers_new = agasc_stats[(agasc_stats["n_obsids_ok"] > 0)] else: outliers_new = agasc_stats[ - (agasc_stats['n_obsids_ok'] > 0) - & (agasc_stats['selected_atol'] - | agasc_stats['selected_rtol'] - | agasc_stats['selected_color'] - | agasc_stats['selected_mag_aca_err']) + (agasc_stats["n_obsids_ok"] > 0) + & ( + agasc_stats["selected_atol"] + | agasc_stats["selected_rtol"] + | agasc_stats["selected_color"] + | agasc_stats["selected_mag_aca_err"] + ) ] - outliers_new['mag_aca'] = outliers_new['mag_obs'] - outliers_new['mag_aca_err'] = outliers_new['mag_obs_err'] + outliers_new["mag_aca"] = outliers_new["mag_obs"] + outliers_new["mag_aca_err"] = outliers_new["mag_obs_err"] outliers_new = outliers_new[MAGS_DTYPE.names].as_array() if outliers_new.dtype != MAGS_DTYPE: @@ -273,49 +299,57 @@ def update_supplement(agasc_stats, filename, include_all=True): updated_stars = None if filename.exists(): # I could do what follows directly in place, but the table is not that large. - with tables.File(filename, 'r') as h5: - if 'mags' in h5.root: + with tables.File(filename, "r") as h5: + if "mags" in h5.root: outliers_current = h5.root.mags[:] # find the indices of agasc_ids in both current and new lists - _, i_new, i_cur = np.intersect1d(outliers_new['agasc_id'], - outliers_current['agasc_id'], - return_indices=True) + _, i_new, i_cur = np.intersect1d( + outliers_new["agasc_id"], + outliers_current["agasc_id"], + return_indices=True, + ) current = outliers_current[i_cur] new = outliers_new[i_new] # from those, find the ones which differ in last observation time - i_cur = i_cur[current['last_obs_time'] != new['last_obs_time']] - i_new = i_new[current['last_obs_time'] != new['last_obs_time']] + i_cur = i_cur[current["last_obs_time"] != new["last_obs_time"]] + i_new = i_new[current["last_obs_time"] != new["last_obs_time"]] # overwrite current values with new values (and calculate diff to return) - updated_stars = np.zeros(len(outliers_new[i_new]), - dtype=MAGS_DTYPE) - updated_stars['mag_aca'] = (outliers_new[i_new]['mag_aca'] - - outliers_current[i_cur]['mag_aca']) - updated_stars['mag_aca_err'] = (outliers_new[i_new]['mag_aca_err'] - - outliers_current[i_cur]['mag_aca_err']) - updated_stars['agasc_id'] = outliers_new[i_new]['agasc_id'] + updated_stars = np.zeros(len(outliers_new[i_new]), dtype=MAGS_DTYPE) + updated_stars["mag_aca"] = ( + outliers_new[i_new]["mag_aca"] - outliers_current[i_cur]["mag_aca"] + ) + updated_stars["mag_aca_err"] = ( + outliers_new[i_new]["mag_aca_err"] + - outliers_current[i_cur]["mag_aca_err"] + ) + updated_stars["agasc_id"] = outliers_new[i_new]["agasc_id"] outliers_current[i_cur] = outliers_new[i_new] # find agasc_ids in new list but not in current list - new_stars = ~np.in1d(outliers_new['agasc_id'], outliers_current['agasc_id']) + new_stars = ~np.in1d( + outliers_new["agasc_id"], outliers_current["agasc_id"] + ) # and add them to the current list - outliers_current = np.concatenate([outliers_current, outliers_new[new_stars]]) + outliers_current = np.concatenate( + [outliers_current, outliers_new[new_stars]] + ) outliers = np.sort(outliers_current) - new_stars = outliers_new[new_stars]['agasc_id'] + new_stars = outliers_new[new_stars]["agasc_id"] if outliers is None: logger.warning('Creating new "mags" table') outliers = outliers_new - new_stars = outliers_new['agasc_id'] + new_stars = outliers_new["agasc_id"] updated_stars = np.array([], dtype=MAGS_DTYPE) - mode = 'r+' if filename.exists() else 'w' + mode = "r+" if filename.exists() else "w" with tables.File(filename, mode) as h5: - if 'mags' in h5.root: - h5.remove_node('/mags') - h5.create_table('/', 'mags', outliers) - save_version(filename, 'mags') + if "mags" in h5.root: + h5.remove_node("/mags") + h5.create_table("/", "mags", outliers) + save_version(filename, "mags") return new_stars, updated_stars @@ -323,32 +357,39 @@ def update_supplement(agasc_stats, filename, include_all=True): def write_obs_status_yaml(obs_stats=None, fails=(), filename=None): obs = [] if obs_stats and len(obs_stats): - obs_stats = obs_stats[~obs_stats['obs_ok']] - mp_starcat_times = np.unique(obs_stats['mp_starcat_time']) + obs_stats = obs_stats[~obs_stats["obs_ok"]] + mp_starcat_times = np.unique(obs_stats["mp_starcat_time"]) for mp_starcat_time in mp_starcat_times: - rows = obs_stats[obs_stats['mp_starcat_time'] == mp_starcat_time] - rows.sort(keys='agasc_id') - obs.append({ - 'mp_starcat_time': mp_starcat_time, - 'obsid': obs_stats['obsid'], - 'agasc_id': list(rows['agasc_id']), - 'status': 1, - 'comments': obs_stats['comment'] - }) + rows = obs_stats[obs_stats["mp_starcat_time"] == mp_starcat_time] + rows.sort(keys="agasc_id") + obs.append( + { + "mp_starcat_time": mp_starcat_time, + "obsid": obs_stats["obsid"], + "agasc_id": list(rows["agasc_id"]), + "status": 1, + "comments": obs_stats["comment"], + } + ) for fail in fails: - if fail['agasc_id'] is None or fail['mp_starcat_time'] is None: + if fail["agasc_id"] is None or fail["mp_starcat_time"] is None: continue - mp_starcat_times = fail['mp_starcat_time'] if type(fail['mp_starcat_time']) is list \ - else [fail['mp_starcat_time']] - agasc_id = fail['agasc_id'] + mp_starcat_times = ( + fail["mp_starcat_time"] + if isinstance(fail["mp_starcat_time"], list) + else [fail["mp_starcat_time"]] + ) + agasc_id = fail["agasc_id"] for mp_starcat_time in mp_starcat_times: - obs.append({ - 'mp_starcat_time': mp_starcat_time, - 'obsid': fail['obsid'], - 'agasc_id': [agasc_id], - 'status': 1, - 'comments': fail['msg'] - }) + obs.append( + { + "mp_starcat_time": mp_starcat_time, + "obsid": fail["obsid"], + "agasc_id": [agasc_id], + "status": 1, + "comments": fail["msg"], + } + ) if len(obs) == 0: if filename and filename.exists(): filename.unlink() @@ -356,13 +397,15 @@ def write_obs_status_yaml(obs_stats=None, fails=(), filename=None): agasc_ids = [] for o in obs: - cat = get_starcheck_catalog(o['obsid']) + cat = get_starcheck_catalog(o["obsid"]) if cat: - cat = cat['cat'] - maxmags = dict(zip(cat['id'], cat['maxmag'])) - agasc_ids += [(agasc_id, maxmags.get(agasc_id, -1)) for agasc_id in o['agasc_id']] + cat = cat["cat"] + maxmags = dict(zip(cat["id"], cat["maxmag"])) + agasc_ids += [ + (agasc_id, maxmags.get(agasc_id, -1)) for agasc_id in o["agasc_id"] + ] else: - agasc_ids += [(agasc_id, -1) for agasc_id in obs['agasc_id']] + agasc_ids += [(agasc_id, -1) for agasc_id in obs["agasc_id"]] agasc_ids = dict(sorted(agasc_ids)) @@ -414,28 +457,29 @@ def write_obs_status_yaml(obs_stats=None, fails=(), filename=None): {%- endfor -%}] comments: {{ obs.comments }} {%- endfor %} -""" # noqa +""" # noqa: E501 tpl = jinja2.Template(yaml_template) result = tpl.render(observations=obs, agasc_ids=agasc_ids) if filename: - with open(filename, 'w') as fh: + with open(filename, "w") as fh: fh.write(result) return result -def do(start, - stop, - output_dir, - agasc_ids=None, - report=False, - reports_dir=None, - report_date=None, - multi_process=False, - include_bad=False, - dry_run=False, - no_progress=None, - email='', - ): +def do( + start, + stop, + output_dir, + agasc_ids=None, + report=False, + reports_dir=None, + report_date=None, + multi_process=False, + include_bad=False, + dry_run=False, + no_progress=None, + email="", +): """ :param start: cxotime.CxoTime @@ -468,7 +512,7 @@ def do(start, # as ascii. It displays a warning which I want to avoid: warnings.filterwarnings("ignore", category=tables.exceptions.FlavorWarning) - filename = output_dir / 'agasc_supplement.h5' + filename = output_dir / "agasc_supplement.h5" if multi_process: get_stats = partial(get_agasc_id_stats_pool, batch_size=10) @@ -477,16 +521,17 @@ def do(start, skip = True if agasc_ids is None: - obs_in_time = ((star_obs_catalogs.STARS_OBS['mp_starcat_time'] >= start) - & (star_obs_catalogs.STARS_OBS['mp_starcat_time'] <= stop)) - agasc_ids = sorted(star_obs_catalogs.STARS_OBS[obs_in_time]['agasc_id']) + obs_in_time = (star_obs_catalogs.STARS_OBS["mp_starcat_time"] >= start) & ( + star_obs_catalogs.STARS_OBS["mp_starcat_time"] <= stop + ) + agasc_ids = sorted(star_obs_catalogs.STARS_OBS[obs_in_time]["agasc_id"]) else: - agasc_ids = np.intersect1d(agasc_ids, star_obs_catalogs.STARS_OBS['agasc_id']) + agasc_ids = np.intersect1d(agasc_ids, star_obs_catalogs.STARS_OBS["agasc_id"]) skip = False agasc_ids = np.unique(agasc_ids) stars_obs = star_obs_catalogs.STARS_OBS[ - np.in1d(star_obs_catalogs.STARS_OBS['agasc_id'], agasc_ids) + np.in1d(star_obs_catalogs.STARS_OBS["agasc_id"], agasc_ids) ] # if supplement exists: @@ -497,79 +542,99 @@ def do(start, # - include only the ones with supplement.last_obs_time < than stars_obs.mp_starcat_time obs_status_override = {} if filename.exists(): - with tables.File(filename, 'r') as h5: - if not include_bad and 'bad' in h5.root: - logger.info('Excluding bad stars') - stars_obs = stars_obs[~np.in1d(stars_obs['agasc_id'], h5.root.bad[:]['agasc_id'])] - - if 'obs' in h5.root: + with tables.File(filename, "r") as h5: + if not include_bad and "bad" in h5.root: + logger.info("Excluding bad stars") + stars_obs = stars_obs[ + ~np.in1d(stars_obs["agasc_id"], h5.root.bad[:]["agasc_id"]) + ] + + if "obs" in h5.root: obs_status_override = table.Table(h5.root.obs[:]) obs_status_override.convert_bytestring_to_unicode() obs_status_override = { - (r['mp_starcat_time'], r['agasc_id']): - {'status': r['status'], 'comments': r['comments']} + (r["mp_starcat_time"], r["agasc_id"]): { + "status": r["status"], + "comments": r["comments"], + } for r in obs_status_override } - if 'mags' in h5.root and len(stars_obs): + if "mags" in h5.root and len(stars_obs): outliers_current = h5.root.mags[:] - times = stars_obs[['agasc_id', 'mp_starcat_time']].group_by( - 'agasc_id').groups.aggregate(lambda d: np.max(CxoTime(d)).date) + times = ( + stars_obs[["agasc_id", "mp_starcat_time"]] + .group_by("agasc_id") + .groups.aggregate(lambda d: np.max(CxoTime(d)).date) + ) if len(outliers_current): - times = table.join(times, - table.Table(outliers_current[['agasc_id', 'last_obs_time']]), - join_type='left') + times = table.join( + times, + table.Table(outliers_current[["agasc_id", "last_obs_time"]]), + join_type="left", + ) else: - times['last_obs_time'] = table.MaskedColumn( - np.zeros(len(times), dtype=h5.root.mags.dtype['last_obs_time']), - mask=np.ones(len(times), dtype=bool) + times["last_obs_time"] = table.MaskedColumn( + np.zeros(len(times), dtype=h5.root.mags.dtype["last_obs_time"]), + mask=np.ones(len(times), dtype=bool), ) if skip: - if hasattr(times['last_obs_time'], 'mask'): + if hasattr(times["last_obs_time"], "mask"): # the mask exists if there are stars in stars_obs # that are not in outliers_current - update = (times['last_obs_time'].mask - | ((~times['last_obs_time'].mask) - & (CxoTime(times['mp_starcat_time']).cxcsec - > times['last_obs_time']).data) - ) + update = times["last_obs_time"].mask | ( + (~times["last_obs_time"].mask) + & ( + CxoTime(times["mp_starcat_time"]).cxcsec + > times["last_obs_time"] + ).data + ) else: - update = (CxoTime(times['mp_starcat_time']).cxcsec > times['last_obs_time']) - - stars_obs = stars_obs[np.in1d(stars_obs['agasc_id'], times[update]['agasc_id'])] - agasc_ids = np.sort(np.unique(stars_obs['agasc_id'])) + update = ( + CxoTime(times["mp_starcat_time"]).cxcsec + > times["last_obs_time"] + ) + + stars_obs = stars_obs[ + np.in1d(stars_obs["agasc_id"], times[update]["agasc_id"]) + ] + agasc_ids = np.sort(np.unique(stars_obs["agasc_id"])) if len(update) - np.sum(update): - logger.info(f'Skipping {len(update) - np.sum(update)} ' - f'stars already in the supplement') + logger.info( + f"Skipping {len(update) - np.sum(update)} " + "stars already in the supplement" + ) if len(stars_obs) == 0: - logger.info(f'There are no new observations to process') + logger.info("There are no new observations to process") return # do the processing - logger.info(f'Will process {len(agasc_ids)} stars on {len(stars_obs)} observations') - logger.info(f'from {start} to {stop}') + logger.info(f"Will process {len(agasc_ids)} stars on {len(stars_obs)} observations") + logger.info(f"from {start} to {stop}") if dry_run: return - obs_stats, agasc_stats, fails = \ - get_stats(agasc_ids, tstop=stop, - obs_status_override=obs_status_override, - no_progress=no_progress) + obs_stats, agasc_stats, fails = get_stats( + agasc_ids, + tstop=stop, + obs_status_override=obs_status_override, + no_progress=no_progress, + ) - failed_global = [f for f in fails if not f['agasc_id'] and not f['obsid']] - failed_stars = [f for f in fails if f['agasc_id'] and not f['obsid']] - failed_obs = [f for f in fails if f['obsid']] + failed_global = [f for f in fails if not f["agasc_id"] and not f["obsid"]] + failed_stars = [f for f in fails if f["agasc_id"] and not f["obsid"]] + failed_obs = [f for f in fails if f["obsid"]] msg = ( - f'Got:\n' - f' {0 if obs_stats is None else len(obs_stats)} OBSIDs,' - f' {0 if agasc_stats is None else len(agasc_stats)} stars,' + "Got:\n" + f" {0 if obs_stats is None else len(obs_stats)} OBSIDs," + f" {0 if agasc_stats is None else len(agasc_stats)} stars," ) if failed_obs: - msg += f' {len(failed_obs)} failed observations,' + msg += f" {len(failed_obs)} failed observations," if failed_stars: - msg += f' {len(failed_stars)} failed stars,' + msg += f" {len(failed_stars)} failed stars," if failed_global: - msg += f' {len(failed_global)} global errors' + msg += f" {len(failed_global)} global errors" logger.info(msg) if not output_dir.exists(): @@ -577,100 +642,105 @@ def do(start, update_mag_stats(obs_stats, agasc_stats, fails, output_dir) - obs_status_file = output_dir / 'obs_status.yml' + obs_status_file = output_dir / "obs_status.yml" try: - write_obs_status_yaml([], fails=failed_obs + failed_stars, filename=obs_status_file) + write_obs_status_yaml( + [], fails=failed_obs + failed_stars, filename=obs_status_file + ) except Exception as e: - logger.warning(f'Failed to write {obs_status_file}: {e}') + logger.warning(f"Failed to write {obs_status_file}: {e}") new_stars, updated_stars = update_supplement(agasc_stats, filename=filename) - logger.info(f' {len(new_stars)} new stars, {len(updated_stars)} updated stars') + logger.info(f" {len(new_stars)} new stars, {len(updated_stars)} updated stars") if agasc_stats is not None and len(agasc_stats): if email: try: bad_obs = ( - (obs_stats['mp_starcat_time'] >= start) - & (obs_stats['mp_starcat_time'] < stop) - & ~obs_stats['obs_ok'] + (obs_stats["mp_starcat_time"] >= start) + & (obs_stats["mp_starcat_time"] < stop) + & ~obs_stats["obs_ok"] ) if np.any(bad_obs): msr.email_bad_obs_report(obs_stats[bad_obs], to=email) except Exception as e: - logger.error(f'Error sending email to {email}: {e}') + logger.error(f"Error sending email to {email}: {e}") if report and len(agasc_stats): if report_date is None: report_dir = reports_dir - report_data_file = report_dir / f'report_data.pkl' + report_data_file = report_dir / "report_data.pkl" nav_links = None report_date = CxoTime.now() else: - report_dir = reports_dir / f'{report_date.date[:8]}' - report_data_file = report_dir / f'report_data_{report_date.date[:8]}.pkl' + report_dir = reports_dir / f"{report_date.date[:8]}" + report_data_file = report_dir / f"report_data_{report_date.date[:8]}.pkl" week = time.TimeDelta(7 * u.day) nav_links = { - 'previous': f'../{(report_date - week).date[:8]}/index.html', - 'up': '..', - 'next': f'../{(report_date + week).date[:8]}/index.html' + "previous": f"../{(report_date - week).date[:8]}/index.html", + "up": "..", + "next": f"../{(report_date + week).date[:8]}/index.html", } # If the report data file exists, the arguments for the report from the file are # modified according to the current run. Otherwise, they are created from scratch. if report_data_file.exists(): - with open(report_data_file, 'rb') as fh: + with open(report_data_file, "rb") as fh: report_data = pickle.load(fh) - logger.info(f'Loading existing report data from {report_data_file}') - multi_star_html_args = report_data['args'] + logger.info(f"Loading existing report data from {report_data_file}") + multi_star_html_args = report_data["args"] # arguments for the report are modified here # merge fails: # - from previous run, take fails that were not run just now # - add current fails - multi_star_html_args['fails'] = fails - multi_star_html_args['no_progress'] = no_progress + multi_star_html_args["fails"] = fails + multi_star_html_args["no_progress"] = no_progress else: - sections = [{ - 'id': 'new_stars', - 'title': 'New Stars', - 'stars': new_stars - }, { - 'id': 'updated_stars', - 'title': 'Updated Stars', - 'stars': updated_stars['agasc_id'] if len(updated_stars) else [] - }, { - 'id': 'other_stars', - 'title': 'Other Stars', - 'stars': list(agasc_stats['agasc_id'][ - ~np.in1d(agasc_stats['agasc_id'], new_stars) - & ~np.in1d(agasc_stats['agasc_id'], updated_stars['agasc_id']) - ]) - } + sections = [ + {"id": "new_stars", "title": "New Stars", "stars": new_stars}, + { + "id": "updated_stars", + "title": "Updated Stars", + "stars": updated_stars["agasc_id"] if len(updated_stars) else [], + }, + { + "id": "other_stars", + "title": "Other Stars", + "stars": list( + agasc_stats["agasc_id"][ + ~np.in1d(agasc_stats["agasc_id"], new_stars) + & ~np.in1d( + agasc_stats["agasc_id"], updated_stars["agasc_id"] + ) + ] + ), + }, ] - multi_star_html_args = dict( - filename='index.html', - sections=sections, - updated_stars=updated_stars, - fails=fails, - report_date=report_date.date, - tstart=start, - tstop=stop, - nav_links=nav_links, - include_all_stars=False, - no_progress=no_progress - ) + multi_star_html_args = { + "filename": "index.html", + "sections": sections, + "updated_stars": updated_stars, + "fails": fails, + "report_date": report_date.date, + "tstart": start, + "tstop": stop, + "nav_links": nav_links, + "include_all_stars": False, + "no_progress": no_progress, + } try: report = msr.MagEstimateReport( - agasc_stats=output_dir / 'mag_stats_agasc.fits', - obs_stats=output_dir / 'mag_stats_obsid.fits', - directory=report_dir + agasc_stats=output_dir / "mag_stats_agasc.fits", + obs_stats=output_dir / "mag_stats_obsid.fits", + directory=report_dir, ) report.multi_star_html(**multi_star_html_args) - latest = reports_dir / 'latest' + latest = reports_dir / "latest" if os.path.lexists(latest): logger.debug('Removing existing "latest" symlink') latest.unlink() @@ -678,27 +748,24 @@ def do(start, latest.symlink_to(report_dir.absolute()) except Exception as e: report_dir = output_dir - logger.error(f'Error when creating report: {e}') + logger.error(f"Error when creating report: {e}") exc_type, exc_value, exc_traceback = sys.exc_info() if exc_type is not None: trace = traceback.format_exception(exc_type, exc_value, exc_traceback) for level in trace: for line in level.splitlines(): logger.debug(line) - logger.debug('') + logger.debug("") finally: report_data_file = report_dir / report_data_file.name if not report_dir.exists(): report_dir.mkdir(parents=True) - report_data = { - 'args': multi_star_html_args, - 'directory': report_dir - } - with open(report_data_file, 'wb') as fh: + report_data = {"args": multi_star_html_args, "directory": report_dir} + with open(report_data_file, "wb") as fh: pickle.dump(report_data, fh) - logger.info(f'Report data saved in {report_data_file}') + logger.info(f"Report data saved in {report_data_file}") elif len(agasc_stats) == 0: - logger.info('Nothing to report (no stars)') + logger.info("Nothing to report (no stars)") now = datetime.datetime.now() logger.info(f"done at {now}") diff --git a/agasc/supplement/utils.py b/agasc/supplement/utils.py index 719c7dd3..3f14f583 100644 --- a/agasc/supplement/utils.py +++ b/agasc/supplement/utils.py @@ -1,61 +1,70 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -from pathlib import Path import logging import warnings -import numpy as np +from pathlib import Path -from ska_helpers.utils import lru_cache_timed +import numpy as np import tables +from astropy.table import Table, unique, vstack from cxotime import CxoTime -from astropy.table import Table, vstack, unique - -from ..paths import SUPPLEMENT_FILENAME, default_agasc_dir +from ska_helpers.utils import lru_cache_timed +from agasc.paths import SUPPLEMENT_FILENAME, default_agasc_dir -__all__ = ['get_supplement_table', 'save_version', - 'update_mags_table', 'update_obs_table', 'add_bad_star'] +__all__ = [ + "get_supplement_table", + "save_version", + "update_mags_table", + "update_obs_table", + "add_bad_star", +] -logger = logging.getLogger('agasc.supplement') +logger = logging.getLogger("agasc.supplement") -AGASC_SUPPLEMENT_TABLES = ('mags', 'bad', 'obs', 'last_updated', 'agasc_versions') +AGASC_SUPPLEMENT_TABLES = ("mags", "bad", "obs", "last_updated", "agasc_versions") -BAD_DTYPE = np.dtype([ - ('agasc_id', np.int32), - ('source', np.int16) -]) +BAD_DTYPE = np.dtype([("agasc_id", np.int32), ("source", np.int16)]) -MAGS_DTYPE = np.dtype([ - ('agasc_id', np.int32), - ('mag_aca', np.float32), - ('mag_aca_err', np.float32), - ('last_obs_time', np.float64) -]) +MAGS_DTYPE = np.dtype( + [ + ("agasc_id", np.int32), + ("mag_aca", np.float32), + ("mag_aca_err", np.float32), + ("last_obs_time", np.float64), + ] +) -OBS_DTYPE = np.dtype([ - ('mp_starcat_time', ' 0 else 1 + dat = _get_table(filename, "bad", BAD_DTYPE, create=create) + default = dat["source"].max() + 1 if len(dat) > 0 else 1 - bad = [[agasc_id, default if source is None else source] - for agasc_id, source in bad] + bad = [ + [agasc_id, default if source is None else source] for agasc_id, source in bad + ] bad_star_ids, bad_star_source = np.array(bad).astype(int).T update = False for agasc_id, source in zip(bad_star_ids, bad_star_source): - if agasc_id not in dat['agasc_id']: + if agasc_id not in dat["agasc_id"]: dat.add_row((agasc_id, source)) - logger.info(f'Appending {agasc_id=} with {source=}') + logger.info(f"Appending {agasc_id=} with {source=}") update = True if not update: return - logger.info('') - logger.info('IMPORTANT:') - logger.info('Edit following if source ID is new:') - logger.info(' https://github.com/sot/agasc/wiki/Add-bad-star-to-AGASC-supplement-manually') - logger.info('') - logger.info('The wiki page also includes instructions for test, review, approval') - logger.info('and installation.') + logger.info("") + logger.info("IMPORTANT:") + logger.info("Edit following if source ID is new:") + logger.info( + " https://github.com/sot/agasc/wiki/Add-bad-star-to-AGASC-supplement-manually" + ) + logger.info("") + logger.info("The wiki page also includes instructions for test, review, approval") + logger.info("and installation.") if not dry_run: - dat.write(str(filename), format='hdf5', path='bad', append=True, overwrite=True) - save_version(filename, 'bad') + dat.write(str(filename), format="hdf5", path="bad", append=True, overwrite=True) + save_version(filename, "bad") def update_obs_table(filename, obs, dry_run=False, create=False): @@ -347,10 +376,15 @@ def update_obs_table(filename, obs, dry_run=False, create=False): Create a supplement file if it does not exist """ - update_table(filename, obs, 'obs', OBS_DTYPE, - keys=['agasc_id', 'mp_starcat_time'], - dry_run=dry_run, - create=create) + update_table( + filename, + obs, + "obs", + OBS_DTYPE, + keys=["agasc_id", "mp_starcat_time"], + dry_run=dry_run, + create=create, + ) def update_mags_table(filename, mags, dry_run=False, create=False): @@ -370,7 +404,12 @@ def update_mags_table(filename, mags, dry_run=False, create=False): :param dry_run: bool Do not save the table. """ - update_table(filename, mags, 'mags', MAGS_DTYPE, - keys=['agasc_id'], - dry_run=dry_run, - create=create) + update_table( + filename, + mags, + "mags", + MAGS_DTYPE, + keys=["agasc_id"], + dry_run=dry_run, + create=create, + ) diff --git a/agasc/tests/test_agasc_1.py b/agasc/tests/test_agasc_1.py index b4b1be81..cbfb8f4c 100644 --- a/agasc/tests/test_agasc_1.py +++ b/agasc/tests/test_agasc_1.py @@ -13,38 +13,45 @@ def test_multi_agasc(): - tempdir = tempfile.mkdtemp() # Make two custom agasc files from the miniagasc, using 20 stars from # around the middle of the table with tables.open_file(agasc.default_agasc_file()) as h5: middle = int(len(h5.root.data) // 2) - stars1 = Table(h5.root.data[middle: middle + 20]) - stars1.write(os.path.join(tempdir, 'stars1.h5'), path='data') - stars2 = Table(h5.root.data[middle + 20:middle + 60]) - stars2.write(os.path.join(tempdir, 'stars2.h5'), path='data') + stars1 = Table(h5.root.data[middle : middle + 20]) + stars1.write(os.path.join(tempdir, "stars1.h5"), path="data") + stars2 = Table(h5.root.data[middle + 20 : middle + 60]) + stars2.write(os.path.join(tempdir, "stars2.h5"), path="data") # Fetch all the stars from a custom agasc and make sure we have the right number of stars # with no errors - all_stars2 = agasc.get_agasc_cone(0, 90, radius=180, - agasc_file=os.path.join(tempdir, 'stars2.h5')) + all_stars2 = agasc.get_agasc_cone( + 0, 90, radius=180, agasc_file=os.path.join(tempdir, "stars2.h5") + ) assert len(all_stars2) == len(stars2) # Fetch all the stars from the other custom agasc and do the same. The point of the two files # is to confirm that the caching behavior in agasc doesn't cause problems with fetches - all_stars1 = agasc.get_agasc_cone(0, 90, radius=180, - agasc_file=os.path.join(tempdir, 'stars1.h5')) + all_stars1 = agasc.get_agasc_cone( + 0, 90, radius=180, agasc_file=os.path.join(tempdir, "stars1.h5") + ) assert len(all_stars1) == len(stars1) # Do a position filtered search using the first star in the table as a reference and make sure # we get the same star from the reference agasc. Do this with the stars2 file as this confirms # that we can switch back and forth between files and get the correct content. - cone2 = agasc.get_agasc_cone(all_stars2['RA'][0], all_stars2['DEC'][0], radius=0.000001, - agasc_file=os.path.join(tempdir, 'stars2.h5')) + cone2 = agasc.get_agasc_cone( + all_stars2["RA"][0], + all_stars2["DEC"][0], + radius=0.000001, + agasc_file=os.path.join(tempdir, "stars2.h5"), + ) # And this is a read of the default agasc file after the custom ones so should confirm that # the custom files didn't break that access. - cone2_full = agasc.get_agasc_cone(all_stars2['RA'][0], all_stars2['DEC'][0], radius=0.000001) - assert cone2[0]['AGASC_ID'] == cone2_full[0]['AGASC_ID'] + cone2_full = agasc.get_agasc_cone( + all_stars2["RA"][0], all_stars2["DEC"][0], radius=0.000001 + ) + assert cone2[0]["AGASC_ID"] == cone2_full[0]["AGASC_ID"] # Confirm that there is just one star in this test setup (not a module test, but confirms test # setup is as intended). assert len(cone2_full) == 1 @@ -63,11 +70,11 @@ def test_update_color1_func(): # Fourth now gets COLOR1 = 1.499 because RSV3=1 => good mag # Fifth is still 1.5 because RSV3=0 (no good mag available so still "bad mag") # Sixth now gets COLOR1 = COLOR2 * 0.850 = 2.0 - stars = Table([color1, color2, rsv3], names=['COLOR1', 'COLOR2', 'RSV3']) + stars = Table([color1, color2, rsv3], names=["COLOR1", "COLOR2", "RSV3"]) update_color1_column(stars) - assert np.allclose(stars['COLOR1'], [1.0, 1.0, 1.5, 1.499, 1.5, 2.0]) - assert np.allclose(stars['COLOR2'], color2) + assert np.allclose(stars["COLOR1"], [1.0, 1.0, 1.5, 1.499, 1.5, 2.0]) + assert np.allclose(stars["COLOR2"], color2) def test_update_color1_get_star(): @@ -82,16 +89,16 @@ def test_update_color1_get_star(): """ star = agasc.get_star(981997696) - assert np.isclose(star['COLOR1'], 1.5) + assert np.isclose(star["COLOR1"], 1.5) star = agasc.get_star(759439648) - assert np.isclose(star['COLOR1'], 1.499) + assert np.isclose(star["COLOR1"], 1.499) star = agasc.get_star(981997696, fix_color1=False) - assert np.isclose(star['COLOR1'], 1.5) + assert np.isclose(star["COLOR1"], 1.5) star = agasc.get_star(759439648, fix_color1=False) - assert np.isclose(star['COLOR1'], 1.5) + assert np.isclose(star["COLOR1"], 1.5) def test_update_color1_get_agasc_cone(): @@ -100,14 +107,14 @@ def test_update_color1_get_agasc_cone(): """ ra, dec = 323.22831196, -13.11621348 stars = agasc.get_agasc_cone(ra, dec, 0.2) - stars.add_index('AGASC_ID') - assert np.isclose(stars.loc[759960152]['COLOR1'], 1.60055, rtol=0, atol=0.0005) - assert np.isclose(stars.loc[759439648]['COLOR1'], 1.499, rtol=0, atol=0.0005) + stars.add_index("AGASC_ID") + assert np.isclose(stars.loc[759960152]["COLOR1"], 1.60055, rtol=0, atol=0.0005) + assert np.isclose(stars.loc[759439648]["COLOR1"], 1.499, rtol=0, atol=0.0005) stars = agasc.get_agasc_cone(ra, dec, 0.2, fix_color1=False) - stars.add_index('AGASC_ID') - assert np.isclose(stars.loc[759960152]['COLOR1'], 1.5, rtol=0, atol=0.0005) - assert np.isclose(stars.loc[759439648]['COLOR1'], 1.5, rtol=0, atol=0.0005) + stars.add_index("AGASC_ID") + assert np.isclose(stars.loc[759960152]["COLOR1"], 1.5, rtol=0, atol=0.0005) + assert np.isclose(stars.loc[759439648]["COLOR1"], 1.5, rtol=0, atol=0.0005) def test_get_agasc_filename(tmp_path, monkeypatch): diff --git a/agasc/tests/test_agasc_2.py b/agasc/tests/test_agasc_2.py index 53c6b2ff..8d5763b6 100644 --- a/agasc/tests/test_agasc_2.py +++ b/agasc/tests/test_agasc_2.py @@ -25,35 +25,37 @@ from pathlib import Path import numpy as np +import pytest import Ska.Shell from astropy.io import ascii -from astropy.table import Table, Row -import pytest +from astropy.table import Row, Table import agasc -os.environ[agasc.SUPPLEMENT_ENABLED_ENV] = 'False' +os.environ[agasc.SUPPLEMENT_ENABLED_ENV] = "False" # See if we can get to ASCDS environment and mp_get_agasc try: - ascrc_file = '{}/.ascrc'.format(os.environ['HOME']) + ascrc_file = "{}/.ascrc".format(os.environ["HOME"]) assert os.path.exists(ascrc_file) - ascds_env = Ska.Shell.getenv('source {} -r release'.format(ascrc_file), shell='tcsh') - assert 'ASCDS_BIN' in ascds_env - cmd = 'mp_get_agasc -r 10 -d 20 -w 0.01' + ascds_env = Ska.Shell.getenv( + "source {} -r release".format(ascrc_file), shell="tcsh" + ) + assert "ASCDS_BIN" in ascds_env + cmd = "mp_get_agasc -r 10 -d 20 -w 0.01" # Run the command to check for bad status (which will throw exception) - Ska.Shell.run_shell(cmd, shell='bash', env=ascds_env) - match = re.search(r'agasc([p0-9]+)', ascds_env['ASCDS_AGASC']) + Ska.Shell.run_shell(cmd, shell="bash", env=ascds_env) + match = re.search(r"agasc([p0-9]+)", ascds_env["ASCDS_AGASC"]) DS_AGASC_VERSION = match.group(1) except Exception: ascds_env = None DS_AGASC_VERSION = None -NO_MAGS_IN_SUPPLEMENT = not any(agasc.get_supplement_table('mags', as_dict=True)) -NO_OBS_IN_SUPPLEMENT = not any(agasc.get_supplement_table('obs', as_dict=True)) +NO_MAGS_IN_SUPPLEMENT = not any(agasc.get_supplement_table("mags", as_dict=True)) +NO_OBS_IN_SUPPLEMENT = not any(agasc.get_supplement_table("obs", as_dict=True)) -HAS_KSH = os.path.exists('/bin/ksh') # dependency of mp_get_agascid +HAS_KSH = os.path.exists("/bin/ksh") # dependency of mp_get_agascid AGASC_COL_DESCR = """ AGASC_ID - a unique long integer used for identification. @@ -116,50 +118,50 @@ TEST_ASCDS = False else: try: - agasc.get_agasc_filename('miniagasc_*', version=DS_AGASC_VERSION) + agasc.get_agasc_filename("miniagasc_*", version=DS_AGASC_VERSION) except FileNotFoundError: TEST_ASCDS = False else: TEST_ASCDS = True # Latest full release of miniagasc -MINIAGASC = agasc.get_agasc_filename('miniagasc_*') +MINIAGASC = agasc.get_agasc_filename("miniagasc_*") def get_ds_agasc_cone(ra, dec): - cmd = 'mp_get_agasc -r {!r} -d {!r} -w {!r}'.format(ra, dec, TEST_RADIUS) + cmd = "mp_get_agasc -r {!r} -d {!r} -w {!r}".format(ra, dec, TEST_RADIUS) lines = Ska.Shell.tcsh(cmd, env=ascds_env) dat = ascii.read(lines, Reader=ascii.NoHeader, names=AGASC_COLNAMES) - ok1 = agasc.sphere_dist(ra, dec, dat['RA'], dat['DEC']) <= TEST_RADIUS - ok2 = dat['MAG_ACA'] - 3.0 * dat['MAG_ACA_ERR'] / 100.0 < 11.5 + ok1 = agasc.sphere_dist(ra, dec, dat["RA"], dat["DEC"]) <= TEST_RADIUS + ok2 = dat["MAG_ACA"] - 3.0 * dat["MAG_ACA_ERR"] / 100.0 < 11.5 dat = dat[ok1 & ok2] - if os.environ.get('WRITE_AGASC_TEST_FILES'): + if os.environ.get("WRITE_AGASC_TEST_FILES"): version = DS_AGASC_VERSION test_file = get_test_file(ra, dec, version) - print(f'\nWriting {test_file} based on mp_get_agasc\n') - dat.write(test_file, format='fits') + print(f"\nWriting {test_file} based on mp_get_agasc\n") + dat.write(test_file, format="fits") return dat def get_test_file(ra, dec, version): - return TEST_DIR / 'data' / f'ref_ra_{ra}_dec_{dec}_{version}.fits.gz' + return TEST_DIR / "data" / f"ref_ra_{ra}_dec_{dec}_{version}.fits.gz" -def get_reference_agasc_values(ra, dec, version='1p7'): +def get_reference_agasc_values(ra, dec, version="1p7"): dat = Table.read(get_test_file(ra, dec, version)) return dat -RAS = np.hstack([0., 180., 0.1, 180., 275.36]) +RAS = np.hstack([0.0, 180.0, 0.1, 180.0, 275.36]) DECS = np.hstack([89.9, -89.9, 0.0, 0.0, 8.09]) # The (275.36, 8.09) coordinate fails unless date=2000:001 due to # mp_get_agasc not accounting for proper motion. -@pytest.mark.parametrize("version", ['1p6', '1p7']) +@pytest.mark.parametrize("version", ["1p6", "1p7"]) @pytest.mark.parametrize("ra,dec", list(zip(RAS, DECS))) def test_agasc_conesearch(ra, dec, version): """ @@ -169,24 +171,24 @@ def test_agasc_conesearch(ra, dec, version): try: ref_stars = get_reference_agasc_values(ra, dec, version=version) except FileNotFoundError: - if os.environ.get('WRITE_AGASC_TEST_FILES'): + if os.environ.get("WRITE_AGASC_TEST_FILES"): ref_stars = agasc.get_agasc_cone( ra, dec, radius=TEST_RADIUS, - agasc_file=agasc.get_agasc_filename('miniagasc_*', version=version), - date='2000:001', - fix_color1=False + agasc_file=agasc.get_agasc_filename("miniagasc_*", version=version), + date="2000:001", + fix_color1=False, ) test_file = get_test_file(ra, dec, version) - print(f'\nWriting {test_file} based on miniagasc\n') - ref_stars.write(test_file, format='fits') - pytest.skip('Reference data unavailable') + print(f"\nWriting {test_file} based on miniagasc\n") + ref_stars.write(test_file, format="fits") + pytest.skip("Reference data unavailable") else: _test_agasc(ra, dec, ref_stars, version) -@pytest.mark.skipif('not TEST_ASCDS') +@pytest.mark.skipif("not TEST_ASCDS") @pytest.mark.parametrize("ra,dec", list(zip(RAS, DECS))) def test_against_ds_agasc(ra, dec): """ @@ -197,61 +199,65 @@ def test_against_ds_agasc(ra, dec): _test_agasc(ra, dec, ref_stars, version=DS_AGASC_VERSION) -def _test_agasc(ra, dec, ref_stars, version='1p7'): - agasc_file = agasc.get_agasc_filename('miniagasc_*', version=version) - stars1 = agasc.get_agasc_cone(ra, dec, radius=TEST_RADIUS, - agasc_file=agasc_file, - date='2000:001', fix_color1=False) - stars1.sort('AGASC_ID') +def _test_agasc(ra, dec, ref_stars, version="1p7"): + agasc_file = agasc.get_agasc_filename("miniagasc_*", version=version) + stars1 = agasc.get_agasc_cone( + ra, + dec, + radius=TEST_RADIUS, + agasc_file=agasc_file, + date="2000:001", + fix_color1=False, + ) + stars1.sort("AGASC_ID") stars2 = ref_stars.copy() - stars2.sort('AGASC_ID') + stars2.sort("AGASC_ID") # First make sure that the common stars are identical - agasc_ids = set(stars1['AGASC_ID']).intersection(set(stars2['AGASC_ID'])) + agasc_ids = set(stars1["AGASC_ID"]).intersection(set(stars2["AGASC_ID"])) for agasc_id in agasc_ids: - star1 = stars1[np.searchsorted(stars1['AGASC_ID'], agasc_id)] - star2 = stars2[np.searchsorted(stars2['AGASC_ID'], agasc_id)] + star1 = stars1[np.searchsorted(stars1["AGASC_ID"], agasc_id)] + star2 = stars2[np.searchsorted(stars2["AGASC_ID"], agasc_id)] for colname in AGASC_COLNAMES: - if star1[colname].dtype.kind == 'f': + if star1[colname].dtype.kind == "f": assert np.all(np.abs(star1[colname] - star2[colname]) < 1e-4) else: assert star1[colname] == star2[colname] # Second make sure that the non-common stars are all just at the edge # of the faint mag limit, due to precision loss in mp_get_agasc - for s1, s2 in ((stars1, stars2), - (stars2, stars1)): - mm1 = set(s1['AGASC_ID']) - set(s2['AGASC_ID']) + for s1, s2 in ((stars1, stars2), (stars2, stars1)): + mm1 = set(s1["AGASC_ID"]) - set(s2["AGASC_ID"]) for agasc_id in mm1: - idx = np.flatnonzero(s1['AGASC_ID'] == agasc_id)[0] + idx = np.flatnonzero(s1["AGASC_ID"] == agasc_id)[0] star = s1[idx] bad_is_star1 = s1 is stars1 - rad = agasc.sphere_dist(ra, dec, star['RA'], star['DEC']) - adj_mag = star['MAG_ACA'] - 3.0 * star['MAG_ACA_ERR'] / 100.0 + rad = agasc.sphere_dist(ra, dec, star["RA"], star["DEC"]) + adj_mag = star["MAG_ACA"] - 3.0 * star["MAG_ACA_ERR"] / 100.0 if adj_mag < 11.5 * 0.99: # Allow for loss of precision in output of mp_get_agasc - print('Bad star', agasc_id, rad, adj_mag, bad_is_star1) + print("Bad star", agasc_id, rad, adj_mag, bad_is_star1) assert False def test_basic(): star = agasc.get_star(1180612288) # High-PM star - assert np.isclose(star['RA'], 219.9053773) - assert np.isclose(star['DEC'], -60.8371572) + assert np.isclose(star["RA"], 219.9053773) + assert np.isclose(star["DEC"], -60.8371572) - stars = agasc.get_agasc_cone(star['RA'], star['DEC'], 0.5) - stars.sort('MAG_ACA') + stars = agasc.get_agasc_cone(star["RA"], star["DEC"], 0.5) + stars.sort("MAG_ACA") agasc_ids = [1180612176, 1180612296, 1180612184, 1180612288, 1180612192] mags = [-0.663, -0.576, -0.373, 0.53, 0.667] - assert np.allclose(stars['AGASC_ID'][:5], agasc_ids) - assert np.allclose(stars['MAG_ACA'][:5], mags) + assert np.allclose(stars["AGASC_ID"][:5], agasc_ids) + assert np.allclose(stars["MAG_ACA"][:5], mags) def test_get_stars1(): # First check that get_stars() gives the same as get_star for the scalar case - star1 = agasc.get_star(1180612288, date='2019:001') - star2 = agasc.get_stars(1180612288, dates='2019:001') + star1 = agasc.get_star(1180612288, date="2019:001") + star2 = agasc.get_stars(1180612288, dates="2019:001") assert isinstance(star2, Row) for name in star1.colnames: assert star1[name] == star2[name] @@ -259,9 +265,9 @@ def test_get_stars1(): def test_get_stars2(): """get_stars() broadcasts ids""" - star0 = agasc.get_star(1180612288, date='2010:001') - star1 = agasc.get_star(1180612288, date='2019:001') - star2 = agasc.get_stars(1180612288, dates=['2010:001', '2019:001']) + star0 = agasc.get_star(1180612288, date="2010:001") + star1 = agasc.get_star(1180612288, date="2019:001") + star2 = agasc.get_stars(1180612288, dates=["2010:001", "2019:001"]) for name in star1.colnames: assert star0[name] == star2[0][name] @@ -272,19 +278,22 @@ def test_get_stars3(): agasc_ids = [1180612176, 1180612296, 1180612184, 1180612288, 1180612192] mags = [-0.663, -0.576, -0.373, 0.53, 0.667] stars = agasc.get_stars(agasc_ids) - assert np.allclose(stars['MAG_ACA'], mags) + assert np.allclose(stars["MAG_ACA"], mags) def test_get_stars_many(): """Test get_stars() with at least GET_STARS_METHOD_THRESHOLD (5000) stars""" - from .. import agasc + from agasc import agasc + stars = agasc.get_agasc_cone(0, 0, radius=0.5) - agasc_ids = stars['AGASC_ID'] - stars1 = agasc.get_stars(agasc_ids, dates='2020:001') # read_where method - stars2 = agasc.get_stars(agasc_ids, dates='2020:001', method_threshold=1) # read entire AGASC + agasc_ids = stars["AGASC_ID"] + stars1 = agasc.get_stars(agasc_ids, dates="2020:001") # read_where method + stars2 = agasc.get_stars( + agasc_ids, dates="2020:001", method_threshold=1 + ) # read entire AGASC - assert stars1.get_stars_method == 'tables_read_where' - assert stars2.get_stars_method == 'read_entire_agasc' + assert stars1.get_stars_method == "tables_read_where" + assert stars2.get_stars_method == "read_entire_agasc" assert stars1.colnames == stars2.colnames for name in stars1.colnames: @@ -292,8 +301,8 @@ def test_get_stars_many(): def test_float16(): - stars = agasc.get_agasc_cone(np.float16(219.90279), np.float16(-60.83358), .015) - assert stars['AGASC_ID'][0] == 1180612176 + stars = agasc.get_agasc_cone(np.float16(219.90279), np.float16(-60.83358), 0.015) + assert stars["AGASC_ID"][0] == 1180612176 def test_proper_motion(): @@ -301,67 +310,81 @@ def test_proper_motion(): Test that the filtering in get_agasc_cone correctly expands the initial search radius and then does final filtering using PM-corrected positions. """ - star = agasc.get_star(1180612288, date='2017:001') # High-PM star + star = agasc.get_star(1180612288, date="2017:001") # High-PM star radius = 2.0 / 3600 # 5 arcsec - stars = agasc.get_agasc_cone(star['RA'], star['DEC'], radius, date='2000:001') + stars = agasc.get_agasc_cone(star["RA"], star["DEC"], radius, date="2000:001") assert len(stars) == 1 - stars = agasc.get_agasc_cone(star['RA'], star['DEC'], radius, date='2017:001') + stars = agasc.get_agasc_cone(star["RA"], star["DEC"], radius, date="2017:001") assert len(stars) == 0 - stars = agasc.get_agasc_cone(star['RA'], star['DEC'], radius, date='2017:001', - pm_filter=False) + stars = agasc.get_agasc_cone( + star["RA"], star["DEC"], radius, date="2017:001", pm_filter=False + ) assert len(stars) == 1 - stars = agasc.get_agasc_cone(star['RA_PMCORR'], star['DEC_PMCORR'], radius, date='2017:001') + stars = agasc.get_agasc_cone( + star["RA_PMCORR"], star["DEC_PMCORR"], radius, date="2017:001" + ) assert len(stars) == 1 - stars = agasc.get_agasc_cone(star['RA_PMCORR'], star['DEC_PMCORR'], radius, date='2017:001', - pm_filter=False) + stars = agasc.get_agasc_cone( + star["RA_PMCORR"], star["DEC_PMCORR"], radius, date="2017:001", pm_filter=False + ) assert len(stars) == 0 @pytest.mark.parametrize( "agasc_id,date,ra_pmcorr,dec_pmcorr,label", - [(1180612288, '2020:001', 219.864331, -60.831868, "high proper motion, epoch 2000"), - (198451217, '2020:001', 247.892206, 19.276605, "epoch 1982 star"), - (501219465, '2020:001', 166.998976, 52.822080, "epoch 1984 star")]) + [ + ( + 1180612288, + "2020:001", + 219.864331, + -60.831868, + "high proper motion, epoch 2000", + ), + (198451217, "2020:001", 247.892206, 19.276605, "epoch 1982 star"), + (501219465, "2020:001", 166.998976, 52.822080, "epoch 1984 star"), + ], +) def test_add_pmcorr_is_consistent(agasc_id, date, ra_pmcorr, dec_pmcorr, label): """ Check that the proper-motion corrected position is consistent reference/regress values. """ star = agasc.get_star(agasc_id, date=date) - assert np.isclose(star['RA_PMCORR'], ra_pmcorr, rtol=0, atol=1e-5) - assert np.isclose(star['DEC_PMCORR'], dec_pmcorr, rtol=0, atol=1e-5) + assert np.isclose(star["RA_PMCORR"], ra_pmcorr, rtol=0, atol=1e-5) + assert np.isclose(star["DEC_PMCORR"], dec_pmcorr, rtol=0, atol=1e-5) def mp_get_agascid(agasc_id): - cmd = 'mp_get_agascid {!r}'.format(agasc_id) + cmd = "mp_get_agascid {!r}".format(agasc_id) lines = Ska.Shell.tcsh(cmd, env=ascds_env) - lines = [line for line in lines if re.match(r'^\s*\d', line)] + lines = [line for line in lines if re.match(r"^\s*\d", line)] dat = ascii.read(lines, Reader=ascii.NoHeader, names=AGASC_COLNAMES) return dat -@pytest.mark.skipif('not HAS_KSH') -@pytest.mark.skipif('not TEST_ASCDS') +@pytest.mark.skipif("not HAS_KSH") +@pytest.mark.skipif("not TEST_ASCDS") @pytest.mark.parametrize("ra,dec", list(zip(RAS[:2], DECS[:2]))) def test_agasc_id(ra, dec, radius=0.2, nstar_limit=5): agasc_file = agasc.get_agasc_filename("miniagasc_*", version=DS_AGASC_VERSION) - print('ra, dec =', ra, dec) - stars = agasc.get_agasc_cone(ra, dec, radius=radius, agasc_file=agasc_file, - fix_color1=False) - stars.sort('AGASC_ID') + print("ra, dec =", ra, dec) + stars = agasc.get_agasc_cone( + ra, dec, radius=radius, agasc_file=agasc_file, fix_color1=False + ) + stars.sort("AGASC_ID") - for agasc_id in stars['AGASC_ID'][:nstar_limit]: - print(' agasc_id =', agasc_id) + for agasc_id in stars["AGASC_ID"][:nstar_limit]: + print(" agasc_id =", agasc_id) star1 = agasc.get_star(agasc_id, agasc_file=agasc_file, fix_color1=False) star2 = mp_get_agascid(agasc_id) for colname in AGASC_COLNAMES: - if star1[colname].dtype.kind == 'f': + if star1[colname].dtype.kind == "f": assert np.all(np.allclose(star1[colname], star2[colname])) else: assert star1[colname] == star2[colname] @@ -372,85 +395,85 @@ def test_proseco_agasc_1p7(): mini_file = agasc.get_agasc_filename("miniagasc_*", version="1p7") # Stars looking toward galactic center (dense!) - p_stars = agasc.get_agasc_cone(-266, -29, 3, - agasc_file=proseco_file, date='2000:001') - m_stars = agasc.get_agasc_cone(-266, -29, 3, - agasc_file=mini_file, date='2000:001') + p_stars = agasc.get_agasc_cone( + -266, -29, 3, agasc_file=proseco_file, date="2000:001" + ) + m_stars = agasc.get_agasc_cone(-266, -29, 3, agasc_file=mini_file, date="2000:001") # Every miniagasc_1p7 star is in proseco_agasc_1p7 - m_ids = m_stars['AGASC_ID'] - p_ids = p_stars['AGASC_ID'] + m_ids = m_stars["AGASC_ID"] + p_ids = p_stars["AGASC_ID"] assert set(m_ids) < set(p_ids) # Values are exactly the same p_id_map = {p_ids[idx]: idx for idx in np.arange(len(p_ids))} for m_star in m_stars: - m_id = m_star['AGASC_ID'] + m_id = m_star["AGASC_ID"] p_star = p_stars[p_id_map[m_id]] for name in p_star.colnames: assert p_star[name] == m_star[name] -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") def test_supplement_get_agasc_cone(): ra, dec = 282.53, -0.38 # Obsid 22429 with a couple of color1=1.5 stars stars1 = agasc.get_agasc_cone( - ra, dec, date='2021:001', agasc_file=MINIAGASC, use_supplement=False + ra, dec, date="2021:001", agasc_file=MINIAGASC, use_supplement=False ) stars2 = agasc.get_agasc_cone( - ra, dec, date='2021:001', agasc_file=MINIAGASC, use_supplement=True + ra, dec, date="2021:001", agasc_file=MINIAGASC, use_supplement=True ) - ok = stars2['MAG_CATID'] == agasc.MAG_CATID_SUPPLEMENT + ok = stars2["MAG_CATID"] == agasc.MAG_CATID_SUPPLEMENT - change_names = ['MAG_CATID', 'COLOR1', 'MAG_ACA', 'MAG_ACA_ERR'] + change_names = ["MAG_CATID", "COLOR1", "MAG_ACA", "MAG_ACA_ERR"] for name in set(stars1.colnames) - set(change_names): assert np.all(stars1[name] == stars2[name]) - assert not np.any(stars1['MAG_CATID'] == agasc.MAG_CATID_SUPPLEMENT) + assert not np.any(stars1["MAG_CATID"] == agasc.MAG_CATID_SUPPLEMENT) # At least 35 stars in this field observed assert np.count_nonzero(ok) >= 35 # At least 7 color=1.5 stars converted to 1.49 (note: total of 23 color=1.5 # stars in this field) - assert np.count_nonzero(stars1['COLOR1'][ok] == 1.49) == 0 - assert np.count_nonzero(stars1['COLOR1'][ok] == 1.50) >= 7 - assert np.count_nonzero(stars2['COLOR1'][ok] == 1.49) >= 7 - assert np.count_nonzero(stars2['COLOR1'][ok] == 1.50) == 0 + assert np.count_nonzero(stars1["COLOR1"][ok] == 1.49) == 0 + assert np.count_nonzero(stars1["COLOR1"][ok] == 1.50) >= 7 + assert np.count_nonzero(stars2["COLOR1"][ok] == 1.49) >= 7 + assert np.count_nonzero(stars2["COLOR1"][ok] == 1.50) == 0 # For the stars that have updated data for the supplement, confirm they don't # have all the same values for MAG_ACA_ERR as the catalog values. Note this # is an integer column. - assert np.any(stars2['MAG_ACA_ERR'][ok] != stars1['MAG_ACA_ERR'][ok]) + assert np.any(stars2["MAG_ACA_ERR"][ok] != stars1["MAG_ACA_ERR"][ok]) # Similarly, in this set the stars with updated magnitudes are different from # the catalog values. - assert np.all(stars2['MAG_ACA'][ok] != stars1['MAG_ACA'][ok]) + assert np.all(stars2["MAG_ACA"][ok] != stars1["MAG_ACA"][ok]) - assert np.all(stars2['MAG_ACA_ERR'][~ok] == stars1['MAG_ACA_ERR'][~ok]) - assert np.all(stars2['MAG_ACA'][~ok] == stars1['MAG_ACA'][~ok]) + assert np.all(stars2["MAG_ACA_ERR"][~ok] == stars1["MAG_ACA_ERR"][~ok]) + assert np.all(stars2["MAG_ACA"][~ok] == stars1["MAG_ACA"][~ok]) -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") def test_supplement_get_star(): agasc_id = 58720672 # Also checks that the default is False given the os.environ override for # this test file. star1 = agasc.get_star(agasc_id, agasc_file=MINIAGASC) star2 = agasc.get_star(agasc_id, agasc_file=MINIAGASC, use_supplement=True) - assert star1['MAG_CATID'] != agasc.MAG_CATID_SUPPLEMENT - assert star2['MAG_CATID'] == agasc.MAG_CATID_SUPPLEMENT + assert star1["MAG_CATID"] != agasc.MAG_CATID_SUPPLEMENT + assert star2["MAG_CATID"] == agasc.MAG_CATID_SUPPLEMENT - assert star1['AGASC_ID'] == star2['AGASC_ID'] + assert star1["AGASC_ID"] == star2["AGASC_ID"] - assert np.isclose(star1['COLOR1'], 1.50) - assert np.isclose(star2['COLOR1'], 1.49) + assert np.isclose(star1["COLOR1"], 1.50) + assert np.isclose(star2["COLOR1"], 1.49) - assert star2['MAG_ACA'] != star1['MAG_ACA'] - assert star2['MAG_ACA_ERR'] != star1['MAG_ACA_ERR'] + assert star2["MAG_ACA"] != star1["MAG_ACA"] + assert star2["MAG_ACA_ERR"] != star1["MAG_ACA_ERR"] -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") def test_supplement_get_star_disable_context_manager(): """Test that disable_supplement_mags context manager works. @@ -458,97 +481,103 @@ def test_supplement_get_star_disable_context_manager(): tests. """ agasc_id = 58720672 - star1 = agasc.get_star(agasc_id, date='2020:001', use_supplement=True) + star1 = agasc.get_star(agasc_id, date="2020:001", use_supplement=True) with agasc.set_supplement_enabled(True): - star2 = agasc.get_star(agasc_id, date='2020:001') + star2 = agasc.get_star(agasc_id, date="2020:001") for name in star1.colnames: assert star1[name] == star2[name] -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") @agasc.set_supplement_enabled(True) def test_supplement_get_star_disable_decorator(): """Test that disable_supplement_mags context manager works""" agasc_id = 58720672 - star1 = agasc.get_star(agasc_id, date='2020:001') - star2 = agasc.get_star(agasc_id, date='2020:001', use_supplement=True) + star1 = agasc.get_star(agasc_id, date="2020:001") + star2 = agasc.get_star(agasc_id, date="2020:001", use_supplement=True) for name in star1.colnames: assert star1[name] == star2[name] -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") def test_supplement_get_stars(): agasc_ids = [58720672, 670303120] star1 = agasc.get_stars(agasc_ids, agasc_file=MINIAGASC) star2 = agasc.get_stars(agasc_ids, agasc_file=MINIAGASC, use_supplement=True) - assert np.all(star1['MAG_CATID'] != agasc.MAG_CATID_SUPPLEMENT) - assert np.all(star2['MAG_CATID'] == agasc.MAG_CATID_SUPPLEMENT) + assert np.all(star1["MAG_CATID"] != agasc.MAG_CATID_SUPPLEMENT) + assert np.all(star2["MAG_CATID"] == agasc.MAG_CATID_SUPPLEMENT) - assert np.all(star1['AGASC_ID'] == star2['AGASC_ID']) + assert np.all(star1["AGASC_ID"] == star2["AGASC_ID"]) - assert np.allclose(star1['COLOR1'], [1.5, 0.24395067]) - assert np.allclose(star2['COLOR1'], [1.49, 0.24395067]) + assert np.allclose(star1["COLOR1"], [1.5, 0.24395067]) + assert np.allclose(star2["COLOR1"], [1.49, 0.24395067]) - assert np.all(star2['MAG_ACA'] != star1['MAG_ACA']) + assert np.all(star2["MAG_ACA"] != star1["MAG_ACA"]) def test_get_supplement_table_bad(): - bad = agasc.get_supplement_table('bad') + bad = agasc.get_supplement_table("bad") assert isinstance(bad, Table) - assert bad.colnames == ['agasc_id', 'source'] + assert bad.colnames == ["agasc_id", "source"] assert len(bad) > 3300 - assert 797847184 in bad['agasc_id'] + assert 797847184 in bad["agasc_id"] def test_get_supplement_table_bad_dict(): - bad = agasc.get_supplement_table('bad', as_dict=True) + bad = agasc.get_supplement_table("bad", as_dict=True) assert isinstance(bad, dict) assert len(bad) > 3300 - assert bad[797847184] == {'source': 1} + assert bad[797847184] == {"source": 1} @agasc.set_supplement_enabled(True) def test_get_bad_star_with_supplement(): agasc_id = 797847184 star = agasc.get_star(agasc_id, use_supplement=True) - assert star['CLASS'] == agasc.BAD_CLASS_SUPPLEMENT + assert star["CLASS"] == agasc.BAD_CLASS_SUPPLEMENT def test_bad_agasc_supplement_env_var(): try: - os.environ[agasc.SUPPLEMENT_ENABLED_ENV] = 'asdfasdf' - with pytest.raises(ValueError, match='env var must be either'): + os.environ[agasc.SUPPLEMENT_ENABLED_ENV] = "asdfasdf" + with pytest.raises(ValueError, match="env var must be either"): agasc.get_star(797847184) finally: - os.environ[agasc.SUPPLEMENT_ENABLED_ENV] = 'False' + os.environ[agasc.SUPPLEMENT_ENABLED_ENV] = "False" -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") def test_get_supplement_table_mags(): - mags = agasc.get_supplement_table('mags') + mags = agasc.get_supplement_table("mags") assert isinstance(mags, Table) - assert 131736 in mags['agasc_id'] + assert 131736 in mags["agasc_id"] assert len(mags) > 80000 - assert mags.colnames == ['agasc_id', 'mag_aca', 'mag_aca_err', 'last_obs_time'] + assert mags.colnames == ["agasc_id", "mag_aca", "mag_aca_err", "last_obs_time"] -@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason='no mags in supplement') +@pytest.mark.skipif(NO_MAGS_IN_SUPPLEMENT, reason="no mags in supplement") def test_get_supplement_table_mags_dict(): - mags = agasc.get_supplement_table('mags', as_dict=True) + mags = agasc.get_supplement_table("mags", as_dict=True) assert isinstance(mags, dict) assert 131736 in mags assert len(mags) > 80000 - assert list(mags[131736].keys()) == ['mag_aca', 'mag_aca_err', 'last_obs_time'] + assert list(mags[131736].keys()) == ["mag_aca", "mag_aca_err", "last_obs_time"] -@pytest.mark.skipif(NO_OBS_IN_SUPPLEMENT, reason='no obs in supplement') +@pytest.mark.skipif(NO_OBS_IN_SUPPLEMENT, reason="no obs in supplement") def test_get_supplement_table_obs(): - obs = agasc.get_supplement_table('obs') + obs = agasc.get_supplement_table("obs") assert isinstance(obs, Table) - assert obs.colnames == ['mp_starcat_time', 'agasc_id', 'obsid', 'status', 'comments'] + assert obs.colnames == [ + "mp_starcat_time", + "agasc_id", + "obsid", + "status", + "comments", + ] -@pytest.mark.skipif(NO_OBS_IN_SUPPLEMENT, reason='no obs in supplement') +@pytest.mark.skipif(NO_OBS_IN_SUPPLEMENT, reason="no obs in supplement") def test_get_supplement_table_obs_dict(): - obs = agasc.get_supplement_table('obs', as_dict=True) + obs = agasc.get_supplement_table("obs", as_dict=True) assert isinstance(obs, dict) diff --git a/agasc/tests/test_agasc_healpix.py b/agasc/tests/test_agasc_healpix.py index 023faa3c..1a67173d 100644 --- a/agasc/tests/test_agasc_healpix.py +++ b/agasc/tests/test_agasc_healpix.py @@ -9,7 +9,6 @@ import agasc from agasc.healpix import get_healpix_info - AGASC_FILES = {} for root, version in [("proseco_agasc_*", "1p8"), ("agasc_healpix_*", "1p7")]: try: diff --git a/agasc/tests/test_obs_status.py b/agasc/tests/test_obs_status.py index a9998e0e..a303cff3 100644 --- a/agasc/tests/test_obs_status.py +++ b/agasc/tests/test_obs_status.py @@ -1,70 +1,112 @@ -import pytest +import builtins import io import os -import numpy as np import pathlib -from astropy import table + +import numpy as np +import pytest import tables -import builtins +from astropy import table -from agasc.supplement.magnitudes import mag_estimate, star_obs_catalogs from agasc.scripts import update_supplement -from agasc.supplement.utils import OBS_DTYPE, BAD_DTYPE +from agasc.supplement.magnitudes import mag_estimate, star_obs_catalogs +from agasc.supplement.utils import BAD_DTYPE, OBS_DTYPE -TEST_DATA_DIR = pathlib.Path(__file__).parent / 'data' +TEST_DATA_DIR = pathlib.Path(__file__).parent / "data" # for the purposes of the supplement tables # star_obs_catalogs.STARS_OBS is used to determine all AGASC IDs in an observation # or the last observation time of a given AGASC_ID. # this array is used to monkey-patch star_obs_catalogs.STARS_OBS -STARS_OBS = np.array([ - (56314, 114950168, '2010:110:14:57:43.442'), (56314, 114950584, '2010:110:14:57:43.442'), - (56314, 114952056, '2010:110:14:57:43.442'), (56314, 114952792, '2010:110:14:57:43.442'), - (56314, 114952824, '2010:110:14:57:43.442'), (56314, 114955056, '2010:110:14:57:43.442'), - (56314, 114956608, '2010:110:14:57:43.442'), (56314, 115347520, '2010:110:14:57:43.442'), - (56312, 357045496, '2010:110:16:59:51.399'), (56312, 357049064, '2010:110:16:59:51.399'), - (56312, 357051640, '2010:110:16:59:51.399'), (56312, 357054680, '2010:110:16:59:51.399'), - (56312, 358220224, '2010:110:16:59:51.399'), (56312, 358222256, '2010:110:16:59:51.399'), - (56312, 358224400, '2010:110:16:59:51.399'), (56312, 358757768, '2010:110:16:59:51.399'), - (56313, 441853632, '2010:110:15:07:49.876'), (56313, 441854760, '2010:110:15:07:49.876'), - (56313, 441855776, '2010:110:15:07:49.876'), (56313, 441856032, '2010:110:15:07:49.876'), - (56313, 441856400, '2010:110:15:07:49.876'), (56313, 441980072, '2010:110:15:07:49.876'), - (56313, 491391592, '2010:110:15:07:49.876'), (56313, 491394504, '2010:110:15:07:49.876'), - (56311, 563087864, '2010:110:18:59:46.600'), (56311, 563088952, '2010:110:18:59:46.600'), - (56311, 563089432, '2010:110:18:59:46.600'), (56311, 563091784, '2010:110:18:59:46.600'), - (56311, 563092520, '2010:110:18:59:46.600'), (56311, 563612488, '2010:110:18:59:46.600'), - (56311, 563612792, '2010:110:18:59:46.600'), (56311, 563617352, '2010:110:18:59:46.600'), - (56310, 624826320, '2010:110:20:33:54.789'), (56310, 624826464, '2010:110:20:33:54.789'), - (56310, 624828488, '2010:110:20:33:54.789'), (56310, 624831328, '2010:110:20:33:54.789'), - (56310, 624831392, '2010:110:20:33:54.789'), (56310, 624954248, '2010:110:20:33:54.789'), - (56310, 624956216, '2010:110:20:33:54.789'), (56310, 625476960, '2010:110:20:33:54.789'), - (12203, 697581832, '2010:111:10:30:46.876'), (12203, 697963056, '2010:111:10:30:46.876'), - (12203, 697963288, '2010:111:10:30:46.876'), (12203, 697970576, '2010:111:10:30:46.876'), - (12203, 697973824, '2010:111:10:30:46.876'), (56308, 732697144, '2010:110:23:23:49.708'), - (56308, 732698416, '2010:110:23:23:49.708'), (56309, 762184312, '2010:110:22:02:30.780'), - (56309, 762184768, '2010:110:22:02:30.780'), (56309, 762185584, '2010:110:22:02:30.780'), - (56309, 762186016, '2010:110:22:02:30.780'), (56309, 762186080, '2010:110:22:02:30.780'), - (56309, 762191224, '2010:110:22:02:30.780'), (56309, 762579584, '2010:110:22:02:30.780'), - (56309, 762581024, '2010:110:22:02:30.780'), (56308, 806748432, '2010:110:23:23:49.708'), - (56308, 806748880, '2010:110:23:23:49.708'), (56308, 806750112, '2010:110:23:23:49.708'), - (56308, 806750408, '2010:110:23:23:49.708'), (56308, 806750912, '2010:110:23:23:49.708'), - (56308, 806751424, '2010:110:23:23:49.708'), (56306, 956708808, '2010:111:02:18:49.052'), - (56306, 957219128, '2010:111:02:18:49.052'), (56306, 957221200, '2010:111:02:18:49.052'), - (56306, 957222432, '2010:111:02:18:49.052'), (56306, 957229080, '2010:111:02:18:49.052'), - (56306, 957230976, '2010:111:02:18:49.052'), (56306, 957233920, '2010:111:02:18:49.052'), - (56306, 957369088, '2010:111:02:18:49.052'), (11849, 1019347720, '2010:111:12:26:54.536'), - (11849, 1019348536, '2010:111:12:26:54.536'), (11849, 1019350904, '2010:111:12:26:54.536'), - (11849, 1019354232, '2010:111:12:26:54.536'), (11849, 1019357032, '2010:111:12:26:54.536'), - (11980, 1198184872, '2010:111:03:21:11.299'), (11980, 1198190648, '2010:111:03:21:11.299'), - (11980, 1198190664, '2010:111:03:21:11.299'), (11980, 1198191400, '2010:111:03:21:11.299'), - (11980, 1198192456, '2010:111:03:21:11.299')], - dtype=[('obsid', ' 0, 'Table.write was never called' + print(status["obs"]) + update_supplement.update_obs_table( + TEST_DATA_DIR / "agasc_supplement_empty.h5", status["obs"], dry_run=False + ) + assert len(mock_write.calls) > 0, "Table.write was never called" def test_update_obs(monkeypatch, mock_open): - monkeypatch.setattr(star_obs_catalogs, 'STARS_OBS', STARS_OBS) + monkeypatch.setattr(star_obs_catalogs, "STARS_OBS", STARS_OBS) def mock_write(*args, **kwargs): mock_write.calls.append((args, kwargs)) - if 'path' in kwargs and kwargs['path'] == 'bad': + if "path" in kwargs and kwargs["path"] == "bad": mock_write.n_calls += 1 ref = table.Table() assert False - if 'path' in kwargs and kwargs['path'] == 'obs': + if "path" in kwargs and kwargs["path"] == "obs": mock_write.n_calls += 1 - ref = table.Table(np.array([ - (56311, 563087864, 1, '', '2010:110:18:59:46.600'), - (56311, 563088952, 1, '', '2010:110:18:59:46.600'), - (56311, 563089432, 1, '', '2010:110:18:59:46.600'), - (56311, 563091784, 1, '', '2010:110:18:59:46.600'), - (56311, 563092520, 1, '', '2010:110:18:59:46.600'), - (56311, 563612488, 1, '', '2010:110:18:59:46.600'), - (56311, 563612792, 1, '', '2010:110:18:59:46.600'), - (56311, 563617352, 1, '', '2010:110:18:59:46.600'), - (56308, 806750112, 0, '', '2010:110:23:23:49.708'), - (11849, 1019348536, 1, 'just removed them', '2010:111:12:26:54.536'), - (11849, 1019350904, 1, 'just removed them', '2010:111:12:26:54.536'), - (56314, 114950168, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 114950584, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 114952056, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 114952792, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 114952824, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 114955056, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 114956608, 1, 'removed because I felt like it', '2010:110:14:57:43.442'), - (56314, 115347520, 1, 'removed because I felt like it', '2010:110:14:57:43.442')], - dtype=[('obsid', ' 0, "Table.write was never called" def test_override(monkeypatch): _monkeypatch_star_obs_catalogs_( - monkeypatch, - test_file=TEST_DATA_DIR / 'mag-stats.h5', - path='/obs_status' + monkeypatch, test_file=TEST_DATA_DIR / "mag-stats.h5", path="/obs_status" ) telem = _monkeypatch_get_telemetry_( - monkeypatch, - test_file=TEST_DATA_DIR / 'mag-stats.h5', - path='/obs_status/telem' + monkeypatch, test_file=TEST_DATA_DIR / "mag-stats.h5", path="/obs_status/telem" ) # Case 1. There are two previously unknown suspect observations out of 5. @@ -760,59 +1225,63 @@ def test_override(monkeypatch): assert len(fails) == 2 assert len(obs_stats) == 5 - assert agasc_stats['n_obsids_fail'] == 2 - assert agasc_stats['n_obsids_ok'] == 3 - assert agasc_stats['n_obsids_suspect'] == 2 # two suspect count as "fail" in this context + assert agasc_stats["n_obsids_fail"] == 2 + assert agasc_stats["n_obsids_ok"] == 3 + assert ( + agasc_stats["n_obsids_suspect"] == 2 + ) # two suspect count as "fail" in this context assert not np.isclose( - np.mean(telem[np.in1d(telem['obsid'], [12800])]['mags']), - agasc_stats['mag_obs'] + np.mean(telem[np.in1d(telem["obsid"], [12800])]["mags"]), agasc_stats["mag_obs"] ) assert np.isclose( - np.mean(telem[np.in1d(telem['obsid'], [12800, 23682, 23683])]['mags']), - agasc_stats['mag_obs'] + np.mean(telem[np.in1d(telem["obsid"], [12800, 23682, 23683])]["mags"]), + agasc_stats["mag_obs"], ) # Case 2. Four observations (including the suspect) are marked with non-zero status # Only the remaining observation should be included obs_status_override = { - ('2018:296:15:53:14.596', 10492752): {'status': 1, 'comments': ''}, - ('2021:015:00:01:45.585', 10492752): {'status': 1, 'comments': ''}, - ('2021:089:02:48:00.575', 10492752): {'status': 1, 'comments': ''}, - ('2021:201:02:58:03.250', 10492752): {'status': 1, 'comments': ''} + ("2018:296:15:53:14.596", 10492752): {"status": 1, "comments": ""}, + ("2021:015:00:01:45.585", 10492752): {"status": 1, "comments": ""}, + ("2021:089:02:48:00.575", 10492752): {"status": 1, "comments": ""}, + ("2021:201:02:58:03.250", 10492752): {"status": 1, "comments": ""}, } - agasc_stats, obs_stats, fails = \ - mag_estimate.get_agasc_id_stats(10492752, obs_status_override=obs_status_override) + agasc_stats, obs_stats, fails = mag_estimate.get_agasc_id_stats( + 10492752, obs_status_override=obs_status_override + ) assert len(fails) == 0 - assert agasc_stats['n_obsids_fail'] == 0 - assert agasc_stats['n_obsids_ok'] == 1 - assert agasc_stats['n_obsids_suspect'] == 0 # no fails because all status==1 skipped + assert agasc_stats["n_obsids_fail"] == 0 + assert agasc_stats["n_obsids_ok"] == 1 + assert ( + agasc_stats["n_obsids_suspect"] == 0 + ) # no fails because all status==1 skipped assert np.isclose( - np.mean(telem[np.in1d(telem['obsid'], [12800])]['mags']), - agasc_stats['mag_obs'] + np.mean(telem[np.in1d(telem["obsid"], [12800])]["mags"]), agasc_stats["mag_obs"] ) assert not np.isclose( - np.mean(telem[np.in1d(telem['obsid'], [12800, 23682, 23683])]['mags']), - agasc_stats['mag_obs'] + np.mean(telem[np.in1d(telem["obsid"], [12800, 23682, 23683])]["mags"]), + agasc_stats["mag_obs"], ) # Case 3: # - One of the suspect observations is previously known and marked as OK (status=0) # - One other observation is marked as not-OK obs_status_override = { - ('2021:015:00:01:45.585', 10492752): {'status': 0, 'comments': ''}, - ('2021:089:02:48:00.575', 10492752): {'status': 1, 'comments': ''}, + ("2021:015:00:01:45.585", 10492752): {"status": 0, "comments": ""}, + ("2021:089:02:48:00.575", 10492752): {"status": 1, "comments": ""}, } - agasc_stats, obs_stats, fails = \ - mag_estimate.get_agasc_id_stats(10492752, obs_status_override=obs_status_override) + agasc_stats, obs_stats, fails = mag_estimate.get_agasc_id_stats( + 10492752, obs_status_override=obs_status_override + ) assert len(fails) == 1 - assert agasc_stats['n_obsids_fail'] == 1 - assert agasc_stats['n_obsids_ok'] == 3 - assert agasc_stats['n_obsids_suspect'] == 1 # one failed + assert agasc_stats["n_obsids_fail"] == 1 + assert agasc_stats["n_obsids_ok"] == 3 + assert agasc_stats["n_obsids_suspect"] == 1 # one failed assert np.isclose( - np.mean(telem[np.in1d(telem['obsid'], [12800, 23681, 23683])]['mags']), - agasc_stats['mag_obs'] + np.mean(telem[np.in1d(telem["obsid"], [12800, 23681, 23683])]["mags"]), + agasc_stats["mag_obs"], ) @@ -830,14 +1299,14 @@ def _remove_list_duplicates(a_list): def _monkeypatch_star_obs_catalogs_(monkeypatch, test_file, path): tables = [ - 'STARS_OBS', + "STARS_OBS", ] - res = {t: table.Table.read(test_file, path=f'{path}/cat/{t}') for t in tables} + res = {t: table.Table.read(test_file, path=f"{path}/cat/{t}") for t in tables} for k in res: res[k].convert_bytestring_to_unicode() - res['STARS_OBS'].add_index('agasc_id') - res['STARS_OBS'].add_index('mp_starcat_time') + res["STARS_OBS"].add_index("agasc_id") + res["STARS_OBS"].add_index("mp_starcat_time") for k in res: monkeypatch.setattr(star_obs_catalogs, k, res[k]) @@ -847,16 +1316,17 @@ def _monkeypatch_get_telemetry_(monkeypatch, test_file, path): telem = table.Table.read(test_file, path=path) def get_telemetry(obs): - obsid = obs['obsid'] - if obs['obsid'] in telem['obsid']: - return telem[telem['obsid'] == obs['obsid']] - raise Exception(f'{obsid=} not in test telemetry') - monkeypatch.setattr(mag_estimate, 'get_telemetry', get_telemetry) + obsid = obs["obsid"] + if obs["obsid"] in telem["obsid"]: + return telem[telem["obsid"] == obs["obsid"]] + raise Exception(f"{obsid=} not in test telemetry") + + monkeypatch.setattr(mag_estimate, "get_telemetry", get_telemetry) return telem -def recreate_mag_stats_test_data(filename=TEST_DATA_DIR / 'mag-stats.h5'): +def recreate_mag_stats_test_data(filename=TEST_DATA_DIR / "mag-stats.h5"): """ Create data to test mag-stats. @@ -874,23 +1344,23 @@ def recreate_mag_stats_test_data(filename=TEST_DATA_DIR / 'mag-stats.h5'): star_obs_catalogs.load() mp_starcat_time = [ - '2011:288:06:14:49.501', - '2021:015:00:01:45.585', - '2021:089:02:48:00.575', - '2021:201:02:58:03.250', - '2018:296:15:53:14.596', + "2011:288:06:14:49.501", + "2021:015:00:01:45.585", + "2021:089:02:48:00.575", + "2021:201:02:58:03.250", + "2018:296:15:53:14.596", ] STARS_OBS = star_obs_catalogs.STARS_OBS[ - np.in1d(star_obs_catalogs.STARS_OBS['mp_starcat_time'], mp_starcat_time) + np.in1d(star_obs_catalogs.STARS_OBS["mp_starcat_time"], mp_starcat_time) ] - STARS_OBS = STARS_OBS.group_by('agasc_id') - STARS_OBS.add_index('agasc_id') + STARS_OBS = STARS_OBS.group_by("agasc_id") + STARS_OBS.add_index("agasc_id") STARS_OBS.write( filename, - path='/obs_status/cat/STARS_OBS', + path="/obs_status/cat/STARS_OBS", serialize_meta=True, append=True, - overwrite=True + overwrite=True, ) telem = mag_estimate.get_telemetry_by_agasc_id(10492752) @@ -899,45 +1369,44 @@ def recreate_mag_stats_test_data(filename=TEST_DATA_DIR / 'mag-stats.h5'): # might be in maneuver mode or in acquisition. These times come from the kadi events v1 version, # but they do not matter much. telem_by_obsid = [ - telem[(telem['obsid'] == 12800) & (telem['times'] > 435047672.)][:100], + telem[(telem["obsid"] == 12800) & (telem["times"] > 435047672.0)][:100], # only 10 points, excluding the beginning - telem[(telem['obsid'] == 23681) & (telem['times'] > 727057549.)][:10], - telem[(telem['obsid'] == 23682) & (telem['times'] > 733462165.)][:100], - telem[(telem['obsid'] == 23683) & (telem['times'] > 743139160.)][:100], - telem[(telem['obsid'] == 48900) & (telem['times'] > 656698074.)][:100], + telem[(telem["obsid"] == 23681) & (telem["times"] > 727057549.0)][:10], + telem[(telem["obsid"] == 23682) & (telem["times"] > 733462165.0)][:100], + telem[(telem["obsid"] == 23683) & (telem["times"] > 743139160.0)][:100], + telem[(telem["obsid"] == 48900) & (telem["times"] > 656698074.0)][:100], ] - telem_by_obsid[-1]['mags_img'] += 0.01 * np.exp(np.arange(100) / 20) - telem_by_obsid[-1]['mags'] += 0.01 * np.exp(np.arange(100) / 20) + telem_by_obsid[-1]["mags_img"] += 0.01 * np.exp(np.arange(100) / 20) + telem_by_obsid[-1]["mags"] += 0.01 * np.exp(np.arange(100) / 20) t = vstack(telem_by_obsid) - t.write(filename, path='/obs_status/telem', serialize_meta=True, append=True) + t.write(filename, path="/obs_status/telem", serialize_meta=True, append=True) -def recreate_test_supplement(supplement_filename=TEST_DATA_DIR / 'agasc_supplement.h5'): +def recreate_test_supplement(supplement_filename=TEST_DATA_DIR / "agasc_supplement.h5"): # this is not a test function, but a function to generate the test supplement from scratch # whenever it needs updating, so all the data is actually contained in this file from _pytest.monkeypatch import MonkeyPatch + monkeypatch = MonkeyPatch() - monkeypatch.setitem(__builtins__, 'open', _open) - monkeypatch.setattr(star_obs_catalogs, 'STARS_OBS', STARS_OBS) + monkeypatch.setitem(__builtins__, "open", _open) + monkeypatch.setattr(star_obs_catalogs, "STARS_OBS", STARS_OBS) - with tables.open_file(str(supplement_filename), 'w'): + with tables.open_file(str(supplement_filename), "w"): pass - print(f'Updating {supplement_filename}') - status = update_supplement.parse_args(filename='file_0.yml') - print('obs') - print(status['obs']) - update_supplement.update_obs_table(supplement_filename, - status['obs'], - dry_run=False) - print('mags') - print(status['mags']) - update_supplement.update_mags_table(supplement_filename, - status['mags'], - dry_run=False) - print('bad') - print(status['bad']) - update_supplement.add_bad_star(supplement_filename, - status['bad'], - dry_run=False) + print(f"Updating {supplement_filename}") + status = update_supplement.parse_args(filename="file_0.yml") + print("obs") + print(status["obs"]) + update_supplement.update_obs_table( + supplement_filename, status["obs"], dry_run=False + ) + print("mags") + print(status["mags"]) + update_supplement.update_mags_table( + supplement_filename, status["mags"], dry_run=False + ) + print("bad") + print(status["bad"]) + update_supplement.add_bad_star(supplement_filename, status["bad"], dry_run=False) diff --git a/create_agasc_h5.py b/create_agasc_h5.py2 similarity index 100% rename from create_agasc_h5.py rename to create_agasc_h5.py2 diff --git a/create_derived_agasc_h5.py b/create_derived_agasc_h5.py index 57a3e2fb..bc48c013 100644 --- a/create_derived_agasc_h5.py +++ b/create_derived_agasc_h5.py @@ -40,10 +40,8 @@ import tables from astropy.table import Table -from agasc.healpix import get_healpix from agasc import default_agasc_dir - - +from agasc.healpix import get_healpix def get_parser(): @@ -69,7 +67,7 @@ def get_parser(): parser.add_argument( "--filter-faint", action="store_true", - help=('Filter: stars["MAG_ACA"] - 3.0 * stars["MAG_ACA_ERR"] / 100.0 < 11.5 '), + help='Filter: stars["MAG_ACA"] - 3.0 * stars["MAG_ACA_ERR"] / 100.0 < 11.5 ', ) parser.add_argument( "--proseco-columns", @@ -169,16 +167,16 @@ def write_derived_agasc(filename: str, stars: np.ndarray, version_num: str): def filter_proseco_columns(stars): print("Excluding columns not needed for proseco") # fmt: off - excludes = ['PLX', 'PLX_ERR', 'PLX_CATID', - 'ACQQ1', 'ACQQ2', 'ACQQ3', 'ACQQ4', 'ACQQ5', 'ACQQ6', - 'XREF_ID1', 'XREF_ID2', 'XREF_ID3', 'XREF_ID4', 'XREF_ID5', - 'RSV4', 'RSV5', 'RSV6', - 'POS_CATID', 'PM_CATID', - 'MAG', 'MAG_ERR', 'MAG_BAND', 'MAG_CATID', - 'COLOR1_ERR', 'C1_CATID', # Keep color1, 2, 3 - 'COLOR2_ERR', 'C2_CATID', - 'RSV2', - 'VAR_CATID'] + excludes = ["PLX", "PLX_ERR", "PLX_CATID", + "ACQQ1", "ACQQ2", "ACQQ3", "ACQQ4", "ACQQ5", "ACQQ6", + "XREF_ID1", "XREF_ID2", "XREF_ID3", "XREF_ID4", "XREF_ID5", + "RSV4", "RSV5", "RSV6", + "POS_CATID", "PM_CATID", + "MAG", "MAG_ERR", "MAG_BAND", "MAG_CATID", + "COLOR1_ERR", "C1_CATID", # Keep color1, 2, 3 + "COLOR2_ERR", "C2_CATID", + "RSV2", + "VAR_CATID"] # fmt: on names = [name for name in stars.dtype.names if name not in excludes] diff --git a/create_indexed_agasc_h5.py b/create_indexed_agasc_h5.py2 similarity index 100% rename from create_indexed_agasc_h5.py rename to create_indexed_agasc_h5.py2 diff --git a/create_near_neighbor_ids.py b/create_near_neighbor_ids.py index c41f2b99..7721159d 100644 --- a/create_near_neighbor_ids.py +++ b/create_near_neighbor_ids.py @@ -81,9 +81,7 @@ def main(): if id != sp["AGASC_ID"]: near_ids.add(id) - outfile = ( - Path(args.outdir) / f"{agasc_full.name[:-3]}_near_neighbor_ids.fits.gz" - ) + outfile = Path(args.outdir) / f"{agasc_full.name[:-3]}_near_neighbor_ids.fits.gz" t = Table([list(near_ids)], names=["near_id"]) print(f"Writing {len(t)} near-neighbor IDs to {outfile}") t.write(str(outfile), format="fits", overwrite=True) diff --git a/dev/profile_get_functions.py b/dev/profile_get_functions.py index 608e360e..79df7697 100644 --- a/dev/profile_get_functions.py +++ b/dev/profile_get_functions.py @@ -1,7 +1,9 @@ -import time -import numpy as np import argparse +import time from pathlib import Path + +import numpy as np + import agasc @@ -25,7 +27,7 @@ def get_ra_decs(): def time_get_agasc_cone(agasc_file, ras, decs, n_cone, cache=False): - kwargs = dict(radius=1.0, agasc_file=agasc_file) + kwargs = {"radius": 1.0, "agasc_file": agasc_file} if cache: kwargs["cache"] = True diff --git a/dev/profile_memory.py b/dev/profile_memory.py index 25a85f39..b40dab9f 100644 --- a/dev/profile_memory.py +++ b/dev/profile_memory.py @@ -6,18 +6,18 @@ import agasc -STD_INFO = dict( - att=(0, 0, 0), - detector="ACIS-S", - sim_offset=0, - focus_offset=0, - date="2018:001", - n_guide=5, - n_fid=3, - t_ccd=-11, - man_angle=90, - dither=8.0, -) +STD_INFO = { + "att": (0, 0, 0), + "detector": "ACIS-S", + "sim_offset": 0, + "focus_offset": 0, + "date": "2018:001", + "n_guide": 5, + "n_fid": 3, + "t_ccd": -11, + "man_angle": 90, + "dither": 8.0, +} print(f"agasc.__version__ = {agasc.__version__}") diff --git a/dev/profile_memory_psrecord.py b/dev/profile_memory_psrecord.py index 040f206a..635c7d4f 100644 --- a/dev/profile_memory_psrecord.py +++ b/dev/profile_memory_psrecord.py @@ -2,28 +2,17 @@ print("importing time") import time + t0 = time.time() + def wait_until_time(t1): t_now = time.time() time.sleep(t1 - (t_now - t0)) print(f"import other dependencies at {time.time() - t0:.2f} s") -import contextlib -import functools import os -import re -from packaging.version import Version -from pathlib import Path -from typing import Optional - -import numexpr -import numpy as np -import tables -from astropy.table import Column, Table -from Chandra.Time import DateTime -import tables wait_until_time(2.0) @@ -50,7 +39,6 @@ def wait_until_time(t1): wait_until_time(8.0) print(f"importing sparkles at {time.time() - t0:.2f} s") -import sparkles wait_until_time(10.0) diff --git a/pyproject.toml b/pyproject.toml index baa1ce0f..c2d93d72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,3 +5,21 @@ ignore = [ "**/*.ipynb", ".git" ] + +[tool.black] +include = '\.pyi?$' +exclude = ''' +/( + \.git + | \.mypy_cache + | \.tox + | \.venv + | \.vscode + | \.eggs + | _build + | buck-out + | build + | dist + | docs +)/ +''' diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000..49b41cc9 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,117 @@ +# Copy ruff settings from pandas +line-length = 100 +target-version = "py310" +# fix = true +unfixable = [] + +select = [ + # isort + "I", + # pyflakes + "F", + # pycodestyle + "E", "W", + # flake8-2020 + "YTT", + # flake8-bugbear + "B", + # flake8-quotes + "Q", + # flake8-debugger + "T10", + # flake8-gettext + "INT", + # pylint + "PLC", "PLE", "PLR", "PLW", + # misc lints + "PIE", + # flake8-pyi + "PYI", + # tidy imports + "TID", + # implicit string concatenation + "ISC", + # type-checking imports + "TCH", + # comprehensions + "C4", + # pygrep-hooks + "PGH" +] + +ignore = [ + # space before : (needed for how black formats slicing) + # "E203", # not yet implemented + # module level import not at top of file + "E402", + # do not assign a lambda expression, use a def + "E731", + # line break before binary operator + # "W503", # not yet implemented + # line break after binary operator + # "W504", # not yet implemented + # controversial + "B006", + # controversial?: Loop control variable not used within loop body + # "B007", + # controversial + "B008", + # setattr is used to side-step mypy + "B009", + # getattr is used to side-step mypy + "B010", + # tests use assert False + "B011", + # tests use comparisons but not their returned value + "B015", + # false positives + "B019", + # Loop control variable overrides iterable it iterates + "B020", + # Function definition does not bind loop variable + "B023", + # No explicit `stacklevel` keyword argument found + "B028", + # Functions defined inside a loop must not use variables redefined in the loop + # "B301", # not yet implemented + # Only works with python >=3.10 + "B905", + # Too many arguments to function call + "PLR0913", + # Too many returns + "PLR0911", + # Too many branches + "PLR0912", + # Too many statements + "PLR0915", + # Redefined loop name + "PLW2901", + # Global statements are discouraged + "PLW0603", + # Docstrings should not be included in stubs + "PYI021", + # No builtin `eval()` allowed + "PGH001", + # compare-to-empty-string + "PLC1901", + # Use typing_extensions.TypeAlias for type aliases + # "PYI026", # not yet implemented + # Use "collections.abc.*" instead of "typing.*" (PEP 585 syntax) + # "PYI027", # not yet implemented + # while int | float can be shortened to float, the former is more explicit + # "PYI041", # not yet implemented + + # Additional checks that don't pass yet + # Useless statement + "B018", + # Within an except clause, raise exceptions with ... + "B904", + # Magic number + "PLR2004", + # Consider `elif` instead of `else` then `if` to remove indentation level + "PLR5501", +] + +exclude = [ + "docs/", +] diff --git a/scripts/make_fits.py b/scripts/make_fits.py index 57ab7772..f8795e14 100644 --- a/scripts/make_fits.py +++ b/scripts/make_fits.py @@ -1,37 +1,39 @@ +import argparse import os from pathlib import Path + import numpy as np -import astropy.io.fits as fits +from astropy.io import fits from astropy.table import Table -import argparse - NEW_COMMENT = ( - ' ', - ' THE AXAF GUIDE and ACQUISITION STAR CATALOG V1.7', - ' ', - ' An all-sky astrometric and photometric catalog', - ' prepared for the operation of AXAF.', - ' ', - 'This file contains data for one of the 9537 regions constituting', - 'the AXAF Guide and Acquisition Star Catalog (AGASC) version 1.7', - 'Additional information on the AGASC may be obtained from the', - 'AXAF Science Center, Smithsonian Astrophysical Observatory,', - '60 Garden St., Cambridge, MA 02138, or in tables elsewhere', + " ", + " THE AXAF GUIDE and ACQUISITION STAR CATALOG V1.7", + " ", + " An all-sky astrometric and photometric catalog", + " prepared for the operation of AXAF.", + " ", + "This file contains data for one of the 9537 regions constituting", + "the AXAF Guide and Acquisition Star Catalog (AGASC) version 1.7", + "Additional information on the AGASC may be obtained from the", + "AXAF Science Center, Smithsonian Astrophysical Observatory,", + "60 Garden St., Cambridge, MA 02138, or in tables elsewhere", 'in this set of volumes (see the "comments" file).', - 'Null values are listed as -9999.', - ' ', - 'V1.7 changes values for MAG_ACA, MAG_ACA_ERR, RSV1, RSV2 and RSV3', - ' ', - ' ', - 'File written on Sept 5 2018', - ' ') + "Null values are listed as -9999.", + " ", + "V1.7 changes values for MAG_ACA, MAG_ACA_ERR, RSV1, RSV2 and RSV3", + " ", + " ", + "File written on Sept 5 2018", + " ", +) def get_options(): parser = argparse.ArgumentParser( - description="Make new AGASC fits files from old AGASC and new h5 source") - parser.add_argument("--h5", default='/proj/sot/ska/data/agasc/agasc1p7.h5') + description="Make new AGASC fits files from old AGASC and new h5 source" + ) + parser.add_argument("--h5", default="/proj/sot/ska/data/agasc/agasc1p7.h5") parser.add_argument("--olddir", default="/data/agasc1p6/agasc") parser.add_argument("--out", default="./testagasc") args = parser.parse_args() @@ -43,30 +45,30 @@ def update_file(src, dest, newdata): hdu = fits.open(src) hdr = hdu[0].header # Update AGASC VERSION - hdr['EXTVER'] = 17 + hdr["EXTVER"] = 17 for row in hdu[1].data: # Set RSV1 to be -9999.0 by default (for most stars this is a no-op, already -9999.0) - row['RSV1'] = -9999.0 + row["RSV1"] = -9999.0 # Find the index of the AGASC ID we have in the sorted table of newdata - idx = np.searchsorted(newdata['AGASC_ID'], row['AGASC_ID']) - if newdata['AGASC_ID'][idx] != row['AGASC_ID']: - idx = np.flatnonzero(newdata['AGASC_ID'] == row['AGASC_ID'])[0] + idx = np.searchsorted(newdata["AGASC_ID"], row["AGASC_ID"]) + if newdata["AGASC_ID"][idx] != row["AGASC_ID"]: + idx = np.flatnonzero(newdata["AGASC_ID"] == row["AGASC_ID"])[0] nrow = newdata[idx] - if nrow['RSV3'] != 1.0: + if nrow["RSV3"] != 1.0: continue - row['MAG_ACA'] = nrow['MAG_ACA'] - row['MAG_ACA_ERR'] = nrow['MAG_ACA_ERR'] - row['RSV1'] = nrow['RSV1'] - row['RSV2'] = nrow['RSV2'] - row['RSV3'] = nrow['RSV3'] + row["MAG_ACA"] = nrow["MAG_ACA"] + row["MAG_ACA_ERR"] = nrow["MAG_ACA_ERR"] + row["RSV1"] = nrow["RSV1"] + row["RSV2"] = nrow["RSV2"] + row["RSV3"] = nrow["RSV3"] # Set all previous lines to a colon (replacing with a space doesn't # seem to actually work) - hdr['comment'][0:19] = ":" + hdr["comment"][0:19] = ":" for idx, line in enumerate(NEW_COMMENT): - hdr['comment'][idx] = line + hdr["comment"][idx] = line # Update table comments for RSV1 RSV2 RSV3 hdu[1].header.comments["TTYPE26"] = "APASS V - i magnitude (COLOR3)" @@ -80,9 +82,9 @@ def update_file(src, dest, newdata): def main(h5file, srcdir, outdir): # This sorts the full agasc in memory, so don't do it on a puny machine tbl = Table.read(h5file) - tbl.sort('AGASC_ID') + tbl.sort("AGASC_ID") # redefine outdir as the agasc subdirectory in outdir - outdir = os.path.join(outdir, 'agasc') + outdir = os.path.join(outdir, "agasc") if not os.path.exists(outdir): os.makedirs(outdir) for topdir in Path(srcdir).glob("*"): @@ -93,7 +95,7 @@ def main(h5file, srcdir, outdir): newfile = newdir.joinpath(f.name) update_file(f, newfile, tbl) -if __name__ == '__main__': + +if __name__ == "__main__": args = get_options() main(args.h5, args.olddir, args.out) - diff --git a/scripts/make_links.py b/scripts/make_links.py index 587788ce..bf0c2dcf 100644 --- a/scripts/make_links.py +++ b/scripts/make_links.py @@ -1,9 +1,7 @@ import os from pathlib import Path -from Ska.Shell import bash - -for agasc_dir in Path('agasc').glob("*"): +for agasc_dir in Path("agasc").glob("*"): updir = agasc_dir.name.upper() if not os.path.exists(updir): os.makedirs(updir) @@ -12,4 +10,3 @@ newname = ffile.name.upper() os.symlink((".." / ffile), newname) os.chdir("..") - diff --git a/setup.py b/setup.py index da0c324b..12e97d4b 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -from setuptools import setup import os +from setuptools import setup + try: from testr.setup_helper import cmdclass except ImportError: @@ -9,38 +10,48 @@ entry_points = { - 'console_scripts': [ - 'agasc-magnitudes-report=agasc.scripts.mag_estimate_report:main', - 'agasc-update-magnitudes=agasc.scripts.update_mag_supplement:main', - 'agasc-update-supplement=agasc.scripts.update_supplement:main', - 'agasc-supplement-tasks=agasc.scripts.supplement_tasks:main', - 'agasc-supplement-diff=agasc.scripts.supplement_diff:main', + "console_scripts": [ + "agasc-magnitudes-report=agasc.scripts.mag_estimate_report:main", + "agasc-update-magnitudes=agasc.scripts.update_mag_supplement:main", + "agasc-update-supplement=agasc.scripts.update_supplement:main", + "agasc-supplement-tasks=agasc.scripts.supplement_tasks:main", + "agasc-supplement-diff=agasc.scripts.supplement_diff:main", ] } -data_files = [( - os.path.join('share', 'agasc'), - ['task_schedules/task_schedule_supplement_dispositions.cfg', - 'task_schedules/task_schedule_update_supplement_rc.cfg'] -)] - - -setup(name='agasc', - use_scm_version=True, - setup_requires=['setuptools_scm', 'setuptools_scm_git_archive'], - description='AGASC catalog access', - author='Jean Connelly, Tom Aldcroft', - author_email='taldcroft@cfa.harvard.edu', - url='http://cxc.harvard.edu/mta/ASPECT/tool_doc/agasc', - packages=['agasc', 'agasc.supplement', 'agasc.supplement.magnitudes', - 'agasc.tests', 'agasc.scripts'], - package_data={ - 'agasc.supplement.magnitudes': ['templates/*'], - 'agasc.tests': ['data/*'], - }, - data_files=data_files, - tests_require=['pytest'], - cmdclass=cmdclass, - entry_points=entry_points, - ) +data_files = [ + ( + os.path.join("share", "agasc"), + [ + "task_schedules/task_schedule_supplement_dispositions.cfg", + "task_schedules/task_schedule_update_supplement_rc.cfg", + ], + ) +] + + +setup( + name="agasc", + use_scm_version=True, + setup_requires=["setuptools_scm", "setuptools_scm_git_archive"], + description="AGASC catalog access", + author="Jean Connelly, Tom Aldcroft", + author_email="taldcroft@cfa.harvard.edu", + url="http://cxc.harvard.edu/mta/ASPECT/tool_doc/agasc", + packages=[ + "agasc", + "agasc.supplement", + "agasc.supplement.magnitudes", + "agasc.tests", + "agasc.scripts", + ], + package_data={ + "agasc.supplement.magnitudes": ["templates/*"], + "agasc.tests": ["data/*"], + }, + data_files=data_files, + tests_require=["pytest"], + cmdclass=cmdclass, + entry_points=entry_points, +) diff --git a/time_agasc.py b/time_agasc.py deleted file mode 100644 index 18c0edf2..00000000 --- a/time_agasc.py +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed under a 3-clause BSD style license - see LICENSE.rst -import sys -from itertools import count -import time -import numpy as np -import agasc - - -def random_ra_dec(nsample): - x = np.random.uniform(-0.98, 0.98, size=nsample) - ras = 360 * np.random.random(nsample) - decs = np.degrees(np.arcsin(x)) - return ras, decs - -radius = 2.0 -nsample = 200 -ras, decs = random_ra_dec(nsample) - -print 'get_agasc_cone' -t0 = time.time() -for ra, dec, cnt in zip(ras, decs, count()): - x = agasc.get_agasc_cone(ra, dec, radius=radius, agasc_file='miniagasc.h5') - print cnt, len(ras), '\r', - sys.stdout.flush() -print -print time.time() - t0 diff --git a/validate_agasc1p7.py b/validate_agasc1p7.py2 similarity index 100% rename from validate_agasc1p7.py rename to validate_agasc1p7.py2